natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[RPC] Refactor the remote module. (#14776) 

* Add dolphinscheduler-extract-base
* Add dolphinscheduler-extract-master
* Add dolphinscheduler-extract-worker
* Add dolphinscheduler-extract-alert
This commit is contained in:
Wenjun Ruan 2023-08-23 09:43:17 +08:00 committed by GitHub
parent 70731a1617
commit 55cc77226f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
391 changed files with 5917 additions and 12313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/labeler/labeler.yml vendored
View File

@ -25,7 +25,7 @@ backend:
- 'dolphinscheduler-dist/**/*'
- 'dolphinscheduler-master/**/*'
- 'dolphinscheduler-registry/**/*'
- 'dolphinscheduler-remote/**/*'
- 'dolphinscheduler-extract/**/*'
- 'dolphinscheduler-scheduler-plugin/**/*'
- 'dolphinscheduler-service/**/*'
- 'dolphinscheduler-spi/**/*'

10
dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
View File

@ -28,10 +28,7 @@
<name>${project.artifactId}</name>
<dependencies>
<!-- dolphinscheduler -->
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-remote</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-meter</artifactId>
@ -51,6 +48,11 @@
<artifactId>dolphinscheduler-registry-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-alert</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>

41
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRequestProcessor.java → dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertOperatorImpl.java
View File

@ -14,49 +14,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.alert.rpc;
import org.apache.dolphinscheduler.alert.service.AlertBootstrapService;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequest;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import org.apache.dolphinscheduler.extract.alert.IAlertOperator;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import io.netty.channel.Channel;
@Component
@Slf4j
public final class AlertRequestProcessor implements NettyRequestProcessor {
@Service
public class AlertOperatorImpl implements IAlertOperator {
private final AlertBootstrapService alertBootstrapService;
public AlertRequestProcessor(AlertBootstrapService alertBootstrapService) {
this.alertBootstrapService = alertBootstrapService;
}
@Autowired
private AlertBootstrapService alertBootstrapService;
@Override
public void process(Channel channel, Message message) {
AlertSendRequest alertSendRequest = JsonSerializer.deserialize(message.getBody(), AlertSendRequest.class);
log.info("Received command : {}", alertSendRequest);
public AlertSendResponse sendAlert(AlertSendRequest alertSendRequest) {
log.info("Received AlertSendRequest : {}", alertSendRequest);
AlertSendResponse alertSendResponse = alertBootstrapService.syncHandler(
alertSendRequest.getGroupId(),
alertSendRequest.getTitle(),
alertSendRequest.getContent(),
alertSendRequest.getWarnType());
channel.writeAndFlush(alertSendResponse.convert2Command(message.getOpaque()));
}
@Override
public MessageType getCommandType() {
return MessageType.ALERT_SEND_REQUEST;
log.info("Handle AlertSendRequest finish: {}", alertSendResponse);
return alertSendResponse;
}
}

35
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/rpc/AlertRpcServer.java
View File

@ -18,43 +18,32 @@
package org.apache.dolphinscheduler.alert.rpc;
import org.apache.dolphinscheduler.alert.config.AlertConfig;
import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.factory.NettyRemotingServerFactory;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import java.util.List;
import org.apache.dolphinscheduler.extract.base.NettyRemotingServer;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
import org.apache.dolphinscheduler.extract.base.server.SpringServerMethodInvokerDiscovery;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Slf4j
@Service
public class AlertRpcServer implements AutoCloseable {
public class AlertRpcServer extends SpringServerMethodInvokerDiscovery implements AutoCloseable {
@Autowired
private List<NettyRequestProcessor> nettyRequestProcessors;
@Autowired
private AlertConfig alertConfig;
private NettyRemotingServer nettyRemotingServer;
public AlertRpcServer(AlertConfig alertConfig) {
super(new NettyRemotingServer(new NettyServerConfig(alertConfig.getPort())));
}
public void start() {
log.info("Starting alert rpc server...");
nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(alertConfig.getPort());
for (NettyRequestProcessor nettyRequestProcessor : nettyRequestProcessors) {
nettyRemotingServer.registerProcessor(nettyRequestProcessor);
log.info("Success register netty processor: {}", nettyRequestProcessor.getClass().getName());
}
log.info("Starting AlertRpcServer...");
nettyRemotingServer.start();
log.info("Started alert rpc server...");
log.info("Started AlertRpcServer...");
}
@Override
public void close() throws Exception {
log.info("Closing alert rpc server...");
public void close() {
log.info("Closing AlertRpcServer...");
nettyRemotingServer.close();
log.info("Closed alert rpc server...");
log.info("Closed AlertRpcServer...");
}
}

2
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/service/AlertBootstrapService.java
View File

@ -37,7 +37,7 @@ import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.AlertSendStatus;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;

2
dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/runner/AlertBootstrapServiceTest.java
View File

@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponse;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse;
import java.util.ArrayList;
import java.util.List;

11
dolphinscheduler-api/pom.xml
View File

@ -37,6 +37,7 @@
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!-- dolphinscheduler -->
<dependency>
@ -82,6 +83,16 @@
<artifactId>dolphinscheduler-storage-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-master</artifactId>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-worker</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>

30
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/CacheEvictAspect.java
View File

@ -18,10 +18,15 @@
package org.apache.dolphinscheduler.api.aspect;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.remote.command.cache.CacheExpireRequest;
import org.apache.dolphinscheduler.service.cache.CacheNotifyService;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.IMasterCacheService;
import org.apache.dolphinscheduler.extract.master.transportor.CacheExpireRequest;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.service.cache.impl.CacheKeyGenerator;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Method;
@ -65,7 +70,7 @@ public class CacheEvictAspect {
private CacheKeyGenerator cacheKeyGenerator;
@Autowired
private CacheNotifyService cacheNotifyService;
private RegistryClient registryClient;
@Pointcut("@annotation(org.springframework.cache.annotation.CacheEvict)")
public void cacheEvictPointCut() {
@ -96,7 +101,7 @@ public class CacheEvictAspect {
}
}
if (StringUtils.isNotEmpty(cacheKey)) {
cacheNotifyService.notifyMaster(new CacheExpireRequest(cacheType, cacheKey).convert2Command());
notifyMaster(cacheType, cacheKey);
}
}
@ -134,4 +139,21 @@ public class CacheEvictAspect {
}
return obj.toString();
}
private void notifyMaster(CacheType cacheType, String cacheKey) {
try {
List<Server> serverList = registryClient.getServerList(RegistryNodeType.MASTER);
if (CollectionUtils.isEmpty(serverList)) {
return;
}
for (Server server : serverList) {
IMasterCacheService masterCacheService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(server.getHost() + ":" + server.getPort(), IMasterCacheService.class);
masterCacheService.cacheExpire(new CacheExpireRequest(cacheType, cacheKey));
}
} catch (Exception e) {
log.error("notify master error", e);
}
}
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ExecutorController.java
View File

@ -40,7 +40,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.commons.lang3.StringUtils;

27
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunction.java
View File

@ -20,24 +20,20 @@ package org.apache.dolphinscheduler.api.executor.workflow.instance.pause.pause;
import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.executor.ExecuteFunction;
import org.apache.dolphinscheduler.api.executor.ExecuteRuntimeException;
import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
public class PauseExecuteFunction implements ExecuteFunction<PauseExecuteRequest, PauseExecuteResult> {
private final ProcessInstanceDao processInstanceDao;
private final ApiRpcClient apiRpcClient;
public PauseExecuteFunction(ProcessInstanceDao processInstanceDao, ApiRpcClient apiRpcClient) {
public PauseExecuteFunction(ProcessInstanceDao processInstanceDao) {
this.processInstanceDao = processInstanceDao;
this.apiRpcClient = apiRpcClient;
}
@Override
@ -59,15 +55,18 @@ public class PauseExecuteFunction implements ExecuteFunction<PauseExecuteRequest
"The workflow instance: %s pause failed, due to update the workflow instance status in DB failed",
workflowInstance.getName()));
}
WorkflowStateEventChangeRequest workflowStateEventChangeRequest = new WorkflowStateEventChangeRequest(
workflowInstance.getId(), 0, workflowInstance.getState(), workflowInstance.getId(), 0);
try {
apiRpcClient.send(Host.of(workflowInstance.getHost()), workflowStateEventChangeRequest.convert2Command());
} catch (RemotingException e) {
// todo: direct call the workflow instance pause method
ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener =
SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(workflowInstance.getHost(), ITaskInstanceExecutionEventListener.class);
iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(
new WorkflowInstanceStateChangeEvent(
workflowInstance.getId(), 0, workflowInstance.getState(), workflowInstance.getId(), 0));
} catch (Exception e) {
throw new ExecuteRuntimeException(
String.format(
"The workflow instance: %s pause failed, due to send rpc request to master: %s failed",
workflowInstance.getName(), workflowInstance.getHost()),
"WorkflowInstance: %s pause failed", workflowInstance.getName()),
e);
}
return new PauseExecuteResult(workflowInstance);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/pause/pause/PauseExecuteFunctionBuilder.java
View File

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.executor.ExecuteContext;
import org.apache.dolphinscheduler.api.executor.ExecuteFunction;
import org.apache.dolphinscheduler.api.executor.ExecuteFunctionBuilder;
import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import java.util.concurrent.CompletableFuture;
@ -36,12 +35,10 @@ public class PauseExecuteFunctionBuilder implements ExecuteFunctionBuilder<Pause
@Autowired
private ProcessInstanceDao processInstanceDao;
@Autowired
private ApiRpcClient apiRpcClient;
@Override
public CompletableFuture<ExecuteFunction<PauseExecuteRequest, PauseExecuteResult>> createWorkflowInstanceExecuteFunction(ExecuteContext executeContext) {
return CompletableFuture.completedFuture(new PauseExecuteFunction(processInstanceDao, apiRpcClient));
return CompletableFuture.completedFuture(new PauseExecuteFunction(processInstanceDao));
}
@Override

30
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunction.java
View File

@ -20,14 +20,13 @@ package org.apache.dolphinscheduler.api.executor.workflow.instance.stop;
import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.executor.ExecuteFunction;
import org.apache.dolphinscheduler.api.executor.ExecuteRuntimeException;
import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import lombok.extern.slf4j.Slf4j;
@ -35,12 +34,9 @@ import lombok.extern.slf4j.Slf4j;
public class StopExecuteFunction implements ExecuteFunction<StopRequest, StopResult> {
private final ProcessInstanceDao processInstanceDao;
// todo: Use ApiRpcClient instead of NettyRemotingClient
private final ApiRpcClient apiRpcClient;
public StopExecuteFunction(ProcessInstanceDao processInstanceDao, ApiRpcClient apiRpcClient) {
public StopExecuteFunction(ProcessInstanceDao processInstanceDao) {
this.processInstanceDao = processInstanceDao;
this.apiRpcClient = apiRpcClient;
}
@Override
@ -60,17 +56,17 @@ public class StopExecuteFunction implements ExecuteFunction<StopRequest, StopRes
if (processInstanceDao.updateById(workflowInstance)) {
log.info("Workflow instance {} ready to stop success, will call master to stop the workflow instance",
workflowInstance.getName());
// todo: Use specific stop command instead of WorkflowStateEventChangeCommand
WorkflowStateEventChangeRequest workflowStateEventChangeRequest = new WorkflowStateEventChangeRequest(
workflowInstance.getId(), 0, workflowInstance.getState(), workflowInstance.getId(), 0);
try {
apiRpcClient.send(Host.of(workflowInstance.getHost()),
workflowStateEventChangeRequest.convert2Command());
} catch (RemotingException e) {
// todo: direct call the workflow instance stop method
ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener =
SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(workflowInstance.getHost(), ITaskInstanceExecutionEventListener.class);
iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(
new WorkflowInstanceStateChangeEvent(
workflowInstance.getId(), 0, workflowInstance.getState(), workflowInstance.getId(), 0));
} catch (Exception e) {
throw new ExecuteRuntimeException(
String.format("Workflow instance: %s stop failed, due to send request to master: %s failed",
workflowInstance.getName(), workflowInstance.getHost()),
e);
String.format("WorkflowInstance: %s stop failed", workflowInstance.getName()), e);
}
// todo: use async and inject the completeFuture in the result.
return new StopResult(workflowInstance);

5
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/executor/workflow/instance/stop/StopExecuteFunctionBuilder.java
View File

@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.executor.ExecuteContext;
import org.apache.dolphinscheduler.api.executor.ExecuteFunction;
import org.apache.dolphinscheduler.api.executor.ExecuteFunctionBuilder;
import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import java.util.concurrent.CompletableFuture;
@ -36,12 +35,10 @@ public class StopExecuteFunctionBuilder implements ExecuteFunctionBuilder<StopRe
@Autowired
private ProcessInstanceDao processInstanceDao;
@Autowired
private ApiRpcClient apiRpcClient;
@Override
public CompletableFuture<ExecuteFunction<StopRequest, StopResult>> createWorkflowInstanceExecuteFunction(ExecuteContext executeContext) {
return CompletableFuture.completedFuture(new StopExecuteFunction(processInstanceDao, apiRpcClient));
return CompletableFuture.completedFuture(new StopExecuteFunction(processInstanceDao));
}
@Override

13
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sClientService.java
View File

@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.api.k8s;
import org.apache.dolphinscheduler.dao.entity.K8sNamespace;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import java.util.Optional;
@ -43,16 +42,16 @@ public class K8sClientService {
private K8sManager k8sManager;
public ResourceQuota upsertNamespaceAndResourceToK8s(K8sNamespace k8sNamespace,
String yamlStr) throws RemotingException {
String yamlStr) {
if (!checkNamespaceToK8s(k8sNamespace.getNamespace(), k8sNamespace.getClusterCode())) {
throw new RemotingException(String.format(
throw new RuntimeException(String.format(
"namespace %s does not exist in k8s cluster, please create namespace in k8s cluster first",
k8sNamespace.getNamespace()));
}
return upsertNamespacedResourceToK8s(k8sNamespace, yamlStr);
}
public Optional<Namespace> deleteNamespaceToK8s(String name, Long clusterCode) throws RemotingException {
public Optional<Namespace> deleteNamespaceToK8s(String name, Long clusterCode) {
Optional<Namespace> result = getNamespaceFromK8s(name, clusterCode);
if (result.isPresent()) {
KubernetesClient client = k8sManager.getK8sClient(clusterCode);
@ -67,7 +66,7 @@ public class K8sClientService {
}
private ResourceQuota upsertNamespacedResourceToK8s(K8sNamespace k8sNamespace,
String yamlStr) throws RemotingException {
String yamlStr) {
KubernetesClient client = k8sManager.getK8sClient(k8sNamespace.getClusterCode());
@ -93,7 +92,7 @@ public class K8sClientService {
.createOrReplace(body);
}
private Optional<Namespace> getNamespaceFromK8s(String name, Long clusterCode) throws RemotingException {
private Optional<Namespace> getNamespaceFromK8s(String name, Long clusterCode) {
NamespaceList listNamespace =
k8sManager.getK8sClient(clusterCode).namespaces().list();
@ -105,7 +104,7 @@ public class K8sClientService {
return list;
}
private boolean checkNamespaceToK8s(String name, Long clusterCode) throws RemotingException {
private boolean checkNamespaceToK8s(String name, Long clusterCode) {
Optional<Namespace> result = getNamespaceFromK8s(name, clusterCode);
return result.isPresent();
}

15
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/k8s/K8sManager.java
View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.k8s;
import org.apache.dolphinscheduler.dao.entity.Cluster;
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.service.utils.ClusterConfUtils;
import java.util.Hashtable;
@ -55,7 +54,7 @@ public class K8sManager {
* @param clusterCode
* @return
*/
public synchronized KubernetesClient getK8sClient(Long clusterCode) throws RemotingException {
public synchronized KubernetesClient getK8sClient(Long clusterCode) {
if (null == clusterCode) {
return null;
}
@ -68,7 +67,7 @@ public class K8sManager {
* @return new client if need updated
*/
public synchronized KubernetesClient getAndUpdateK8sClient(Long clusterCode,
boolean update) throws RemotingException {
boolean update) {
if (null == clusterCode) {
return null;
}
@ -99,7 +98,7 @@ public class K8sManager {
}
}
private void createK8sClientInner(Long clusterCode) throws RemotingException {
private void createK8sClientInner(Long clusterCode) {
Cluster cluster = clusterMapper.queryByClusterCode(clusterCode);
if (cluster == null) {
return;
@ -111,20 +110,20 @@ public class K8sManager {
try {
client = getClient(k8sConfig);
clientMap.put(clusterCode, client);
} catch (RemotingException e) {
} catch (Exception e) {
log.error("cluster code ={},fail to get k8s ApiClient: {}", clusterCode, e.getMessage());
throw new RemotingException("fail to get k8s ApiClient:" + e.getMessage());
throw new RuntimeException("fail to get k8s ApiClient:" + e.getMessage());
}
}
}
private KubernetesClient getClient(String configYaml) throws RemotingException {
private KubernetesClient getClient(String configYaml) throws RuntimeException {
try {
Config config = Config.fromKubeconfig(configYaml);
return new KubernetesClientBuilder().withConfig(config).build();
} catch (Exception e) {
log.error("Fail to get k8s ApiClient", e);
throw new RemotingException("fail to get k8s ApiClient:" + e.getMessage());
throw new RuntimeException("fail to get k8s ApiClient:" + e.getMessage());
}
}

41
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/rpc/ApiRpcClient.java
View File

@ -1,41 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.api.rpc;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.springframework.stereotype.Component;
@Component
public class ApiRpcClient {
private final NettyRemotingClient nettyRemotingClient;
public ApiRpcClient() {
this.nettyRemotingClient = new NettyRemotingClient(new NettyClientConfig());
}
public void send(Host host, Message message) throws RemotingException {
nettyRemotingClient.send(host, message);
}
}

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ExecutorService.java
View File

@ -29,7 +29,7 @@ import org.apache.dolphinscheduler.common.enums.TaskDependType;
import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import java.util.Map;

2
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MetricsCleanUpService.java
View File

@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.api.service;
public interface MetricsCleanUpService {
void cleanUpWorkflowMetricsByDefinitionCode(String workflowDefinitionCode);
void cleanUpWorkflowMetricsByDefinitionCode(Long workflowDefinitionCode);
void cleanUpApiResponseTimeMetricsByUserId(int userId);

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ClusterServiceImpl.java
View File

@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.dao.entity.K8sNamespace;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.service.utils.ClusterConfUtils;
import org.apache.commons.collections4.CollectionUtils;
@ -321,7 +320,7 @@ public class ClusterServiceImpl extends BaseServiceImpl implements ClusterServic
&& !config.equals(ClusterConfUtils.getK8sConfig(clusterExist.getConfig()))) {
try {
k8sManager.getAndUpdateK8sClient(code, true);
} catch (RemotingException e) {
} catch (Exception e) {
log.error("Update K8s error.", e);
putMsg(result, Status.K8S_CLIENT_OPS_ERROR, name);
return result;

71
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
View File

@ -82,16 +82,17 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.ILogicTaskInstanceOperator;
import org.apache.dolphinscheduler.extract.master.IStreamingTaskOperator;
import org.apache.dolphinscheduler.extract.master.ITaskInstanceExecutionEventListener;
import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.remote.command.task.TaskExecuteStartMessage;
import org.apache.dolphinscheduler.remote.command.task.TaskForceStartRequest;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataRequest;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowExecutingDataResponse;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowStateEventChangeRequest;
import org.apache.dolphinscheduler.remote.dto.WorkflowExecuteDto;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.cron.CronUtils;
import org.apache.dolphinscheduler.service.exceptions.CronParseException;
@ -164,9 +165,6 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
@Autowired
private TaskDefinitionLogMapper taskDefinitionLogMapper;
@Autowired
private StateEventCallbackService stateEventCallbackService;
@Autowired
private TaskDefinitionMapper taskDefinitionMapper;
@ -655,10 +653,12 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
executionStatus.getDesc(), processInstance.getName());
// directly send the process instance state change event to target master, not guarantee the event send
// success
WorkflowStateEventChangeRequest workflowStateEventChangeRequest = new WorkflowStateEventChangeRequest(
WorkflowInstanceStateChangeEvent workflowStateEventChangeRequest = new WorkflowInstanceStateChangeEvent(
processInstance.getId(), 0, processInstance.getState(), processInstance.getId(), 0);
Host host = new Host(processInstance.getHost());
stateEventCallbackService.sendResult(host, workflowStateEventChangeRequest.convert2Command());
ITaskInstanceExecutionEventListener iTaskInstanceExecutionEventListener =
SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(processInstance.getHost(), ITaskInstanceExecutionEventListener.class);
iTaskInstanceExecutionEventListener.onWorkflowInstanceInstanceStateChange(workflowStateEventChangeRequest);
putMsg(result, Status.SUCCESS);
} else {
log.error("Process instance state update error, processInstanceName:{}.", processInstance.getName());
@ -684,9 +684,10 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
taskGroupQueue.setForceStart(Flag.YES.getCode());
processService.updateTaskGroupQueue(taskGroupQueue);
log.info("Sending force start command to master: {}.", processInstance.getHost());
stateEventCallbackService.sendResult(
Host.of(processInstance.getHost()),
new TaskForceStartRequest(processInstance.getId(), taskGroupQueue.getTaskId()).convert2Command());
ILogicTaskInstanceOperator iLogicTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(processInstance.getHost(), ILogicTaskInstanceOperator.class);
iLogicTaskInstanceOperator.forceStartTaskInstance(
new TaskInstanceForceStartRequest(processInstance.getId(), taskGroupQueue.getTaskId()));
putMsg(result, Status.SUCCESS);
return result;
}
@ -1151,19 +1152,9 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
log.error("Process instance does not exist, processInstanceId:{}.", processInstanceId);
return null;
}
Host host = new Host(processInstance.getHost());
WorkflowExecutingDataRequest requestCommand = new WorkflowExecutingDataRequest();
requestCommand.setProcessInstanceId(processInstanceId);
Message message =
stateEventCallbackService.sendSync(host, requestCommand.convert2Command());
if (message == null) {
log.error("Query executing process instance from master error, processInstanceId:{}.",
processInstanceId);
return null;
}
WorkflowExecutingDataResponse responseCommand =
JSONUtils.parseObject(message.getBody(), WorkflowExecutingDataResponse.class);
return responseCommand.getWorkflowExecuteDto();
IWorkflowInstanceService iWorkflowInstanceService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(processInstance.getHost(), IWorkflowInstanceService.class);
return iWorkflowInstanceService.getWorkflowExecutingData(processInstanceId);
}
@Override
@ -1183,9 +1174,9 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
checkMasterExists();
// todo dispatch improvement
List<Server> masterServerList = monitorService.getServerListFromRegistry(true);
Host host = new Host(masterServerList.get(0).getHost(), masterServerList.get(0).getPort());
Server server = masterServerList.get(0);
TaskExecuteStartMessage taskExecuteStartMessage = new TaskExecuteStartMessage();
StreamingTaskTriggerRequest taskExecuteStartMessage = new StreamingTaskTriggerRequest();
taskExecuteStartMessage.setExecutorId(loginUser.getId());
taskExecuteStartMessage.setExecutorName(loginUser.getUserName());
taskExecuteStartMessage.setProjectCode(projectCode);
@ -1198,15 +1189,17 @@ public class ExecutorServiceImpl extends BaseServiceImpl implements ExecutorServ
taskExecuteStartMessage.setStartParams(startParams);
taskExecuteStartMessage.setDryRun(dryRun);
Message response =
stateEventCallbackService.sendSync(host, taskExecuteStartMessage.convert2Command());
if (response != null) {
log.info("Send task execute start command complete, response is {}.", response);
IStreamingTaskOperator streamingTaskOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(server.getHost() + ":" + server.getPort(), IStreamingTaskOperator.class);
StreamingTaskTriggerResponse streamingTaskTriggerResponse =
streamingTaskOperator.triggerStreamingTask(taskExecuteStartMessage);
if (streamingTaskTriggerResponse.isSuccess()) {
log.info("Send task execute start command complete, response is {}.", streamingTaskOperator);
putMsg(result, Status.SUCCESS);
} else {
log.error(
"Start to execute stream task instance error, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}.",
projectCode, taskDefinitionCode, taskDefinitionVersion);
"Start to execute stream task instance error, projectCode:{}, taskDefinitionCode:{}, taskVersion:{}, response: {}.",
projectCode, taskDefinitionCode, taskDefinitionVersion, streamingTaskTriggerResponse);
putMsg(result, Status.START_TASK_INSTANCE_ERROR);
}
return result;

93
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
View File

@ -36,8 +36,18 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClient;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.IMasterLogService;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryResponse;
import org.apache.dolphinscheduler.extract.worker.IWorkerLogService;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceLogPageQueryResponse;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@ -67,9 +77,6 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
@Autowired
private TaskInstanceDao taskInstanceDao;
@Autowired
private LogClient logClient;
@Autowired
ProjectMapper projectMapper;
@ -203,25 +210,46 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
* @return log string data
*/
private String queryLog(TaskInstance taskInstance, int skipLineNum, int limit) {
Host host = Host.of(taskInstance.getHost());
String logPath = taskInstance.getLogPath();
log.info("Query task instance log, taskInstanceId:{}, taskInstanceName:{}, host:{}, logPath:{}, port:{}",
taskInstance.getId(), taskInstance.getName(), host.getIp(), logPath, host.getPort());
final String logPath = taskInstance.getLogPath();
final String host = taskInstance.getHost();
log.info("Query task instance log, taskInstanceId:{}, taskInstanceName:{}, host: {}, logPath:{}",
taskInstance.getId(), taskInstance.getName(), taskInstance.getHost(), logPath);
StringBuilder sb = new StringBuilder();
if (skipLineNum == 0) {
String head = String.format(LOG_HEAD_FORMAT,
logPath,
host,
taskInstance.getHost(),
Constants.SYSTEM_LINE_SEPARATOR);
sb.append(head);
}
String logContent = logClient
.rollViewLog(host.getIp(), host.getPort(), logPath, skipLineNum, limit);
if (skipLineNum == 0 && StringUtils.isEmpty(logContent) && RemoteLogUtils.isRemoteLoggingEnable()) {
String logContent = null;
if (TaskUtils.isLogicTask(taskInstance.getTaskType())) {
IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), IMasterLogService.class);
try {
LogicTaskInstanceLogPageQueryRequest logicTaskInstanceLogPageQueryRequest =
new LogicTaskInstanceLogPageQueryRequest(taskInstance.getId(), logPath, skipLineNum, limit);
LogicTaskInstanceLogPageQueryResponse logicTaskInstanceLogPageQueryResponse =
masterLogService.pageQueryLogicTaskInstanceLog(logicTaskInstanceLogPageQueryRequest);
logContent = logicTaskInstanceLogPageQueryResponse.getLogContent();
} catch (Exception ex) {
log.error("Query LogicTaskInstance log error", ex);
}
} else {
IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(host, IWorkerLogService.class);
try {
TaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest =
new TaskInstanceLogPageQueryRequest(taskInstance.getId(), logPath, skipLineNum, limit);
TaskInstanceLogPageQueryResponse taskInstanceLogPageQueryResponse =
iWorkerLogService.pageQueryTaskInstanceLog(taskInstanceLogPageQueryRequest);
logContent = taskInstanceLogPageQueryResponse.getLogContent();
} catch (Exception ex) {
log.error("Query LogicTaskInstance log error", ex);
}
}
if (logContent == null && RemoteLogUtils.isRemoteLoggingEnable()) {
// When getting the log for the first time (skipLineNum=0) returns empty, get the log from remote target
try {
log.info("Get log {} from remote target", logPath);
@ -233,9 +261,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
log.error("Error while getting log from remote target", e);
}
}
sb.append(logContent);
return sb.toString();
}
@ -246,7 +272,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
* @return log byte array
*/
private byte[] getLogBytes(TaskInstance taskInstance) {
Host host = Host.of(taskInstance.getHost());
String host = taskInstance.getHost();
String logPath = taskInstance.getLogPath();
byte[] head = String.format(LOG_HEAD_FORMAT,
@ -254,9 +280,34 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
host,
Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8);
byte[] logBytes = logClient.getLogBytes(host.getIp(), host.getPort(), logPath);
byte[] logBytes = new byte[0];
if (TaskUtils.isLogicTask(taskInstance.getTaskType())) {
IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), IMasterLogService.class);
try {
LogicTaskInstanceLogFileDownloadRequest logicTaskInstanceLogFileDownloadRequest =
new LogicTaskInstanceLogFileDownloadRequest(taskInstance.getId(), logPath);
LogicTaskInstanceLogFileDownloadResponse logicTaskInstanceLogFileDownloadResponse =
masterLogService.getLogicTaskInstanceWholeLogFileBytes(logicTaskInstanceLogFileDownloadRequest);
logBytes = logicTaskInstanceLogFileDownloadResponse.getLogBytes();
} catch (Exception ex) {
log.error("Query LogicTaskInstance log error", ex);
}
} else {
IWorkerLogService iWorkerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(host, IWorkerLogService.class);
try {
TaskInstanceLogFileDownloadRequest taskInstanceLogFileDownloadRequest =
new TaskInstanceLogFileDownloadRequest(taskInstance.getId(), logPath);
TaskInstanceLogFileDownloadResponse taskInstanceWholeLogFileBytes =
iWorkerLogService.getTaskInstanceWholeLogFileBytes(taskInstanceLogFileDownloadRequest);
logBytes = taskInstanceWholeLogFileBytes.getLogBytes();
} catch (Exception ex) {
log.error("Query LogicTaskInstance log error", ex);
}
}
if (logBytes.length == 0 && RemoteLogUtils.isRemoteLoggingEnable()) {
if ((logBytes == null || logBytes.length == 0) && RemoteLogUtils.isRemoteLoggingEnable()) {
// get task log from remote target
try {
log.info("Get log {} from remote target", logPath);

34
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MetricsCleanUpServiceImpl.java
View File

@ -18,13 +18,12 @@
package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics;
import org.apache.dolphinscheduler.api.rpc.ApiRpcClient;
import org.apache.dolphinscheduler.api.service.MetricsCleanUpService;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.IWorkflowInstanceService;
import org.apache.dolphinscheduler.registry.api.RegistryClient;
import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType;
import org.apache.dolphinscheduler.remote.command.workflow.WorkflowMetricsCleanUpRequest;
import org.apache.dolphinscheduler.remote.utils.Host;
import java.util.List;
@ -37,26 +36,27 @@ import org.springframework.stereotype.Service;
@Slf4j
public class MetricsCleanUpServiceImpl implements MetricsCleanUpService {
@Autowired
private ApiRpcClient apiRpcClient;
@Autowired
private RegistryClient registryClient;
@Override
public void cleanUpWorkflowMetricsByDefinitionCode(String workflowDefinitionCode) {
WorkflowMetricsCleanUpRequest workflowMetricsCleanUpRequest = new WorkflowMetricsCleanUpRequest();
workflowMetricsCleanUpRequest.setProcessDefinitionCode(workflowDefinitionCode);
public void cleanUpWorkflowMetricsByDefinitionCode(Long workflowDefinitionCode) {
List<Server> masterNodeList = registryClient.getServerList(RegistryNodeType.MASTER);
for (Server server : masterNodeList) {
try {
final String host = String.format("%s:%s", server.getHost(), server.getPort());
apiRpcClient.send(Host.of(host), workflowMetricsCleanUpRequest.convert2Command());
} catch (Exception e) {
log.error(
"Fail to clean up workflow related metrics on {} when deleting workflow definition {}, error message {}",
server.getHost(), workflowDefinitionCode, e.getMessage());
}
cleanUpWorkflowMetrics(server, workflowDefinitionCode);
}
}
private void cleanUpWorkflowMetrics(Server server, Long workflowDefinitionCode) {
try {
IWorkflowInstanceService iWorkflowInstanceService =
SingletonJdkDynamicRpcClientProxyFactory.getInstance().getProxyClient(
String.format("%s:%s", server.getHost(), server.getPort()), IWorkflowInstanceService.class);
iWorkflowInstanceService.clearWorkflowMetrics(workflowDefinitionCode);
} catch (Exception e) {
log.error(
"Fail to clean up workflow related metrics on {} when deleting workflow definition {}, error message {}",
server.getHost(), workflowDefinitionCode, e.getMessage());
}
}

4
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
View File

@ -984,7 +984,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
for (ProcessDefinition process : processDefinitionList) {
try {
this.deleteProcessDefinitionByCode(loginUser, process.getCode());
metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(process.getCode()));
metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(process.getCode());
} catch (Exception e) {
throw new ServiceException(Status.DELETE_PROCESS_DEFINE_ERROR, process.getName(), e.getMessage());
}
@ -1069,7 +1069,7 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
// we delete the workflow definition at last to avoid using transaction here.
// If delete error, we can call this interface again.
processDefinitionDao.deleteByWorkflowDefinitionCode(processDefinition.getCode());
metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(code));
metricsCleanUpService.cleanUpWorkflowMetricsByDefinitionCode(code);
log.info("Success delete workflow definition workflowDefinitionCode: {}", code);
}

54
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TaskInstanceServiceImpl.java
View File

@ -43,12 +43,17 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.repository.DqExecuteResultDao;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.dao.utils.TaskCacheUtils;
import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcClientProxyFactory;
import org.apache.dolphinscheduler.extract.master.IMasterLogService;
import org.apache.dolphinscheduler.extract.worker.IStreamingTaskInstanceOperator;
import org.apache.dolphinscheduler.extract.worker.ITaskInstanceOperator;
import org.apache.dolphinscheduler.extract.worker.IWorkerLogService;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceKillResponse;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointRequest;
import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstanceTriggerSavepointResponse;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.dolphinscheduler.remote.command.task.TaskKillRequest;
import org.apache.dolphinscheduler.remote.command.task.TaskSavePointRequest;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClient;
import org.apache.dolphinscheduler.plugin.task.api.utils.TaskUtils;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.commons.lang3.StringUtils;
@ -98,12 +103,6 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
@Autowired
TaskDefinitionMapper taskDefinitionMapper;
@Autowired
private StateEventCallbackService stateEventCallbackService;
@Autowired
private LogClient logClient;
@Autowired
private DqExecuteResultDao dqExecuteResultDao;
@ -290,13 +289,13 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
putMsg(result, Status.TASK_INSTANCE_NOT_FOUND);
return result;
}
TaskSavePointRequest command = new TaskSavePointRequest(taskInstanceId);
Host host = new Host(taskInstance.getHost());
stateEventCallbackService.sendResult(host, command.convert2Command());
IStreamingTaskInstanceOperator streamingTaskInstanceOperator =
SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), IStreamingTaskInstanceOperator.class);
TaskInstanceTriggerSavepointResponse taskInstanceTriggerSavepointResponse =
streamingTaskInstanceOperator.triggerSavepoint(new TaskInstanceTriggerSavepointRequest(taskInstanceId));
log.info("StreamingTaskInstance trigger savepoint response: {}", taskInstanceTriggerSavepointResponse);
putMsg(result, Status.SUCCESS);
return result;
}
@ -322,11 +321,14 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
return result;
}
TaskKillRequest command = new TaskKillRequest(taskInstanceId);
Host host = new Host(taskInstance.getHost());
stateEventCallbackService.sendResult(host, command.convert2Command());
putMsg(result, Status.SUCCESS);
// todo: we only support streaming task for now
ITaskInstanceOperator iTaskInstanceOperator = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), ITaskInstanceOperator.class);
TaskInstanceKillResponse taskInstanceKillResponse =
iTaskInstanceOperator.killTask(new TaskInstanceKillRequest(taskInstanceId));
log.info("TaskInstance kill response: {}", taskInstanceKillResponse);
putMsg(result, Status.SUCCESS);
return result;
}
@ -378,7 +380,15 @@ public class TaskInstanceServiceImpl extends BaseServiceImpl implements TaskInst
for (TaskInstance taskInstance : needToDeleteTaskInstances) {
// delete log
if (StringUtils.isNotEmpty(taskInstance.getLogPath())) {
logClient.removeTaskLog(Host.of(taskInstance.getHost()), taskInstance.getLogPath());
if (TaskUtils.isLogicTask(taskInstance.getTaskType())) {
IMasterLogService masterLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), IMasterLogService.class);
masterLogService.removeLogicTaskInstanceLog(taskInstance.getLogPath());
} else {
IWorkerLogService workerLogService = SingletonJdkDynamicRpcClientProxyFactory.getInstance()
.getProxyClient(taskInstance.getHost(), IWorkerLogService.class);
workerLogService.removeTaskInstanceLog(taskInstance.getLogPath());
}
}
}

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/k8s/K8sManagerTest.java
View File

@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.k8s;
import org.apache.dolphinscheduler.dao.entity.Cluster;
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import java.util.ArrayList;
import java.util.List;
@ -57,7 +56,7 @@ public class K8sManagerTest {
}
@Test
public void getK8sClient() throws RemotingException {
public void getK8sClient() {
Mockito.when(clusterMapper.selectList(Mockito.any())).thenReturn(getClusterList());
KubernetesClient result = k8sManager.getK8sClient(1L);

3
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ClusterServiceTest.java
View File

@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.dao.entity.Cluster;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ClusterMapper;
import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.commons.collections4.CollectionUtils;
@ -122,7 +121,7 @@ public class ClusterServiceTest {
}
@Test
public void testUpdateClusterByCode() throws RemotingException {
public void testUpdateClusterByCode() {
User loginUser = getGeneralUser();
Map<String, Object> result =
clusterService.updateClusterByCode(loginUser, 1L, clusterName, getConfig(), getDesc());

11
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ExecuteFunctionServiceTest.java
View File

@ -67,7 +67,6 @@ import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupQueueMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.repository.ProcessInstanceDao;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.service.command.CommandService;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.process.TriggerRelationService;
@ -161,9 +160,6 @@ public class ExecuteFunctionServiceTest {
@Mock
private ProcessDefinitionService processDefinitionService;
@Mock
private StateEventCallbackService stateEventCallbackService;
private int processDefinitionId = 1;
private int processDefinitionVersion = 1;
@ -258,13 +254,6 @@ public class ExecuteFunctionServiceTest {
.thenReturn(processTaskRelations);
}
@Test
public void testForceStartTaskInstance() {
Map<String, Object> result = executorService.forceStartTaskInstance(loginUser, taskQueueId);
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
}
/**
* not complement
*/

11
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/LoggerServiceTest.java
View File

@ -33,7 +33,6 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.repository.TaskInstanceDao;
import org.apache.dolphinscheduler.service.log.LogClient;
import java.text.MessageFormat;
import java.util.HashMap;
@ -75,9 +74,6 @@ public class LoggerServiceTest {
@Mock
private TaskDefinitionMapper taskDefinitionMapper;
@Mock
private LogClient logClient;
@Test
public void testQueryLog() {
@ -134,6 +130,7 @@ public class LoggerServiceTest {
User loginUser = new User();
loginUser.setId(1);
TaskInstance taskInstance = new TaskInstance();
taskInstance.setId(1);
taskInstance.setExecutorId(loginUser.getId() + 1);
Mockito.when(taskInstanceDao.queryById(1)).thenReturn(taskInstance);
@ -179,11 +176,9 @@ public class LoggerServiceTest {
// SUCCESS
Mockito.doNothing().when(projectService).checkProjectAndAuthThrowException(loginUser, project, DOWNLOAD_LOG);
Mockito.when(logClient.getLogBytes(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString()))
.thenReturn(new byte[0]);
Mockito.when(projectMapper.queryProjectByTaskInstanceId(1)).thenReturn(project);
byte[] result = loggerService.getLogBytes(loginUser, 1);
Assertions.assertEquals(62, result.length);
Assertions.assertEquals(47, result.length);
}
@Test
@ -238,8 +233,6 @@ public class LoggerServiceTest {
.thenReturn(result);
Mockito.when(taskInstanceDao.queryById(1)).thenReturn(taskInstance);
Mockito.when(taskDefinitionMapper.queryByCode(taskInstance.getTaskCode())).thenReturn(taskDefinition);
Mockito.when(logClient.getLogBytes(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString()))
.thenReturn(new byte[0]);
loggerService.getLogBytes(loginUser, projectCode, 1);
}

10
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionServiceTest.java
View File

@ -472,7 +472,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool {
@Test
public void deleteProcessDefinitionByCodeTest() {
Mockito.when(projectMapper.queryByCode(projectCode)).thenReturn(getProject(projectCode));
Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L));
Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(46L);
Project project = getProject(projectCode);
@ -520,7 +520,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool {
Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), processDefinition.getCode()))
.thenReturn(Collections.emptySet());
processDefinitionService.deleteProcessDefinitionByCode(user, 46L);
Mockito.verify(metricsCleanUpService, times(1)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L));
Mockito.verify(metricsCleanUpService, times(1)).cleanUpWorkflowMetricsByDefinitionCode(46L);
// scheduler online
Schedule schedule = getSchedule();
@ -547,7 +547,7 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool {
Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), processDefinition.getCode()))
.thenReturn(Collections.emptySet());
Assertions.assertDoesNotThrow(() -> processDefinitionService.deleteProcessDefinitionByCode(user, 46L));
Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(46L));
Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(46L);
}
@Test
@ -596,11 +596,11 @@ public class ProcessDefinitionServiceTest extends BaseServiceTestTool {
Mockito.when(workFlowLineageService.queryTaskDepOnProcess(project.getCode(), process.getCode()))
.thenReturn(Collections.emptySet());
putMsg(result, Status.SUCCESS, projectCode);
Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(11L));
Mockito.doNothing().when(metricsCleanUpService).cleanUpWorkflowMetricsByDefinitionCode(11L);
Map<String, Object> deleteSuccess =
processDefinitionService.batchDeleteProcessDefinitionByCodes(user, projectCode, singleCodes);
Assertions.assertEquals(Status.SUCCESS, deleteSuccess.get(Constants.STATUS));
Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(String.valueOf(11L));
Mockito.verify(metricsCleanUpService, times(2)).cleanUpWorkflowMetricsByDefinitionCode(11L);
}
@Test

10
dolphinscheduler-common/pom.xml
View File

@ -140,6 +140,16 @@
<artifactId>system-lambda</artifactId>
</dependency>
<!-- logback -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>

11
dolphinscheduler-dist/release-docs/LICENSE
View File

@ -222,7 +222,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
avro 1.7.7: https://github.com/apache/avro, Apache 2.0
bonecp 0.8.0.RELEASE: https://github.com/wwadge/bonecp, Apache 2.0
bucket4j-core 6.2.0: https://mvnrepository.com/artifact/com.github.vladimir-bukhtoyarov/bucket4j-core, Apache 2.0
byte-buddy 1.9.16: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.16, Apache 2.0
caffeine 2.9.3: https://mvnrepository.com/artifact/com.github.ben-manes.caffeine/caffeine/2.9.3, Apache 2.0
classmate 1.5.1: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.5.1, Apache 2.0
clickhouse-jdbc 0.4.6: https://mvnrepository.com/artifact/com.clickhouse/clickhouse-jdbc/0.4.6, Apache 2.0
@ -408,10 +407,6 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
woodstox-core 5.3.0: https://mvnrepository.com/artifact/com.fasterxml.woodstox/woodstox-core/5.3.0, Apache 2.0
zookeeper 3.8.0: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.8.0, Apache 2.0
presto-jdbc 0.238.1 https://mvnrepository.com/artifact/com.facebook.presto/presto-jdbc/0.238.1
protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0
protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0
protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0
protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0
prometheus client_java(simpleclient) 0.15.0: https://github.com/prometheus/client_java, Apache 2.0
snowflake snowflake-2010: https://github.com/twitter-archive/snowflake/tree/snowflake-2010, Apache 2.0
trino-jdbc 402: https://mvnrepository.com/artifact/io.trino/trino-jdbc/402, Apache 2.0
@ -712,12 +707,6 @@ The following components are provided under a MPL 1.1 license. See project link
The text of each license is also included at licenses/LICENSE-[project].txt.
jamon-runtime 2.3.1: https://mvnrepository.com/artifact/org.jamon/jamon-runtime/2.3.1, MPL-1.1
javassist 3.27.0-GA: https://github.com/jboss-javassist/javassist, MPL-1.1
========================================
WTFPL License
========================================
reflections 0.9.12: https://github.com/ronmamo/reflections WTFPL
========================================
CC0-1.0 licenses

28
dolphinscheduler-dist/release-docs/NOTICE
View File

@ -515,34 +515,6 @@ The Apache Software Foundation (http://www.apache.org/).
--------------------------------------------------------------------------------
==============================================================
protostuff
Copyright 2009 David Yu dyuproject@gmail.com
==============================================================
protobuf is copyright Google inc unless otherwise noted.
It is licensed under the BSD license.
jackson-core-asl is copyright FasterXml unless otherwise noted.
It is licensed under the apache 2.0 license.
antlr is copyright Terence Parr unless otherwise noted.
It is licensed under the BSD license.
stringtemplate is copyright Terence Parr unless otherwise noted.
It is licensed under the BSD license.
velocity is licensed under the apache 2.0 license.
B64Code.java is copyright Mort Bay Consulting Pty Ltd unless otherwise noted.
It is licensed under the apache 2.0 license.
jarjar is copyright Google inc unless otherwise noted.
It is licensed under the apache 2.0 license.
guava is copyright Google inc unless otherwise noted.
It is licensed under the apache 2.0 license.
This product includes parquet-tools, initially developed at ARRIS, Inc. with
the following copyright notice:

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-byte-buddy.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

469
dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt
View File

@ -1,469 +0,0 @@
MOZILLA PUBLIC LICENSE
Version 1.1
---------------
1. Definitions.
1.0.1. "Commercial Use" means distribution or otherwise making the
Covered Code available to a third party.
1.1. "Contributor" means each entity that creates or contributes to
the creation of Modifications.
1.2. "Contributor Version" means the combination of the Original
Code, prior Modifications used by a Contributor, and the Modifications
made by that particular Contributor.
1.3. "Covered Code" means the Original Code or Modifications or the
combination of the Original Code and Modifications, in each case
including portions thereof.
1.4. "Electronic Distribution Mechanism" means a mechanism generally
accepted in the software development community for the electronic
transfer of data.
1.5. "Executable" means Covered Code in any form other than Source
Code.
1.6. "Initial Developer" means the individual or entity identified
as the Initial Developer in the Source Code notice required by Exhibit
A.
1.7. "Larger Work" means a work which combines Covered Code or
portions thereof with code not governed by the terms of this License.
1.8. "License" means this document.
1.8.1. "Licensable" means having the right to grant, to the maximum
extent possible, whether at the time of the initial grant or
subsequently acquired, any and all of the rights conveyed herein.
1.9. "Modifications" means any addition to or deletion from the
substance or structure of either the Original Code or any previous
Modifications. When Covered Code is released as a series of files, a
Modification is:
A. Any addition to or deletion from the contents of a file
containing Original Code or previous Modifications.
B. Any new file that contains any part of the Original Code or
previous Modifications.
1.10. "Original Code" means Source Code of computer software code
which is described in the Source Code notice required by Exhibit A as
Original Code, and which, at the time of its release under this
License is not already Covered Code governed by this License.
1.10.1. "Patent Claims" means any patent claim(s), now owned or
hereafter acquired, including without limitation, method, process,
and apparatus claims, in any patent Licensable by grantor.
1.11. "Source Code" means the preferred form of the Covered Code for
making modifications to it, including all modules it contains, plus
any associated interface definition files, scripts used to control
compilation and installation of an Executable, or source code
differential comparisons against either the Original Code or another
well known, available Covered Code of the Contributor's choice. The
Source Code can be in a compressed or archival form, provided the
appropriate decompression or de-archiving software is widely available
for no charge.
1.12. "You" (or "Your") means an individual or a legal entity
exercising rights under, and complying with all of the terms of, this
License or a future version of this License issued under Section 6.1.
For legal entities, "You" includes any entity which controls, is
controlled by, or is under common control with You. For purposes of
this definition, "control" means (a) the power, direct or indirect,
to cause the direction or management of such entity, whether by
contract or otherwise, or (b) ownership of more than fifty percent
(50%) of the outstanding shares or beneficial ownership of such
entity.
2. Source Code License.
2.1. The Initial Developer Grant.
The Initial Developer hereby grants You a world-wide, royalty-free,
non-exclusive license, subject to third party intellectual property
claims:
(a) under intellectual property rights (other than patent or
trademark) Licensable by Initial Developer to use, reproduce,
modify, display, perform, sublicense and distribute the Original
Code (or portions thereof) with or without Modifications, and/or
as part of a Larger Work; and
(b) under Patents Claims infringed by the making, using or
selling of Original Code, to make, have made, use, practice,
sell, and offer for sale, and/or otherwise dispose of the
Original Code (or portions thereof).
(c) the licenses granted in this Section 2.1(a) and (b) are
effective on the date Initial Developer first distributes
Original Code under the terms of this License.
(d) Notwithstanding Section 2.1(b) above, no patent license is
granted: 1) for code that You delete from the Original Code; 2)
separate from the Original Code; or 3) for infringements caused
by: i) the modification of the Original Code or ii) the
combination of the Original Code with other software or devices.
2.2. Contributor Grant.
Subject to third party intellectual property claims, each Contributor
hereby grants You a world-wide, royalty-free, non-exclusive license
(a) under intellectual property rights (other than patent or
trademark) Licensable by Contributor, to use, reproduce, modify,
display, perform, sublicense and distribute the Modifications
created by such Contributor (or portions thereof) either on an
unmodified basis, with other Modifications, as Covered Code
and/or as part of a Larger Work; and
(b) under Patent Claims infringed by the making, using, or
selling of Modifications made by that Contributor either alone
and/or in combination with its Contributor Version (or portions
of such combination), to make, use, sell, offer for sale, have
made, and/or otherwise dispose of: 1) Modifications made by that
Contributor (or portions thereof); and 2) the combination of
Modifications made by that Contributor with its Contributor
Version (or portions of such combination).
(c) the licenses granted in Sections 2.2(a) and 2.2(b) are
effective on the date Contributor first makes Commercial Use of
the Covered Code.
(d) Notwithstanding Section 2.2(b) above, no patent license is
granted: 1) for any code that Contributor has deleted from the
Contributor Version; 2) separate from the Contributor Version;
3) for infringements caused by: i) third party modifications of
Contributor Version or ii) the combination of Modifications made
by that Contributor with other software (except as part of the
Contributor Version) or other devices; or 4) under Patent Claims
infringed by Covered Code in the absence of Modifications made by
that Contributor.
3. Distribution Obligations.
3.1. Application of License.
The Modifications which You create or to which You contribute are
governed by the terms of this License, including without limitation
Section 2.2. The Source Code version of Covered Code may be
distributed only under the terms of this License or a future version
of this License released under Section 6.1, and You must include a
copy of this License with every copy of the Source Code You
distribute. You may not offer or impose any terms on any Source Code
version that alters or restricts the applicable version of this
License or the recipients' rights hereunder. However, You may include
an additional document offering the additional rights described in
Section 3.5.
3.2. Availability of Source Code.
Any Modification which You create or to which You contribute must be
made available in Source Code form under the terms of this License
either on the same media as an Executable version or via an accepted
Electronic Distribution Mechanism to anyone to whom you made an
Executable version available; and if made available via Electronic
Distribution Mechanism, must remain available for at least twelve (12)
months after the date it initially became available, or at least six
(6) months after a subsequent version of that particular Modification
has been made available to such recipients. You are responsible for
ensuring that the Source Code version remains available even if the
Electronic Distribution Mechanism is maintained by a third party.
3.3. Description of Modifications.
You must cause all Covered Code to which You contribute to contain a
file documenting the changes You made to create that Covered Code and
the date of any change. You must include a prominent statement that
the Modification is derived, directly or indirectly, from Original
Code provided by the Initial Developer and including the name of the
Initial Developer in (a) the Source Code, and (b) in any notice in an
Executable version or related documentation in which You describe the
origin or ownership of the Covered Code.
3.4. Intellectual Property Matters
(a) Third Party Claims.
If Contributor has knowledge that a license under a third party's
intellectual property rights is required to exercise the rights
granted by such Contributor under Sections 2.1 or 2.2,
Contributor must include a text file with the Source Code
distribution titled "LEGAL" which describes the claim and the
party making the claim in sufficient detail that a recipient will
know whom to contact. If Contributor obtains such knowledge after
the Modification is made available as described in Section 3.2,
Contributor shall promptly modify the LEGAL file in all copies
Contributor makes available thereafter and shall take other steps
(such as notifying appropriate mailing lists or newsgroups)
reasonably calculated to inform those who received the Covered
Code that new knowledge has been obtained.
(b) Contributor APIs.
If Contributor's Modifications include an application programming
interface and Contributor has knowledge of patent licenses which
are reasonably necessary to implement that API, Contributor must
also include this information in the LEGAL file.
(c) Representations.
Contributor represents that, except as disclosed pursuant to
Section 3.4(a) above, Contributor believes that Contributor's
Modifications are Contributor's original creation(s) and/or
Contributor has sufficient rights to grant the rights conveyed by
this License.
3.5. Required Notices.
You must duplicate the notice in Exhibit A in each file of the Source
Code. If it is not possible to put such notice in a particular Source
Code file due to its structure, then You must include such notice in a
location (such as a relevant directory) where a user would be likely
to look for such a notice. If You created one or more Modification(s)
You may add your name as a Contributor to the notice described in
Exhibit A. You must also duplicate this License in any documentation
for the Source Code where You describe recipients' rights or ownership
rights relating to Covered Code. You may choose to offer, and to
charge a fee for, warranty, support, indemnity or liability
obligations to one or more recipients of Covered Code. However, You
may do so only on Your own behalf, and not on behalf of the Initial
Developer or any Contributor. You must make it absolutely clear than
any such warranty, support, indemnity or liability obligation is
offered by You alone, and You hereby agree to indemnify the Initial
Developer and every Contributor for any liability incurred by the
Initial Developer or such Contributor as a result of warranty,
support, indemnity or liability terms You offer.
3.6. Distribution of Executable Versions.
You may distribute Covered Code in Executable form only if the
requirements of Section 3.1-3.5 have been met for that Covered Code,
and if You include a notice stating that the Source Code version of
the Covered Code is available under the terms of this License,
including a description of how and where You have fulfilled the
obligations of Section 3.2. The notice must be conspicuously included
in any notice in an Executable version, related documentation or
collateral in which You describe recipients' rights relating to the
Covered Code. You may distribute the Executable version of Covered
Code or ownership rights under a license of Your choice, which may
contain terms different from this License, provided that You are in
compliance with the terms of this License and that the license for the
Executable version does not attempt to limit or alter the recipient's
rights in the Source Code version from the rights set forth in this
License. If You distribute the Executable version under a different
license You must make it absolutely clear that any terms which differ
from this License are offered by You alone, not by the Initial
Developer or any Contributor. You hereby agree to indemnify the
Initial Developer and every Contributor for any liability incurred by
the Initial Developer or such Contributor as a result of any such
terms You offer.
3.7. Larger Works.
You may create a Larger Work by combining Covered Code with other code
not governed by the terms of this License and distribute the Larger
Work as a single product. In such a case, You must make sure the
requirements of this License are fulfilled for the Covered Code.
4. Inability to Comply Due to Statute or Regulation.
If it is impossible for You to comply with any of the terms of this
License with respect to some or all of the Covered Code due to
statute, judicial order, or regulation then You must: (a) comply with
the terms of this License to the maximum extent possible; and (b)
describe the limitations and the code they affect. Such description
must be included in the LEGAL file described in Section 3.4 and must
be included with all distributions of the Source Code. Except to the
extent prohibited by statute or regulation, such description must be
sufficiently detailed for a recipient of ordinary skill to be able to
understand it.
5. Application of this License.
This License applies to code to which the Initial Developer has
attached the notice in Exhibit A and to related Covered Code.
6. Versions of the License.
6.1. New Versions.
Netscape Communications Corporation ("Netscape") may publish revised
and/or new versions of the License from time to time. Each version
will be given a distinguishing version number.
6.2. Effect of New Versions.
Once Covered Code has been published under a particular version of the
License, You may always continue to use it under the terms of that
version. You may also choose to use such Covered Code under the terms
of any subsequent version of the License published by Netscape. No one
other than Netscape has the right to modify the terms applicable to
Covered Code created under this License.
6.3. Derivative Works.
If You create or use a modified version of this License (which you may
only do in order to apply it to code which is not already Covered Code
governed by this License), You must (a) rename Your license so that
the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
"MPL", "NPL" or any confusingly similar phrase do not appear in your
license (except to note that your license differs from this License)
and (b) otherwise make it clear that Your version of the license
contains terms which differ from the Mozilla Public License and
Netscape Public License. (Filling in the name of the Initial
Developer, Original Code or Contributor in the notice described in
Exhibit A shall not of themselves be deemed to be modifications of
this License.)
7. DISCLAIMER OF WARRANTY.
COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
8. TERMINATION.
8.1. This License and the rights granted hereunder will terminate
automatically if You fail to comply with terms herein and fail to cure
such breach within 30 days of becoming aware of the breach. All
sublicenses to the Covered Code which are properly granted shall
survive any termination of this License. Provisions which, by their
nature, must remain in effect beyond the termination of this License
shall survive.
8.2. If You initiate litigation by asserting a patent infringement
claim (excluding declatory judgment actions) against Initial Developer
or a Contributor (the Initial Developer or Contributor against whom
You file such action is referred to as "Participant") alleging that:
(a) such Participant's Contributor Version directly or indirectly
infringes any patent, then any and all rights granted by such
Participant to You under Sections 2.1 and/or 2.2 of this License
shall, upon 60 days notice from Participant terminate prospectively,
unless if within 60 days after receipt of notice You either: (i)
agree in writing to pay Participant a mutually agreeable reasonable
royalty for Your past and future use of Modifications made by such
Participant, or (ii) withdraw Your litigation claim with respect to
the Contributor Version against such Participant. If within 60 days
of notice, a reasonable royalty and payment arrangement are not
mutually agreed upon in writing by the parties or the litigation claim
is not withdrawn, the rights granted by Participant to You under
Sections 2.1 and/or 2.2 automatically terminate at the expiration of
the 60 day notice period specified above.
(b) any software, hardware, or device, other than such Participant's
Contributor Version, directly or indirectly infringes any patent, then
any rights granted to You by such Participant under Sections 2.1(b)
and 2.2(b) are revoked effective as of the date You first made, used,
sold, distributed, or had made, Modifications made by that
Participant.
8.3. If You assert a patent infringement claim against Participant
alleging that such Participant's Contributor Version directly or
indirectly infringes any patent where such claim is resolved (such as
by license or settlement) prior to the initiation of patent
infringement litigation, then the reasonable value of the licenses
granted by such Participant under Sections 2.1 or 2.2 shall be taken
into account in determining the amount or value of any payment or
license.
8.4. In the event of termination under Sections 8.1 or 8.2 above,
all end user license agreements (excluding distributors and resellers)
which have been validly granted by You or any distributor hereunder
prior to termination shall survive termination.
9. LIMITATION OF LIABILITY.
UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
10. U.S. GOVERNMENT END USERS.
The Covered Code is a "commercial item," as that term is defined in
48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
software" and "commercial computer software documentation," as such
terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
all U.S. Government End Users acquire Covered Code with only those
rights set forth herein.
11. MISCELLANEOUS.
This License represents the complete agreement concerning subject
matter hereof. If any provision of this License is held to be
unenforceable, such provision shall be reformed only to the extent
necessary to make it enforceable. This License shall be governed by
California law provisions (except to the extent applicable law, if
any, provides otherwise), excluding its conflict-of-law provisions.
With respect to disputes in which at least one party is a citizen of,
or an entity chartered or registered to do business in the United
States of America, any litigation relating to this License shall be
subject to the jurisdiction of the Federal Courts of the Northern
District of California, with venue lying in Santa Clara County,
California, with the losing party responsible for costs, including
without limitation, court costs and reasonable attorneys' fees and
expenses. The application of the United Nations Convention on
Contracts for the International Sale of Goods is expressly excluded.
Any law or regulation which provides that the language of a contract
shall be construed against the drafter shall not apply to this
License.
12. RESPONSIBILITY FOR CLAIMS.
As between Initial Developer and the Contributors, each party is
responsible for claims and damages arising, directly or indirectly,
out of its utilization of rights under this License and You agree to
work with Initial Developer and Contributors to distribute such
responsibility on an equitable basis. Nothing herein is intended or
shall be deemed to constitute any admission of liability.
13. MULTIPLE-LICENSED CODE.
Initial Developer may designate portions of the Covered Code as
"Multiple-Licensed". "Multiple-Licensed" means that the Initial
Developer permits you to utilize portions of the Covered Code under
Your choice of the MPL or the alternative licenses, if any, specified
by the Initial Developer in the file described in Exhibit A.
EXHIBIT A -Mozilla Public License.
``The contents of this file are subject to the Mozilla Public License
Version 1.1 (the "License"); you may not use this file except in
compliance with the License. You may obtain a copy of the License at
https://www.mozilla.org/MPL/
Software distributed under the License is distributed on an "AS IS"
basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
License for the specific language governing rights and limitations
under the License.
The Original Code is ______________________________________.
The Initial Developer of the Original Code is ________________________.
Portions created by ______________________ are Copyright (C) ______
_______________________. All Rights Reserved.
Contributor(s): ______________________________________.
Alternatively, the contents of this file may be used under the terms
of the _____ license (the "[___] License"), in which case the
provisions of [______] License are applicable instead of those
above. If you wish to allow use of your version of this file only
under the terms of the [____] License and not to allow others to use
your version of this file under the MPL, indicate your decision by
deleting the provisions above and replace them with the notice and
other provisions required by the [___] License. If you do not delete
the provisions above, a recipient may use your version of this file
under either the MPL or the [___] License."
[NOTE: The text of this Exhibit A may differ slightly from the text of
the notices in the Source Code files of the Original Code. You should
use the text of this Exhibit A rather than the text found in the
Original Code Source Code for Your Modifications.]

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

202
dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt
View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

13
dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt
View File

@ -1,13 +0,0 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.

39
dolphinscheduler-extract/dolphinscheduler-extract-alert/pom.xml Normal file
View File

@ -0,0 +1,39 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-extract-alert</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-base</artifactId>
</dependency>
</dependencies>
</project>

31
dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/IAlertOperator.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.alert;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendRequest;
import org.apache.dolphinscheduler.extract.alert.request.AlertSendResponse;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
@RpcService
public interface IAlertOperator {
@RpcMethod
AlertSendResponse sendAlert(AlertSendRequest alertSendRequest);
}

11
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendRequest.java
View File

@ -15,10 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.alert;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
package org.apache.dolphinscheduler.extract.alert.request;
import lombok.AllArgsConstructor;
import lombok.Data;
@ -27,7 +24,7 @@ import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AlertSendRequest implements RequestMessageBuilder {
public class AlertSendRequest {
private int groupId;
@ -37,8 +34,4 @@ public class AlertSendRequest implements RequestMessageBuilder {
private int warnType;
@Override
public MessageType getCommandType() {
return MessageType.ALERT_SEND_REQUEST;
}
}

13
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/alert/AlertSendResponse.java → dolphinscheduler-extract/dolphinscheduler-extract-alert/src/main/java/org/apache/dolphinscheduler/extract/alert/request/AlertSendResponse.java
View File

@ -15,10 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.alert;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder;
package org.apache.dolphinscheduler.extract.alert.request;
import java.io.Serializable;
import java.util.List;
@ -30,7 +27,7 @@ import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class AlertSendResponse implements ResponseMessageBuilder {
public class AlertSendResponse {
/**
* true:All alert are successful,
@ -40,11 +37,6 @@ public class AlertSendResponse implements ResponseMessageBuilder {
private List<AlertSendResponseResult> resResults;
@Override
public MessageType getCommandType() {
return MessageType.RESPONSE;
}
@Data
@NoArgsConstructor
@AllArgsConstructor
@ -55,4 +47,5 @@ public class AlertSendResponse implements ResponseMessageBuilder {
private String message;
}
}

69
dolphinscheduler-extract/dolphinscheduler-extract-base/pom.xml Normal file
View File

@ -0,0 +1,69 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-extract-base</artifactId>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-bom</artifactId>
<version>${project.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
</dependencies>
</project>

7
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/WorkerRpcProcessor.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcRequest.java
View File

@ -15,7 +15,10 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.processor;
package org.apache.dolphinscheduler.extract.base;
import java.io.Serializable;
public interface IRpcRequest extends Serializable {
public interface WorkerRpcProcessor extends NettyRequestProcessor {
}

13
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/processor/BaseLogProcessor.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/IRpcResponse.java
View File

@ -15,11 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.processor;
package org.apache.dolphinscheduler.extract.base;
import lombok.extern.slf4j.Slf4j;
public interface IRpcResponse {
@Slf4j
public abstract class BaseLogProcessor implements MasterRpcProcessor, WorkerRpcProcessor {
boolean isSuccess();
String getMessage();
byte[] getBody();
byte[] toBytes();
}

97
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyClientHandler.java Normal file
View File

@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.extract.base.future.ResponseFuture;
import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils;
import java.util.concurrent.ExecutorService;
import lombok.extern.slf4j.Slf4j;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.timeout.IdleStateEvent;
@ChannelHandler.Sharable
@Slf4j
public class NettyClientHandler extends ChannelInboundHandlerAdapter {
private final NettyRemotingClient nettyRemotingClient;
private final ExecutorService callbackExecutor;
public NettyClientHandler(NettyRemotingClient nettyRemotingClient, ExecutorService callbackExecutor) {
this.nettyRemotingClient = nettyRemotingClient;
this.callbackExecutor = callbackExecutor;
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
ctx.channel().close();
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
processReceived((Transporter) msg);
}
private void processReceived(final Transporter transporter) {
ResponseFuture future = ResponseFuture.getFuture(transporter.getHeader().getOpaque());
if (future == null) {
log.warn("Cannot find the ResponseFuture if transporter: {}", transporter);
return;
}
StandardRpcResponse deserialize = JsonSerializer.deserialize(transporter.getBody(), StandardRpcResponse.class);
future.setIRpcResponse(deserialize);
future.release();
if (future.getInvokeCallback() != null) {
future.removeFuture();
this.callbackExecutor.submit(future::executeInvokeCallback);
} else {
future.putResponse(deserialize);
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.error("NettyClientHandler catch an exception : {}", cause.getMessage(), cause);
nettyRemotingClient.closeChannel(ChannelUtils.toAddress(ctx.channel()));
ctx.channel().close();
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
ctx.channel()
.writeAndFlush(HeartBeatTransporter.getHeartBeatTransporter())
.addListener(ChannelFutureListener.CLOSE_ON_FAILURE);
if (log.isDebugEnabled()) {
log.debug("Client send heart beat to: {}", ChannelUtils.getRemoteAddress(ctx.channel()));
}
} else {
super.userEventTriggered(ctx, evt);
}
}
}

146
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClient.java
View File

@ -15,25 +15,23 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.remote.exceptions.RemotingTimeoutException;
import org.apache.dolphinscheduler.remote.exceptions.RemotingTooMuchRequestException;
import org.apache.dolphinscheduler.remote.future.InvokeCallback;
import org.apache.dolphinscheduler.remote.future.ReleaseSemaphore;
import org.apache.dolphinscheduler.remote.future.ResponseFuture;
import org.apache.dolphinscheduler.remote.handler.NettyClientHandler;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.CallerThreadExecutePolicy;
import org.apache.dolphinscheduler.remote.utils.Constants;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.remote.utils.NettyUtils;
import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemotingException;
import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException;
import org.apache.dolphinscheduler.extract.base.exception.RemotingTooMuchRequestException;
import org.apache.dolphinscheduler.extract.base.future.InvokeCallback;
import org.apache.dolphinscheduler.extract.base.future.ReleaseSemaphore;
import org.apache.dolphinscheduler.extract.base.future.ResponseFuture;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder;
import org.apache.dolphinscheduler.extract.base.utils.CallerThreadExecutePolicy;
import org.apache.dolphinscheduler.extract.base.utils.Constants;
import org.apache.dolphinscheduler.extract.base.utils.Host;
import org.apache.dolphinscheduler.extract.base.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
import java.net.InetSocketAddress;
import java.util.concurrent.ConcurrentHashMap;
@ -64,8 +62,6 @@ public class NettyRemotingClient implements AutoCloseable {
private final Bootstrap bootstrap = new Bootstrap();
private final NettyEncoder encoder = new NettyEncoder();
private final ConcurrentHashMap<Host, Channel> channels = new ConcurrentHashMap<>(128);
private final AtomicBoolean isStarted = new AtomicBoolean(false);
@ -123,24 +119,20 @@ public class NettyRemotingClient implements AutoCloseable {
public void initChannel(SocketChannel ch) {
ch.pipeline()
.addLast("client-idle-handler",
new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0,
new IdleStateHandler(
Constants.NETTY_CLIENT_HEART_BEAT_TIME,
0,
0,
TimeUnit.MILLISECONDS))
.addLast(new NettyDecoder(), clientHandler, encoder);
.addLast(new TransporterDecoder(), clientHandler, new TransporterEncoder());
}
});
this.responseFutureExecutor.scheduleWithFixedDelay(ResponseFuture::scanFutureTable, 0, 1, TimeUnit.SECONDS);
isStarted.compareAndSet(false, true);
}
/**
* async send
*
* @param host host
* @param command command
* @param timeoutMillis timeoutMillis
* @param invokeCallback callback function
*/
public void sendAsync(final Host host, final Message command,
public void sendAsync(final Host host,
final Transporter transporter,
final long timeoutMillis,
final InvokeCallback invokeCallback) throws InterruptedException, RemotingException {
final Channel channel = getChannel(host);
@ -150,7 +142,7 @@ public class NettyRemotingClient implements AutoCloseable {
/*
* request unique identification
*/
final long opaque = command.getOpaque();
final long opaque = transporter.getHeader().getOpaque();
/*
* control concurrency number
*/
@ -166,7 +158,7 @@ public class NettyRemotingClient implements AutoCloseable {
invokeCallback,
releaseSemaphore);
try {
channel.writeAndFlush(command).addListener(future -> {
channel.writeAndFlush(transporter).addListener(future -> {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
@ -185,7 +177,7 @@ public class NettyRemotingClient implements AutoCloseable {
});
} catch (Exception ex) {
responseFuture.release();
throw new RemotingException(String.format("send command to host: %s failed", host), ex);
throw new RemotingException(String.format("Send transporter to host: %s failed", host), ex);
}
} else {
String message = String.format(
@ -195,23 +187,15 @@ public class NettyRemotingClient implements AutoCloseable {
}
}
/**
* sync send
*
* @param host host
* @param message command
* @param timeoutMillis timeoutMillis
* @return command
*/
public Message sendSync(final Host host, final Message message,
final long timeoutMillis) throws InterruptedException, RemotingException {
public IRpcResponse sendSync(final Host host, final Transporter transporter,
final long timeoutMillis) throws InterruptedException, RemotingException {
final Channel channel = getChannel(host);
if (channel == null) {
throw new RemotingException(String.format("connect to : %s fail", host));
}
final long opaque = message.getOpaque();
final long opaque = transporter.getHeader().getOpaque();
final ResponseFuture responseFuture = new ResponseFuture(opaque, timeoutMillis, null, null);
channel.writeAndFlush(message).addListener(future -> {
channel.writeAndFlush(transporter).addListener(future -> {
if (future.isSuccess()) {
responseFuture.setSendOk(true);
return;
@ -220,74 +204,22 @@ public class NettyRemotingClient implements AutoCloseable {
}
responseFuture.setCause(future.cause());
responseFuture.putResponse(null);
log.error("send command {} to host {} failed: {}", message, host, responseFuture.getCause());
log.error("Send Sync request {} to host {} failed", transporter, host, responseFuture.getCause());
});
/*
* sync wait for result
*/
Message result = responseFuture.waitResponse();
if (result == null) {
IRpcResponse iRpcResponse = responseFuture.waitResponse();
if (iRpcResponse == null) {
if (responseFuture.isSendOK()) {
throw new RemotingTimeoutException(host.toString(), timeoutMillis, responseFuture.getCause());
} else {
throw new RemotingException(host.toString(), responseFuture.getCause());
}
}
return result;
return iRpcResponse;
}
/**
* send task
*
* @param host host
* @param message command
*/
public void send(final Host host, final Message message) throws RemotingException {
Channel channel = getChannel(host);
if (channel == null) {
throw new RemotingException(String.format("connect to : %s fail", host));
}
try {
ChannelFuture future = channel.writeAndFlush(message).await();
if (future.isSuccess()) {
log.debug("send command : {} , to : {} successfully.", message, host.getAddress());
} else {
String msg = String.format("send command : %s , to :%s failed", message, host.getAddress());
log.error(msg, future.cause());
throw new RemotingException(msg);
}
} catch (RemotingException remotingException) {
throw remotingException;
} catch (Exception e) {
log.error("Send command {} to address {} encounter error.", message, host.getAddress());
throw new RemotingException(
String.format("Send command : %s , to :%s encounter error", message, host.getAddress()), e);
}
}
/**
* register processor
*
* @param processor processor
*/
public void registerProcessor(final NettyRequestProcessor processor) {
this.registerProcessor(processor, null);
}
/**
* register processor
*
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final NettyRequestProcessor processor,
final ExecutorService executor) {
this.clientHandler.registerProcessor(processor.getCommandType(), processor, executor);
}
/**
* get channel
*/
public Channel getChannel(Host host) {
Channel channel = channels.get(host);
if (channel != null && channel.isActive()) {
@ -299,7 +231,7 @@ public class NettyRemotingClient implements AutoCloseable {
/**
* create channel
*
* @param host host
* @param host host
* @param isSync sync flag
* @return channel
*/
@ -344,9 +276,6 @@ public class NettyRemotingClient implements AutoCloseable {
}
}
/**
* close channels
*/
private void closeChannels() {
for (Channel channel : this.channels.values()) {
channel.close();
@ -354,11 +283,6 @@ public class NettyRemotingClient implements AutoCloseable {
this.channels.clear();
}
/**
* close channel
*
* @param host host
*/
public void closeChannel(Host host) {
Channel channel = this.channels.remove(host);
if (channel != null) {

9
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingClientFactory.java
View File

@ -15,10 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.factory;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig;
import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@ -27,9 +26,7 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class NettyRemotingClientFactory {
public NettyRemotingClient buildNettyRemotingClient() {
NettyClientConfig nettyClientConfig = new NettyClientConfig();
log.info("NettyRemotingClient initialized with config: {}", nettyClientConfig);
public NettyRemotingClient buildNettyRemotingClient(NettyClientConfig nettyClientConfig) {
return new NettyRemotingClient(nettyClientConfig);
}
}

88
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServer.java
View File

@ -15,16 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.remote.codec.NettyDecoder;
import org.apache.dolphinscheduler.remote.codec.NettyEncoder;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
import org.apache.dolphinscheduler.remote.exceptions.RemoteException;
import org.apache.dolphinscheduler.remote.handler.NettyServerHandler;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants;
import org.apache.dolphinscheduler.remote.utils.NettyUtils;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
import org.apache.dolphinscheduler.extract.base.exception.RemoteException;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder;
import org.apache.dolphinscheduler.extract.base.server.JdkDynamicServerHandler;
import org.apache.dolphinscheduler.extract.base.server.ServerMethodInvoker;
import org.apache.dolphinscheduler.extract.base.utils.Constants;
import org.apache.dolphinscheduler.extract.base.utils.NettyUtils;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -53,51 +53,23 @@ import io.netty.handler.timeout.IdleStateHandler;
@Slf4j
public class NettyRemotingServer {
/**
* server bootstrap
*/
private final ServerBootstrap serverBootstrap = new ServerBootstrap();
/**
* default executor
*/
private final ExecutorService defaultExecutor = Executors.newFixedThreadPool(Constants.CPUS);
private final ExecutorService defaultExecutor =
Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 2);
/**
* boss group
*/
private final EventLoopGroup bossGroup;
/**
* worker group
*/
private final EventLoopGroup workGroup;
/**
* server config
*/
private final NettyServerConfig serverConfig;
/**
* server handler
*/
private final NettyServerHandler serverHandler = new NettyServerHandler(this);
private final JdkDynamicServerHandler serverHandler = new JdkDynamicServerHandler(this);
/**
* started flag
*/
private final AtomicBoolean isStarted = new AtomicBoolean(false);
/**
* Netty server bind fail message
*/
private static final String NETTY_BIND_FAILURE_MSG = "NettyRemotingServer bind %s fail";
/**
* server init
*
* @param serverConfig server config
*/
public NettyRemotingServer(final NettyServerConfig serverConfig) {
this.serverConfig = serverConfig;
ThreadFactory bossThreadFactory =
@ -113,9 +85,6 @@ public class NettyRemotingServer {
}
}
/**
* server start
*/
public void start() {
if (isStarted.compareAndSet(false, true)) {
this.serverBootstrap
@ -160,42 +129,21 @@ public class NettyRemotingServer {
*/
private void initNettyChannel(SocketChannel ch) {
ch.pipeline()
.addLast("encoder", new NettyEncoder())
.addLast("decoder", new NettyDecoder())
.addLast("encoder", new TransporterEncoder())
.addLast("decoder", new TransporterDecoder())
.addLast("server-idle-handle",
new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS))
.addLast("handler", serverHandler);
}
/**
* register processor
*
* @param processor processor
*/
public void registerProcessor(final NettyRequestProcessor processor) {
this.registerProcessor(processor, null);
}
/**
* register processor
*
* @param processor processor
* @param executor thread executor
*/
public void registerProcessor(final NettyRequestProcessor processor,
final ExecutorService executor) {
this.serverHandler.registerProcessor(processor.getCommandType(), processor, executor);
}
/**
* get default thread executor
*
* @return thread executor
*/
public ExecutorService getDefaultExecutor() {
return defaultExecutor;
}
public void registerMethodInvoker(ServerMethodInvoker methodInvoker) {
serverHandler.registerMethodInvoker(methodInvoker);
}
public void close() {
if (isStarted.compareAndSet(true, false)) {
try {

21
dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/NettyRemotingServerFactory.java
View File

@ -15,23 +15,16 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.microbench.common;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.rpc.base.RpcService;
import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig;
/**
* UserService
*/
@RpcService("IUserService")
public class UserService implements IUserService {
import lombok.experimental.UtilityClass;
@Override
public Boolean say(String s) {
return true;
}
@UtilityClass
public class NettyRemotingServerFactory {
@Override
public Integer hi(int num) {
return ++num;
public NettyRemotingServer buildNettyRemotingServer(NettyServerConfig nettyServerConfig) {
return new NettyRemotingServer(nettyServerConfig);
}
}

33
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcMethod.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface RpcMethod {
long timeout() default 3000L;
}

11
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/RpcService.java
View File

@ -15,16 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.rpc.base;
package org.apache.dolphinscheduler.extract.base;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.TYPE})
/**
* Used to mark a class as a RPC service
*/
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface RpcService {
String value() default "";
}

42
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/log/RollViewLogRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcRequest.java
View File

@ -15,40 +15,34 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.log;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* roll view log request command
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class RollViewLogRequest implements RequestMessageBuilder {
@NoArgsConstructor
public class StandardRpcRequest implements IRpcRequest {
/**
* log path
*/
private String path;
private byte[][] args;
/**
* skip line number
*/
private int skipLineNum;
private Class<?>[] argsTypes;
/**
* query line number
*/
private int limit;
@Override
public MessageType getCommandType() {
return MessageType.ROLL_VIEW_LOG_REQUEST;
public static StandardRpcRequest of(Object[] args) {
if (args == null || args.length == 0) {
return new StandardRpcRequest(null, null);
}
final byte[][] argsBytes = new byte[args.length][];
final Class<?>[] argsTypes = new Class[args.length];
for (int i = 0; i < args.length; i++) {
argsBytes[i] = JsonSerializer.serialize(args[i]);
argsTypes[i] = args[i].getClass();
}
return new StandardRpcRequest(argsBytes, argsTypes);
}
}

38
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchResponse.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/StandardRpcResponse.java
View File

@ -15,36 +15,40 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.task;
package org.apache.dolphinscheduler.extract.base;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.ResponseMessageBuilder;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TaskDispatchResponse implements ResponseMessageBuilder {
public class StandardRpcResponse implements IRpcResponse {
private Integer taskInstanceId;
private boolean dispatchSuccess;
private boolean success;
private String message;
public static TaskDispatchResponse success(Integer taskInstanceId) {
return new TaskDispatchResponse(taskInstanceId, true, "dispatch success");
private byte[] body;
private Class<?> bodyType;
public static StandardRpcResponse success(byte[] body, Class<?> bodyType) {
StandardRpcResponse rpcResponse = new StandardRpcResponse();
rpcResponse.setSuccess(true);
rpcResponse.setBody(body);
rpcResponse.setBodyType(bodyType);
return rpcResponse;
}
public static TaskDispatchResponse failed(Integer taskInstanceId, String message) {
return new TaskDispatchResponse(taskInstanceId, false, message);
public static StandardRpcResponse fail(String message) {
StandardRpcResponse rpcResponse = new StandardRpcResponse();
rpcResponse.setSuccess(false);
rpcResponse.setMessage(message);
return rpcResponse;
}
@Override
public MessageType getCommandType() {
return MessageType.TASK_DISPATCH_RESPONSE;
public byte[] toBytes() {
return JsonSerializer.serialize(this);
}
}

50
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Pair.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/BaseRemoteMethodInvoker.java
View File

@ -15,42 +15,28 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.client;
/**
* key value pair
*
* @param <L> L generic type
* @param <R> R generic type
*/
public class Pair<L, R> {
import org.apache.dolphinscheduler.extract.base.NettyRemotingClient;
import org.apache.dolphinscheduler.extract.base.utils.Host;
private L left;
import java.lang.reflect.Method;
private R right;
public abstract class BaseRemoteMethodInvoker implements ClientMethodInvoker {
public Pair(L left, R right) {
this.left = left;
this.right = right;
protected final String methodIdentifier;
protected final NettyRemotingClient nettyRemotingClient;
protected final Method localMethod;
protected final Host serverHost;
public BaseRemoteMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) {
this.serverHost = serverHost;
this.localMethod = localMethod;
this.nettyRemotingClient = nettyRemotingClient;
this.methodIdentifier = localMethod.toGenericString();
}
public L getLeft() {
return left;
}
public void setLeft(L left) {
this.left = left;
}
public R getRight() {
return right;
}
public void setRight(R right) {
this.right = right;
}
public static <L, R> Pair<L, R> of(L left, R right) {
return new Pair<>(left, right);
}
}

58
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientInvocationHandler.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.client;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.extract.base.NettyRemotingClient;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.utils.Host;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class ClientInvocationHandler implements InvocationHandler {
private final NettyRemotingClient nettyRemotingClient;
private final Map<String, ClientMethodInvoker> methodInvokerMap;
private final Host serverHost;
public ClientInvocationHandler(Host serverHost, NettyRemotingClient nettyRemotingClient) {
this.serverHost = checkNotNull(serverHost);
this.nettyRemotingClient = checkNotNull(nettyRemotingClient);
this.methodInvokerMap = new ConcurrentHashMap<>();
}
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
if (method.getAnnotation(RpcMethod.class) == null) {
return method.invoke(proxy, args);
}
ClientMethodInvoker methodInvoker = methodInvokerMap.computeIfAbsent(
method.toGenericString(), m -> new SyncClientMethodInvoker(serverHost, method, nettyRemotingClient));
return methodInvoker.invoke(proxy, method, args);
}
}

26
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/ClientMethodInvoker.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.client;
import java.lang.reflect.Method;
public interface ClientMethodInvoker {
Object invoke(Object proxy, Method method, Object[] args) throws Throwable;
}

16
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/IRpcClientProxyFactory.java
View File

@ -15,18 +15,18 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.rpc.common;
package org.apache.dolphinscheduler.extract.base.client;
/**
* AbstractRpcCallBack
*/
public abstract class AbstractRpcCallBack {
public interface IRpcClientProxyFactory {
/**
* When sending an asynchronous message, this method will be called after the response is successfully sent.
* Create the client proxy.
*
* @param object response
* @param serverHost which server host to connect
* @param clientInterface which need to be proxied
* @param <T> InterfaceType
* @return proxy client
*/
public abstract void run(Object object);
<T> T getProxyClient(String serverHost, Class<T> clientInterface);
}

51
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/JdkDynamicRpcClientProxyFactory.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.client;
import org.apache.dolphinscheduler.extract.base.NettyRemotingClient;
import org.apache.dolphinscheduler.extract.base.utils.Host;
import java.lang.reflect.Proxy;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* This class is used to create a proxy client which will transform local method invocation to remove invocation.
*/
public class JdkDynamicRpcClientProxyFactory implements IRpcClientProxyFactory {
private final NettyRemotingClient nettyRemotingClient;
// todo: use guava cache to avoid memory leak
private final Map<String, Map<String, Object>> proxyClientCache = new ConcurrentHashMap<>();
public JdkDynamicRpcClientProxyFactory(NettyRemotingClient nettyRemotingClient) {
this.nettyRemotingClient = nettyRemotingClient;
}
@SuppressWarnings("unchecked")
@Override
public <T> T getProxyClient(String serverHost, Class<T> clientInterface) {
return (T) proxyClientCache
.computeIfAbsent(serverHost, key -> new ConcurrentHashMap<>())
.computeIfAbsent(clientInterface.getName(),
key -> Proxy.newProxyInstance(
clientInterface.getClassLoader(), new Class[]{clientInterface},
new ClientInvocationHandler(Host.of(serverHost), nettyRemotingClient)));
}
}

29
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskPauseRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SingletonJdkDynamicRpcClientProxyFactory.java
View File

@ -15,31 +15,22 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.task;
package org.apache.dolphinscheduler.extract.base.client;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
import org.apache.dolphinscheduler.extract.base.NettyRemotingClientFactory;
import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
public class SingletonJdkDynamicRpcClientProxyFactory extends JdkDynamicRpcClientProxyFactory {
@Data
@NoArgsConstructor
@ToString(callSuper = true)
public class TaskPauseRequest implements RequestMessageBuilder {
private static final SingletonJdkDynamicRpcClientProxyFactory INSTANCE =
new SingletonJdkDynamicRpcClientProxyFactory();
private static final long serialVersionUID = -1L;
private Integer taskInstanceId;
public TaskPauseRequest(Integer taskInstanceId) {
this.taskInstanceId = taskInstanceId;
private SingletonJdkDynamicRpcClientProxyFactory() {
super(NettyRemotingClientFactory.buildNettyRemotingClient(new NettyClientConfig()));
}
@Override
public MessageType getCommandType() {
return MessageType.PAUSE_TASK_INSTANCE;
public static SingletonJdkDynamicRpcClientProxyFactory getInstance() {
return INSTANCE;
}
}

56
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/SyncClientMethodInvoker.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.client;
import org.apache.dolphinscheduler.extract.base.IRpcResponse;
import org.apache.dolphinscheduler.extract.base.NettyRemotingClient;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.StandardRpcRequest;
import org.apache.dolphinscheduler.extract.base.exception.MethodInvocationException;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import org.apache.dolphinscheduler.extract.base.utils.Host;
import java.lang.reflect.Method;
public class SyncClientMethodInvoker extends BaseRemoteMethodInvoker {
public SyncClientMethodInvoker(Host serverHost, Method localMethod, NettyRemotingClient nettyRemotingClient) {
super(serverHost, localMethod, nettyRemotingClient);
}
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
RpcMethod sync = method.getAnnotation(RpcMethod.class);
Transporter transporter = new Transporter();
transporter.setBody(JsonSerializer.serialize(StandardRpcRequest.of(args)));
transporter.setHeader(TransporterHeader.of(methodIdentifier));
IRpcResponse iRpcResponse =
nettyRemotingClient.sendSync(serverHost, transporter, sync.timeout());
if (!iRpcResponse.isSuccess()) {
throw MethodInvocationException.of(iRpcResponse.getMessage());
}
if (iRpcResponse.getBody() == null) {
return null;
}
Class<?> responseClass = method.getReturnType();
return JsonSerializer.deserialize(iRpcResponse.getBody(), responseClass);
}
}

6
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyClientConfig.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java
View File

@ -15,9 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.config;
import org.apache.dolphinscheduler.remote.utils.Constants;
package org.apache.dolphinscheduler.extract.base.config;
import lombok.AllArgsConstructor;
import lombok.Builder;
@ -34,7 +32,7 @@ public class NettyClientConfig {
* worker threads，default get machine cpus
*/
@Builder.Default
private int workerThreads = Constants.CPUS;
private int workerThreads = Runtime.getRuntime().availableProcessors() * 2;
/**
* whether tpc delay

21
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java
View File

@ -14,9 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.config;
import org.apache.dolphinscheduler.remote.utils.Constants;
package org.apache.dolphinscheduler.extract.base.config;
import lombok.Data;
@ -29,32 +28,36 @@ public class NettyServerConfig {
private int soBacklog = 1024;
/**
* whether tpc delay
* whether tpc delay
*/
private boolean tcpNoDelay = true;
/**
* whether keep alive
* whether keep alive
*/
private boolean soKeepalive = true;
/**
* send buffer size
* send buffer size
*/
private int sendBufferSize = 65535;
/**
* receive buffer size
* receive buffer size
*/
private int receiveBufferSize = 65535;
/**
* worker threads，default get machine cpus
* worker threads，default get machine cpus
*/
private int workerThread = Constants.CPUS;
private int workerThread = Runtime.getRuntime().availableProcessors() * 2;
/**
* listen port
* listen port
*/
private int listenPort = 12346;
public NettyServerConfig(int listenPort) {
this.listenPort = listenPort;
}
}

30
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/MethodInvocationException.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.exception;
public class MethodInvocationException extends RuntimeException {
public MethodInvocationException(String message) {
super(message);
}
public static MethodInvocationException of(String message) {
return new MethodInvocationException(message);
}
}

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemoteException.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemoteException.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.exceptions;
package org.apache.dolphinscheduler.extract.base.exception;
/**
* Custom runtime exception

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingException.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingException.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.exceptions;
package org.apache.dolphinscheduler.extract.base.exception;
/**
* remote exception

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTimeoutException.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTimeoutException.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.exceptions;
package org.apache.dolphinscheduler.extract.base.exception;
/**
* timeout exception

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/exceptions/RemotingTooMuchRequestException.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/exception/RemotingTooMuchRequestException.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.exceptions;
package org.apache.dolphinscheduler.extract.base.exception;
/**
* too much request exception

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/InvokeCallback.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/InvokeCallback.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.future;
package org.apache.dolphinscheduler.extract.base.future;
/**
* invoke callback

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ReleaseSemaphore.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ReleaseSemaphore.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.future;
package org.apache.dolphinscheduler.extract.base.future;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;

72
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/future/ResponseFuture.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/future/ResponseFuture.java
View File

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.future;
package org.apache.dolphinscheduler.extract.base.future;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.extract.base.IRpcResponse;
import java.util.Iterator;
import java.util.Map;
@ -25,50 +25,41 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.extern.slf4j.Slf4j;
/**
* response future
*/
@ToString
@Slf4j
public class ResponseFuture {
private static final ConcurrentHashMap<Long, ResponseFuture> FUTURE_TABLE = new ConcurrentHashMap<>(256);
/**
* request unique identification
*/
private final long opaque;
/**
* timeout
*/
// remove the timeout
private final long timeoutMillis;
/**
* invokeCallback function
*/
private final InvokeCallback invokeCallback;
/**
* releaseSemaphore
*/
private final ReleaseSemaphore releaseSemaphore;
private final CountDownLatch latch = new CountDownLatch(1);
private final long beginTimestamp = System.currentTimeMillis();
/**
* response command
*/
private Message responseMessage;
@Getter
@Setter
private IRpcResponse iRpcResponse;
private volatile boolean sendOk = true;
private Throwable cause;
public ResponseFuture(long opaque, long timeoutMillis, InvokeCallback invokeCallback,
public ResponseFuture(long opaque,
long timeoutMillis,
InvokeCallback invokeCallback,
ReleaseSemaphore releaseSemaphore) {
this.opaque = opaque;
this.timeoutMillis = timeoutMillis;
@ -82,20 +73,15 @@ public class ResponseFuture {
*
* @return command
*/
public Message waitResponse() throws InterruptedException {
public IRpcResponse waitResponse() throws InterruptedException {
if (!latch.await(timeoutMillis, TimeUnit.MILLISECONDS)) {
log.warn("Wait response timeout, request id {}", opaque);
log.warn("Wait response in {}/ms timeout, request id {}", timeoutMillis, opaque);
}
return this.responseMessage;
return this.iRpcResponse;
}
/**
* put response
*
* @param responseMessage responseCommand
*/
public void putResponse(final Message responseMessage) {
this.responseMessage = responseMessage;
public void putResponse(final IRpcResponse iRpcResponse) {
this.iRpcResponse = iRpcResponse;
this.latch.countDown();
FUTURE_TABLE.remove(opaque);
}
@ -155,14 +141,6 @@ public class ResponseFuture {
return beginTimestamp;
}
public Message getResponseCommand() {
return responseMessage;
}
public void setResponseCommand(Message responseMessage) {
this.responseMessage = responseMessage;
}
public InvokeCallback getInvokeCallback() {
return invokeCallback;
}
@ -199,18 +177,4 @@ public class ResponseFuture {
}
}
@Override
public String toString() {
return "ResponseFuture{"
+ "opaque=" + opaque
+ ", timeoutMillis=" + timeoutMillis
+ ", invokeCallback=" + invokeCallback
+ ", releaseSemaphore=" + releaseSemaphore
+ ", latch=" + latch
+ ", beginTimestamp=" + beginTimestamp
+ ", responseCommand=" + responseMessage
+ ", sendOk=" + sendOk
+ ", cause=" + cause
+ '}';
}
}

25
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/HeartBeatTransporter.java
View File

@ -15,28 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.rpc.common;
package org.apache.dolphinscheduler.extract.base.protocal;
public enum ResponseEventType {
import org.apache.dolphinscheduler.extract.base.StandardRpcResponse;
ACK((byte) 1, "ack"),
BUSINESS_RSP((byte) 2, "business response");
public class HeartBeatTransporter extends Transporter {
private Byte type;
public static final String METHOD_IDENTIFY = "HEART_BEAT";
private String description;
private static final Transporter HEART_BEAT_TRANSPORTER =
of(TransporterHeader.of(METHOD_IDENTIFY), new StandardRpcResponse());
ResponseEventType(Byte type, String description) {
this.type = type;
this.description = description;
}
public Byte getType() {
return type;
}
public String getDescription() {
return description;
public static Transporter getHeartBeatTransporter() {
return HEART_BEAT_TRANSPORTER;
}
}

55
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/Transporter.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.protocal;
import org.apache.dolphinscheduler.extract.base.StandardRpcRequest;
import org.apache.dolphinscheduler.extract.base.StandardRpcResponse;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import java.io.Serializable;
import lombok.Data;
import lombok.NonNull;
@Data
public class Transporter implements Serializable {
private static final long serialVersionUID = -1L;
public static final byte MAGIC = (byte) 0xbabe;
public static final byte VERSION = 0;
private TransporterHeader header;
private byte[] body;
public static Transporter of(@NonNull TransporterHeader header, StandardRpcResponse iRpcResponse) {
return of(header, JsonSerializer.serialize(iRpcResponse));
}
public static Transporter of(@NonNull TransporterHeader header, StandardRpcRequest iRpcRequest) {
return of(header, JsonSerializer.serialize(iRpcRequest));
}
public static Transporter of(@NonNull TransporterHeader header, byte[] body) {
Transporter transporter = new Transporter();
transporter.setHeader(header);
transporter.setBody(body);
return transporter;
}
}

94
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterDecoder.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.protocal;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.ReplayingDecoder;
@Slf4j
public class TransporterDecoder extends ReplayingDecoder<TransporterDecoder.State> {
public TransporterDecoder() {
super(State.MAGIC);
}
private int headerLength;
private byte[] header;
private int bodyLength;
private byte[] body;
@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
switch (state()) {
case MAGIC:
checkMagic(in.readByte());
checkpoint(State.VERSION);
case VERSION:
checkVersion(in.readByte());
checkpoint(State.HEADER_LENGTH);
case HEADER_LENGTH:
headerLength = in.readInt();
checkpoint(State.HEADER);
case HEADER:
header = new byte[headerLength];
in.readBytes(header);
checkpoint(State.BODY_LENGTH);
case BODY_LENGTH:
bodyLength = in.readInt();
checkpoint(State.BODY);
case BODY:
body = new byte[bodyLength];
in.readBytes(body);
Transporter transporter =
Transporter.of(JsonSerializer.deserialize(header, TransporterHeader.class), body);
out.add(transporter);
checkpoint(State.MAGIC);
break;
default:
log.warn("unknown decoder state {}", state());
}
}
private void checkMagic(byte magic) {
if (magic != Transporter.MAGIC) {
throw new IllegalArgumentException("illegal packet [magic]" + magic);
}
}
private void checkVersion(byte version) {
if (version != Transporter.VERSION) {
throw new IllegalArgumentException("illegal protocol [version]" + version);
}
}
enum State {
MAGIC,
VERSION,
HEADER_LENGTH,
HEADER,
BODY_LENGTH,
BODY;
}
}

45
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyEncoder.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterEncoder.java
View File

@ -15,46 +15,35 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.codec;
package org.apache.dolphinscheduler.extract.base.protocal;
import org.apache.dolphinscheduler.remote.command.Message;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.extract.base.exception.RemotingException;
import io.netty.buffer.ByteBuf;
import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.MessageToByteEncoder;
/**
* netty encoder
*/
@Sharable
public class NettyEncoder extends MessageToByteEncoder<Message> {
public class TransporterEncoder extends MessageToByteEncoder<Transporter> {
/**
* encode
*
* @param ctx channel handler context
* @param msg command
* @param out byte buffer
*/
@Override
protected void encode(ChannelHandlerContext ctx, Message msg, ByteBuf out) throws Exception {
if (msg == null) {
protected void encode(ChannelHandlerContext ctx, Transporter transporter, ByteBuf out) throws Exception {
if (transporter == null) {
throw new RemotingException("encode msg is null");
}
out.writeByte(Message.MAGIC);
out.writeByte(Message.VERSION);
out.writeByte(msg.getType().ordinal());
out.writeLong(msg.getOpaque());
writeContext(msg, out);
out.writeInt(msg.getBody().length);
out.writeBytes(msg.getBody());
out.writeByte(Transporter.MAGIC);
out.writeByte(Transporter.VERSION);
// write header
byte[] header = transporter.getHeader().toBytes();
out.writeInt(header.length);
out.writeBytes(header);
// write body
byte[] body = transporter.getBody();
out.writeInt(body.length);
out.writeBytes(body);
}
private void writeContext(Message msg, ByteBuf out) {
byte[] headerBytes = msg.getContext().toBytes();
out.writeInt(headerBytes.length);
out.writeBytes(headerBytes);
}
}

63
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/protocal/TransporterHeader.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.protocal;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
import lombok.Data;
@Data
public class TransporterHeader implements Serializable {
private static final long serialVersionUID = -1L;
private static final AtomicLong REQUEST_ID = new AtomicLong(1);
private String methodIdentifier;
private long opaque;
// Used for JsonDeSerializer
public TransporterHeader() {
}
public TransporterHeader(String methodIdentifier) {
this(REQUEST_ID.getAndIncrement(), methodIdentifier);
}
public TransporterHeader(long opaque, String methodIdentifier) {
this.opaque = opaque;
this.methodIdentifier = methodIdentifier;
}
public static TransporterHeader of(String methodIdentifier) {
return new TransporterHeader(methodIdentifier);
}
public static TransporterHeader of(long opaque, String methodIdentifier) {
return new TransporterHeader(opaque, methodIdentifier);
}
public byte[] toBytes() {
return JsonSerializer.serialize(this);
}
}

19
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/JsonSerializer.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/serialize/JsonSerializer.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.serialize;
import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT;
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
@ -68,14 +68,15 @@ public class JsonSerializer {
* @return byte array
*/
public static <T> byte[] serialize(T obj) {
String json = "";
if (obj == null) {
return null;
}
try {
json = objectMapper.writeValueAsString(obj);
return objectMapper.writeValueAsString(obj).getBytes(StandardCharsets.UTF_8);
} catch (JsonProcessingException e) {
log.error("serializeToString exception!", e);
return null;
}
return json.getBytes(Constants.UTF8);
}
/**
@ -99,12 +100,15 @@ public class JsonSerializer {
/**
* deserialize
*
* @param src byte array
* @param src byte array
* @param clazz class
* @param <T> deserialize type
* @param <T> deserialize type
* @return deserialize type
*/
public static <T> T deserialize(byte[] src, Class<T> clazz) {
if (src == null) {
return null;
}
String json = new String(src, StandardCharsets.UTF_8);
try {
@ -113,7 +117,6 @@ public class JsonSerializer {
log.error("deserialize exception!", e);
return null;
}
}
}

169
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/JdkDynamicServerHandler.java Normal file
View File

@ -0,0 +1,169 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.server;
import static com.google.common.base.Preconditions.checkNotNull;
import org.apache.dolphinscheduler.extract.base.NettyRemotingServer;
import org.apache.dolphinscheduler.extract.base.StandardRpcRequest;
import org.apache.dolphinscheduler.extract.base.StandardRpcResponse;
import org.apache.dolphinscheduler.extract.base.protocal.HeartBeatTransporter;
import org.apache.dolphinscheduler.extract.base.protocal.Transporter;
import org.apache.dolphinscheduler.extract.base.protocal.TransporterHeader;
import org.apache.dolphinscheduler.extract.base.serialize.JsonSerializer;
import org.apache.dolphinscheduler.extract.base.utils.ChannelUtils;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionException;
import lombok.extern.slf4j.Slf4j;
import io.netty.channel.Channel;
import io.netty.channel.ChannelConfig;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.handler.timeout.IdleStateEvent;
@Slf4j
@ChannelHandler.Sharable
public class JdkDynamicServerHandler extends ChannelInboundHandlerAdapter {
private final NettyRemotingServer nettyRemotingServer;
private final Map<String, ServerMethodInvoker> methodInvokerMap;
public JdkDynamicServerHandler(NettyRemotingServer nettyRemotingServer) {
this.nettyRemotingServer = nettyRemotingServer;
this.methodInvokerMap = new ConcurrentHashMap<>();
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
ctx.channel().close();
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
processReceived(ctx.channel(), (Transporter) msg);
}
public void registerMethodInvoker(ServerMethodInvoker methodInvoker) {
checkNotNull(methodInvoker);
checkNotNull(methodInvoker.getMethodIdentify());
methodInvokerMap.put(methodInvoker.getMethodIdentify(), methodInvoker);
}
private void processReceived(final Channel channel, final Transporter transporter) {
final String methodIdentifier = transporter.getHeader().getMethodIdentifier();
if (HeartBeatTransporter.METHOD_IDENTIFY.equals(methodIdentifier)) {
if (log.isDebugEnabled()) {
log.debug("server receive heart beat from: host: {}", ChannelUtils.getRemoteAddress(channel));
}
return;
}
ServerMethodInvoker methodInvoker = methodInvokerMap.get(methodIdentifier);
try {
if (methodInvoker == null) {
log.error("Cannot find the ServerMethodInvoker of : {}", transporter);
StandardRpcResponse iRpcResponse =
StandardRpcResponse.fail("Cannot find the ServerMethodInvoker of " + methodIdentifier);
TransporterHeader transporterHeader =
TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier);
Transporter response = Transporter.of(transporterHeader, iRpcResponse);
channel.writeAndFlush(response);
return;
}
nettyRemotingServer.getDefaultExecutor().submit(() -> {
StandardRpcResponse iRpcResponse;
try {
StandardRpcRequest standardRpcRequest =
JsonSerializer.deserialize(transporter.getBody(), StandardRpcRequest.class);
Object[] args;
if (standardRpcRequest.getArgs().length == 0) {
args = null;
} else {
args = new Object[standardRpcRequest.getArgs().length];
for (int i = 0; i < standardRpcRequest.getArgs().length; i++) {
args[i] = JsonSerializer.deserialize(standardRpcRequest.getArgs()[i],
standardRpcRequest.getArgsTypes()[i]);
}
}
Object result = methodInvoker.invoke(args);
if (result == null) {
iRpcResponse = StandardRpcResponse.success(null, null);
} else {
iRpcResponse = StandardRpcResponse.success(JsonSerializer.serialize(result), result.getClass());
}
} catch (Throwable e) {
log.error("Invoke method {} failed, {}.", methodIdentifier, e.getMessage(), e);
iRpcResponse = StandardRpcResponse.fail(e.getMessage());
}
TransporterHeader transporterHeader =
TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier);
Transporter response = Transporter.of(transporterHeader, iRpcResponse);
channel.writeAndFlush(response);
});
} catch (RejectedExecutionException e) {
log.warn("NettyRemotingServer's thread pool is full, discard msg {} from {}", transporter,
ChannelUtils.getRemoteAddress(channel));
StandardRpcResponse iRpcResponse = StandardRpcResponse.fail("NettyRemotingServer's thread pool is full");
TransporterHeader transporterHeader =
TransporterHeader.of(transporter.getHeader().getOpaque(), methodIdentifier);
Transporter response = Transporter.of(transporterHeader, iRpcResponse);
channel.writeAndFlush(response);
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.error("exceptionCaught : {}", cause.getMessage(), cause);
ctx.channel().close();
}
@Override
public void channelWritabilityChanged(ChannelHandlerContext ctx) {
Channel ch = ctx.channel();
ChannelConfig config = ch.config();
if (!ch.isWritable()) {
if (log.isWarnEnabled()) {
log.warn("{} is not writable, over high water level : {}",
ch, config.getWriteBufferHighWaterMark());
}
config.setAutoRead(false);
} else {
if (log.isWarnEnabled()) {
log.warn("{} is writable, to low water : {}", ch, config.getWriteBufferLowWaterMark());
}
config.setAutoRead(true);
}
}
@Override
public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception {
if (evt instanceof IdleStateEvent) {
ctx.channel().close();
} else {
super.userEventTriggered(ctx, evt);
}
}
}

9
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/cache/CacheNotifyService.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvoker.java
View File

@ -15,11 +15,12 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.service.cache;
package org.apache.dolphinscheduler.extract.base.server;
import org.apache.dolphinscheduler.remote.command.Message;
public interface ServerMethodInvoker {
public interface CacheNotifyService {
String getMethodIdentify();
Object invoke(final Object... arg) throws Throwable;
void notifyMaster(Message message);
}

46
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/ServerMethodInvokerImpl.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.server;
import java.lang.reflect.Method;
public class ServerMethodInvokerImpl implements ServerMethodInvoker {
private final Object serviceBean;
private final Method method;
private final String methodIdentify;
public ServerMethodInvokerImpl(Object serviceBean, Method method) {
this.serviceBean = serviceBean;
this.method = method;
this.methodIdentify = method.toGenericString();
}
@Override
public Object invoke(Object... args) throws Throwable {
// todo: check the request param when register
return method.invoke(serviceBean, args);
}
@Override
public String getMethodIdentify() {
return methodIdentify;
}
}

66
dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/SpringServerMethodInvokerDiscovery.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.base.server;
import org.apache.dolphinscheduler.extract.base.NettyRemotingServer;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import java.lang.reflect.Method;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.config.BeanPostProcessor;
import org.springframework.lang.Nullable;
@Slf4j
public class SpringServerMethodInvokerDiscovery implements BeanPostProcessor {
protected final NettyRemotingServer nettyRemotingServer;
public SpringServerMethodInvokerDiscovery(NettyRemotingServer nettyRemotingServer) {
this.nettyRemotingServer = nettyRemotingServer;
}
@Nullable
@Override
public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException {
Class<?>[] interfaces = bean.getClass().getInterfaces();
for (Class<?> anInterface : interfaces) {
if (anInterface.getAnnotation(RpcService.class) == null) {
continue;
}
registerRpcMethodInvoker(anInterface, bean, beanName);
}
return bean;
}
private void registerRpcMethodInvoker(Class<?> anInterface, Object bean, String beanName) {
Method[] declaredMethods = anInterface.getDeclaredMethods();
for (Method method : declaredMethods) {
RpcMethod rpcMethod = method.getAnnotation(RpcMethod.class);
if (rpcMethod == null) {
continue;
}
ServerMethodInvoker methodInvoker = new ServerMethodInvokerImpl(bean, method);
nettyRemotingServer.registerMethodInvoker(methodInvoker);
log.info("Register ServerMethodInvoker: {} to bean: {}", methodInvoker.getMethodIdentify(), beanName);
}
}
}

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/CallerThreadExecutePolicy.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/CallerThreadExecutePolicy.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/ChannelUtils.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/ChannelUtils.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import org.apache.dolphinscheduler.common.utils.NetUtils;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Constants.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import org.apache.dolphinscheduler.common.utils.NetUtils;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Host.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/Host.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import static org.apache.dolphinscheduler.common.constants.Constants.COLON;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NamedThreadFactory.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NamedThreadFactory.java
View File

@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/NettyUtils.java → dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/utils/NettyUtils.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.utils;
package org.apache.dolphinscheduler.extract.base.utils;
import io.netty.channel.epoll.Epoll;
import io.netty.channel.epoll.EpollServerSocketChannel;

45
dolphinscheduler-extract/dolphinscheduler-extract-master/pom.xml Normal file
View File

@ -0,0 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
~
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-extract-master</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-extract-base</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-task-api</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

50
dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ILogicTaskInstanceOperator.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskDispatchResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskKillResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskPauseResponse;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartRequest;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceForceStartResponse;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceWakeupRequest;
@RpcService
public interface ILogicTaskInstanceOperator {
@RpcMethod
LogicTaskDispatchResponse dispatchLogicTask(LogicTaskDispatchRequest taskDispatchRequest);
@RpcMethod
LogicTaskKillResponse killLogicTask(LogicTaskKillRequest taskKillRequest);
@RpcMethod
LogicTaskPauseResponse pauseLogicTask(LogicTaskPauseRequest taskPauseRequest);
@RpcMethod
TaskInstanceForceStartResponse forceStartTaskInstance(TaskInstanceForceStartRequest taskForceStartRequest);
@RpcMethod
void wakeupTaskInstance(TaskInstanceWakeupRequest taskWakeupRequest);
}

30
dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterCacheService.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.transportor.CacheExpireRequest;
@RpcService
public interface IMasterCacheService {
@RpcMethod
void cacheExpire(CacheExpireRequest cacheExpireRequest);
}

38
dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IMasterLogService.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogFileDownloadResponse;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryRequest;
import org.apache.dolphinscheduler.extract.master.transportor.LogicTaskInstanceLogPageQueryResponse;
@RpcService
public interface IMasterLogService {
@RpcMethod
LogicTaskInstanceLogFileDownloadResponse getLogicTaskInstanceWholeLogFileBytes(LogicTaskInstanceLogFileDownloadRequest logicTaskInstanceLogFileDownloadRequest);
@RpcMethod
LogicTaskInstanceLogPageQueryResponse pageQueryLogicTaskInstanceLog(LogicTaskInstanceLogPageQueryRequest taskInstanceLogPageQueryRequest);
@RpcMethod
void removeLogicTaskInstanceLog(String taskInstanceLogAbsolutePath);
}

31
dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IStreamingTaskOperator.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerRequest;
import org.apache.dolphinscheduler.extract.master.transportor.StreamingTaskTriggerResponse;
@RpcService
public interface IStreamingTaskOperator {
@RpcMethod
StreamingTaskTriggerResponse triggerStreamingTask(StreamingTaskTriggerRequest streamingTaskTriggerRequest);
}

43
dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/ITaskInstanceExecutionEventListener.java Normal file
View File

@ -0,0 +1,43 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionFinishEvent;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionInfoEvent;
import org.apache.dolphinscheduler.extract.master.transportor.TaskInstanceExecutionRunningEvent;
import org.apache.dolphinscheduler.extract.master.transportor.WorkflowInstanceStateChangeEvent;
@RpcService
public interface ITaskInstanceExecutionEventListener {
@RpcMethod
void onTaskInstanceExecutionRunning(TaskInstanceExecutionRunningEvent taskInstanceExecutionRunningEvent);
@RpcMethod
void onTaskInstanceExecutionFinish(TaskInstanceExecutionFinishEvent taskInstanceExecutionFinishEvent);
@RpcMethod
void onTaskInstanceExecutionInfoUpdate(TaskInstanceExecutionInfoEvent taskInstanceExecutionInfoEvent);
// todo: Split this into a separate interface
@RpcMethod
void onWorkflowInstanceInstanceStateChange(WorkflowInstanceStateChangeEvent workflowInstanceStateChangeEvent);
}

21
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/workflow/WorkflowMetricsCleanUpRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/IWorkflowInstanceService.java
View File

@ -15,20 +15,19 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.workflow;
package org.apache.dolphinscheduler.extract.master;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
import org.apache.dolphinscheduler.extract.base.RpcMethod;
import org.apache.dolphinscheduler.extract.base.RpcService;
import org.apache.dolphinscheduler.extract.master.dto.WorkflowExecuteDto;
import lombok.Data;
@RpcService
public interface IWorkflowInstanceService {
@Data
public class WorkflowMetricsCleanUpRequest implements RequestMessageBuilder {
@RpcMethod
void clearWorkflowMetrics(Long workflowDefinitionCode);
private String processDefinitionCode;
@RpcMethod
WorkflowExecuteDto getWorkflowExecutingData(Integer workflowInstanceId);
@Override
public MessageType getCommandType() {
return MessageType.WORKFLOW_METRICS_CLEANUP;
}
}

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/TaskInstanceExecuteDto.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/TaskInstanceExecuteDto.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.dto;
package org.apache.dolphinscheduler.extract.master.dto;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.Priority;

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/dto/WorkflowExecuteDto.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/dto/WorkflowExecuteDto.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.dto;
package org.apache.dolphinscheduler.extract.master.dto;
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.FailureStrategy;

13
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/cache/CacheExpireRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/CacheExpireRequest.java
View File

@ -15,27 +15,20 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.cache;
package org.apache.dolphinscheduler.extract.master.transportor;
import org.apache.dolphinscheduler.common.enums.CacheType;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
public class CacheExpireRequest implements RequestMessageBuilder {
@NoArgsConstructor
public class CacheExpireRequest {
private CacheType cacheType;
private String cacheKey;
@Override
public MessageType getCommandType() {
return MessageType.CACHE_EXPIRE;
}
}

33
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/ITaskInstanceExecutionEvent.java
View File

@ -15,27 +15,26 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.rpc.protocol;
package org.apache.dolphinscheduler.extract.master.transportor;
public class RpcProtocol<T> {
public interface ITaskInstanceExecutionEvent {
private MessageHeader msgHeader;
long getEventCreateTime();
private T body;
void setEventCreateTime(long eventCreateTime);
public MessageHeader getMsgHeader() {
return msgHeader;
long getEventSendTime();
void setEventSendTime(long eventSendTime);
void setHost(String host);
TaskInstanceExecutionEventType getEventType();
enum TaskInstanceExecutionEventType {
RUNNING,
FINISH,
RUNNING_INFO
}
public void setMsgHeader(MessageHeader msgHeader) {
this.msgHeader = msgHeader;
}
public T getBody() {
return body;
}
public void setBody(T body) {
this.body = body;
}
}

17
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/command/task/TaskDispatchRequest.java → dolphinscheduler-extract/dolphinscheduler-extract-master/src/main/java/org/apache/dolphinscheduler/extract/master/transportor/LogicTaskDispatchRequest.java
View File

@ -15,31 +15,22 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.command.task;
package org.apache.dolphinscheduler.extract.master.transportor;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.remote.command.MessageType;
import org.apache.dolphinscheduler.remote.command.RequestMessageBuilder;
import java.io.Serializable;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* The task dispatch message, means dispatch a task to worker.
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class TaskDispatchRequest implements RequestMessageBuilder {
public class LogicTaskDispatchRequest implements Serializable {
private static final long serialVersionUID = -1L;
private TaskExecutionContext taskExecutionContext;
@Override
public MessageType getCommandType() {
return MessageType.TASK_DISPATCH_REQUEST;
}
}

Some files were not shown because too many files have changed in this diff Show More