From e0087819b5a16ded8a3578f06bbb64392754aa72 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 11 Jan 2021 16:34:17 +0800 Subject: [PATCH 01/68] rpc --- dolphinscheduler-remote/pom.xml | 4 ++ .../dolphinscheduler/remote/rpc/Invoker.java | 12 +++++ .../rpc/client/ConsumerInterceptor.java | 49 +++++++++++++++++ .../remote/rpc/client/ConsumerInvoker.java | 18 +++++++ .../remote/rpc/client/IRpcClient.java | 12 +++++ .../remote/rpc/client/RpcClient.java | 33 ++++++++++++ .../remote/rpc/common/RpcRequest.java | 54 +++++++++++++++++++ .../remote/rpc/common/RpcResponse.java | 44 +++++++++++++++ .../remote/rpc/filter/Filter.java | 11 ++++ .../remote/rpc/filter/FilterChain.java | 38 +++++++++++++ .../remote/rpc/filter/FilterWrapper.java | 32 +++++++++++ .../remote/rpc/filter/LoaderFilters.java | 26 +++++++++ .../remote/rpc/filter/SelectorFilter.java | 41 ++++++++++++++ .../remote/rpc/selector/AbstractSelector.java | 29 ++++++++++ .../remote/rpc/selector/RandomSelector.java | 45 ++++++++++++++++ .../remote/rpc/selector/Selector.java | 16 ++++++ 16 files changed, 464 insertions(+) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 3ac7b914a5..e40b7e1274 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -47,6 +47,10 @@ org.slf4j slf4j-api + + net.bytebuddy + byte-buddy + junit junit diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java new file mode 100644 index 0000000000..e93dfa4e66 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java @@ -0,0 +1,12 @@ +package org.apache.dolphinscheduler.remote.rpc; + +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; + +/** + * Invoker + */ +public interface Invoker { + + RpcResponse invoke(RpcRequest req) throws Throwable; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java new file mode 100644 index 0000000000..e34db2dd9c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -0,0 +1,49 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import net.bytebuddy.implementation.bind.annotation.AllArguments; +import net.bytebuddy.implementation.bind.annotation.Origin; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.filter.FilterChain; + +import java.lang.reflect.Method; +import java.util.UUID; + +/** + * ConsumerInterceptor + */ +public class ConsumerInterceptor { + + private Invoker invoker; + + + private FilterChain filterChain; + + public ConsumerInterceptor(Invoker invoker) { + this.filterChain = new FilterChain(invoker); + this.invoker = this.filterChain.buildFilterChain(); + } + + + @RuntimeType + public Object intercept(@AllArguments Object[] args, @Origin Method method) throws Throwable { + RpcRequest request = buildReq(args, method); + //todo + System.out.println(invoker.invoke(request)); + return null; + + } + + private RpcRequest buildReq(Object[] args, Method method) { + RpcRequest request = new RpcRequest(); + request.setRequestId(UUID.randomUUID().toString()); + request.setClassName(method.getDeclaringClass().getName()); + request.setMethodName(method.getName()); + request.setParameterTypes(method.getParameterTypes()); + request.setParameters(args); + return request; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java new file mode 100644 index 0000000000..5ef800ec5d --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java @@ -0,0 +1,18 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +/** + * @author jiangli + * @date 2021-01-09 15:27 + */ +public class ConsumerInvoker implements Invoker { + @Override + public RpcResponse invoke(RpcRequest req) throws Throwable { + + System.out.println(req.getRequestId()+"kris"); + return null; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java new file mode 100644 index 0000000000..0b384cbe62 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java @@ -0,0 +1,12 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +/** + * @author jiangli + * @date 2021-01-09 10:58 + */ +public interface IRpcClient { + + + T create(Class clazz) throws Exception; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java new file mode 100644 index 0000000000..83b64fcc31 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java @@ -0,0 +1,33 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import net.bytebuddy.ByteBuddy; +import net.bytebuddy.implementation.MethodDelegation; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author jiangli + * @date 2021-01-09 10:59 + */ +public class RpcClient implements IRpcClient{ + + private ConcurrentHashMap classMap=new ConcurrentHashMap<>(); + + + @Override + public T create(Class clazz) throws Exception { + if(!classMap.containsKey(clazz.getName())){ + T proxy = new ByteBuddy() + .subclass(clazz) + .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(new ConsumerInvoker()))) + .make() + .load(getClass().getClassLoader()) + .getLoaded() + .getDeclaredConstructor().newInstance(); + + classMap.putIfAbsent(clazz.getName(),proxy); + } + return (T) classMap.get(clazz.getName()); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java new file mode 100644 index 0000000000..1e0eb2caf2 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java @@ -0,0 +1,54 @@ +package org.apache.dolphinscheduler.remote.rpc.common; + +/** + * @author jiangli + * @date 2021-01-09 13:21 + */ +public class RpcRequest { + + private String requestId; + private String className; + private String methodName; + private Class[] parameterTypes; + private Object[] parameters; + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } + + public String getMethodName() { + return methodName; + } + + public void setMethodName(String methodName) { + this.methodName = methodName; + } + + public Class[] getParameterTypes() { + return parameterTypes; + } + + public void setParameterTypes(Class[] parameterTypes) { + this.parameterTypes = parameterTypes; + } + + public Object[] getParameters() { + return parameters; + } + + public void setParameters(Object[] parameters) { + this.parameters = parameters; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java new file mode 100644 index 0000000000..f39889bdaf --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java @@ -0,0 +1,44 @@ +package org.apache.dolphinscheduler.remote.rpc.common; + +/** + * RpcResponse + */ +public class RpcResponse { + + private String requestId; + private String msg; + private Object result; + private Byte status; + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public Object getResult() { + return result; + } + + public void setResult(Object result) { + this.result = result; + } + + public Byte getStatus() { + return status; + } + + public void setStatus(Byte status) { + this.status = status; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java new file mode 100644 index 0000000000..11786c0dde --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java @@ -0,0 +1,11 @@ +package org.apache.dolphinscheduler.remote.rpc.filter; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +public interface Filter { + + + RpcResponse filter(Invoker invoker, RpcRequest req) throws Throwable; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java new file mode 100644 index 0000000000..abb0f3a5bf --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java @@ -0,0 +1,38 @@ +package org.apache.dolphinscheduler.remote.rpc.filter; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; + +import java.util.List; + +/** + * FilterChain + */ +public class FilterChain { + + + private List filters; + + private Invoker invoker; + + + public FilterChain(List filters, Invoker invoker) { + this.filters = filters; + this.invoker = invoker; + } + + public FilterChain(Invoker invoker) { + this(LoaderFilters.create().getFilters(), invoker); + } + + public Invoker buildFilterChain() { + // 最后一个 + Invoker last = invoker; + + for (int i = filters.size() - 1; i >= 0; i--) { + last = new FilterWrapper(filters.get(i), last); + } + // 第一个 + return last; + + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java new file mode 100644 index 0000000000..ade79ef2d0 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java @@ -0,0 +1,32 @@ +package org.apache.dolphinscheduler.remote.rpc.filter; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +/** + * @author jiangli + * @date 2021-01-11 11:48 + */ +public class FilterWrapper implements Invoker { + + + private Filter next; + + private Invoker invoker; + + + public FilterWrapper(Filter next, Invoker invoker) { + this.next = next; + this.invoker = invoker; + } + + @Override + public RpcResponse invoke(RpcRequest args) throws Throwable { + if (next != null) { + return next.filter(invoker, args); + } else { + return invoker.invoke(args); + } + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java new file mode 100644 index 0000000000..be80385b04 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java @@ -0,0 +1,26 @@ +package org.apache.dolphinscheduler.remote.rpc.filter; + +import java.util.ArrayList; +import java.util.List; + +/** + * LoaderFilters + */ +public class LoaderFilters { + + + private List filterList = new ArrayList<>(); + + private LoaderFilters() { + } + + public static LoaderFilters create() { + + return new LoaderFilters(); + } + + public List getFilters() { + filterList.add(SelectorFilter.getInstance()); + return filterList; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java new file mode 100644 index 0000000000..5cccedbc40 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java @@ -0,0 +1,41 @@ +package org.apache.dolphinscheduler.remote.rpc.filter; + +import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * SelectorFilter + */ +public class SelectorFilter implements Filter { + + + private static final Logger logger = LoggerFactory.getLogger(SelectorFilter.class); + + + private SelectorFilter selectorFilter = SelectorFilter.getInstance(); + + public static SelectorFilter getInstance() { + return SelectorFilterInner.INSTANCE; + } + + + private static class SelectorFilterInner { + + private static final SelectorFilter INSTANCE = new SelectorFilter(); + } + + private SelectorFilter() { + } + + @Override + public RpcResponse filter(Invoker invoker, RpcRequest req) throws Throwable { + RpcResponse rsp = new RpcResponse(); + rsp.setMsg("ms"); + return rsp; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java new file mode 100644 index 0000000000..8e8d68ae39 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java @@ -0,0 +1,29 @@ +package org.apache.dolphinscheduler.remote.rpc.selector; + +import org.apache.dolphinscheduler.common.utils.CollectionUtils; + +import java.util.Collection; + +/** + * AbstractSelector + */ +public abstract class AbstractSelector implements Selector{ + @Override + public T select(Collection source) { + + if (CollectionUtils.isEmpty(source)) { + throw new IllegalArgumentException("Empty source."); + } + + /** + * if only one , return directly + */ + if (source.size() == 1) { + return (T)source.toArray()[0]; + } + return doSelect(source); + } + + protected abstract T doSelect(Collection source); + +} \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java new file mode 100644 index 0000000000..b48017ebf2 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java @@ -0,0 +1,45 @@ +package org.apache.dolphinscheduler.remote.rpc.selector; + +import org.apache.dolphinscheduler.remote.utils.Host; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +/** + * @author jiangli + * @date 2021-01-11 12:00 + */ +public class RandomSelector extends AbstractSelector { + + @Override + public Host doSelect(final Collection source) { + + List hosts = new ArrayList<>(source); + int size = hosts.size(); + int[] weights = new int[size]; + int totalWeight = 0; + int index = 0; + + for (Host host : hosts) { + totalWeight += host.getWeight(); + weights[index] = host.getWeight(); + index++; + } + + if (totalWeight > 0) { + int offset = ThreadLocalRandom.current().nextInt(totalWeight); + + for (int i = 0; i < size; i++) { + offset -= weights[i]; + if (offset < 0) { + return hosts.get(i); + } + } + } + return hosts.get(ThreadLocalRandom.current().nextInt(size)); + } + +} + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java new file mode 100644 index 0000000000..d511294496 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java @@ -0,0 +1,16 @@ +package org.apache.dolphinscheduler.remote.rpc.selector; + +import java.util.Collection; + +/** + * Selector + */ +public interface Selector { + + /** + * select + * @param source source + * @return T + */ + T select(Collection source); +} From fd051530fa563917c10d177d04d5766e811ab3dc Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 12 Jan 2021 09:49:56 +0800 Subject: [PATCH 02/68] rpc --- .../remote/rpc/IUserService.java | 10 ++ .../dolphinscheduler/remote/rpc/MainTest.java | 21 +++ .../remote/rpc/UserService.java | 12 ++ .../remote/rpc/filter/Filter.java | 4 + .../remote/rpc/filter/FilterWrapper.java | 1 + .../remote/rpc/filter/SelectorFilter.java | 19 +++ .../rpc/filter/directory/Directory.java | 59 ++++++++ .../rpc/filter/selector/HostWeight.java | 87 +++++++++++ .../selector/LowerWeightRoundRobin.java | 56 +++++++ .../filter/selector/RoundRobinSelector.java | 141 ++++++++++++++++++ 10 files changed, 410 insertions(+) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java new file mode 100644 index 0000000000..e834995028 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java @@ -0,0 +1,10 @@ +package org.apache.dolphinscheduler.remote.rpc; + +/** + * @author jiangli + * @date 2021-01-11 21:05 + */ +public interface IUserService { + + String say(); +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java new file mode 100644 index 0000000000..3071b4e0b4 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -0,0 +1,21 @@ +package org.apache.dolphinscheduler.remote.rpc; + +import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; +import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; + +/** + * @author jiangli + * @date 2021-01-11 21:06 + */ +public class MainTest { + + public static void main(String[] args) throws Exception { + + RpcClient rpcClient = new RpcClient(); + IUserService userService = rpcClient.create(IUserService.class); + for (int i = 0; i < 100; i++) { + userService.say(); + } + + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java new file mode 100644 index 0000000000..bf1e6623e3 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -0,0 +1,12 @@ +package org.apache.dolphinscheduler.remote.rpc; + +/** + * @author jiangli + * @date 2021-01-11 21:05 + */ +public class UserService implements IUserService{ + @Override + public String say() { + return null; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java index 11786c0dde..30f1ff8196 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java @@ -4,6 +4,10 @@ import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +import com.amazonaws.Response; + + public interface Filter { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java index ade79ef2d0..e390f1a735 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java @@ -3,6 +3,7 @@ package org.apache.dolphinscheduler.remote.rpc.filter; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.utils.Host; /** * @author jiangli diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java index 5cccedbc40..dea5a0f652 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java @@ -3,6 +3,13 @@ package org.apache.dolphinscheduler.remote.rpc.filter; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.filter.directory.Directory; +import org.apache.dolphinscheduler.remote.rpc.selector.RandomSelector; +import org.apache.dolphinscheduler.remote.utils.Host; + +import java.nio.channels.Selector; +import java.util.ArrayList; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,6 +41,18 @@ public class SelectorFilter implements Filter { @Override public RpcResponse filter(Invoker invoker, RpcRequest req) throws Throwable { + Directory.getInstance().addServer("default","127.0.0.1:8080"); + Directory.getInstance().addServer("default","127.0.0.2:8080"); + Directory.getInstance().addServer("default","127.0.0.3:8080"); + List hosts = Directory.getInstance().getDirectory("default"); + List candidateHosts = new ArrayList<>(hosts.size()); + hosts.forEach(node -> { + Host nodeHost = Host.of(node); + nodeHost.setWorkGroup("default"); + candidateHosts.add(nodeHost); + }); + RandomSelector randomSelector = new RandomSelector(); + System.out.println(randomSelector.doSelect(candidateHosts)); RpcResponse rsp = new RpcResponse(); rsp.setMsg("ms"); return rsp; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java new file mode 100644 index 0000000000..c090e0ebb3 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java @@ -0,0 +1,59 @@ +package org.apache.dolphinscheduler.remote.rpc.filter.directory; + +import org.apache.dolphinscheduler.remote.rpc.filter.SelectorFilter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Directory + */ +public class Directory { + + + private static final Logger logger = LoggerFactory.getLogger(Directory.class); + + + private SelectorFilter selectorFilter = SelectorFilter.getInstance(); + + public static Directory getInstance() { + return Directory.DirectoryInner.INSTANCE; + } + + private static class DirectoryInner { + + private static final Directory INSTANCE = new Directory(); + } + + private Directory() { + } + + + private ConcurrentHashMap> directoryMap = new ConcurrentHashMap<>(); + + public List getDirectory(String serviceName) { + return directoryMap.get(serviceName); + } + + public boolean addServer(String serviceName, String servicePath) { + synchronized (this) { + if (directoryMap.containsKey(serviceName)) { + directoryMap.get(serviceName).add(servicePath); + return true; + } + } + directoryMap.putIfAbsent(serviceName, new ArrayList<>(Collections.singletonList(servicePath))); + return true; + } + + public boolean removeServer(String serviceName, String servicePath) { + + return true; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java new file mode 100644 index 0000000000..e6352aa8d5 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.filter.selector; + +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.remote.utils.Host; + +/** + * host weight + */ +public class HostWeight { + + private final int CPU_FACTOR = 10; + + private final int MEMORY_FACTOR = 20; + + private final int LOAD_AVERAGE_FACTOR = 70; + + private final Host host; + + private final double weight; + + private double currentWeight; + + public HostWeight(Host host, double cpu, double memory, double loadAverage) { + this.weight = getWeight(cpu, memory, loadAverage, host); + this.host = host; + this.currentWeight = weight; + } + + public double getCurrentWeight() { + return currentWeight; + } + + public double getWeight() { + return weight; + } + + public void setCurrentWeight(double currentWeight) { + this.currentWeight = currentWeight; + } + + public Host getHost() { + return host; + } + + @Override + public String toString() { + return "HostWeight{" + + "host=" + host + + ", weight=" + weight + + ", currentWeight=" + currentWeight + + '}'; + } + + private double getWeight(double cpu, double memory, double loadAverage, Host host) { + double calculateWeight = cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR; + return getWarmUpWeight(host, calculateWeight); + } + + /** + * If the warm-up is not over, add the weight + */ + private double getWarmUpWeight(Host host, double weight) { + long startTime = host.getStartTime(); + long uptime = System.currentTimeMillis() - startTime; + if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { + return weight * Constants.WARM_UP_TIME / uptime; + } + return weight; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java new file mode 100644 index 0000000000..115ca311d0 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.filter.selector; + + +import org.apache.dolphinscheduler.remote.rpc.selector.AbstractSelector; + +import java.util.Collection; + +/** + * lower weight round robin + */ +public class LowerWeightRoundRobin extends AbstractSelector { + + /** + * select + * + * @param sources sources + * @return HostWeight + */ + @Override + public HostWeight doSelect(Collection sources) { + double totalWeight = 0; + double lowWeight = 0; + HostWeight lowerNode = null; + for (HostWeight hostWeight : sources) { + totalWeight += hostWeight.getWeight(); + hostWeight.setCurrentWeight(hostWeight.getCurrentWeight() + hostWeight.getWeight()); + if (lowerNode == null || lowWeight > hostWeight.getCurrentWeight()) { + lowerNode = hostWeight; + lowWeight = hostWeight.getCurrentWeight(); + } + } + lowerNode.setCurrentWeight(lowerNode.getCurrentWeight() + totalWeight); + return lowerNode; + + } +} + + + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java new file mode 100644 index 0000000000..5859b4c96a --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.filter.selector; + +import org.apache.dolphinscheduler.remote.rpc.selector.AbstractSelector; +import org.apache.dolphinscheduler.remote.utils.Host; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * Smooth Weight Round Robin + */ +public class RoundRobinSelector extends AbstractSelector { + + private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); + + private static final int RECYCLE_PERIOD = 100000; + + private AtomicBoolean updateLock = new AtomicBoolean(); + + protected static class WeightedRoundRobin { + private int weight; + private AtomicLong current = new AtomicLong(0); + private long lastUpdate; + + int getWeight() { + return weight; + } + + void setWeight(int weight) { + this.weight = weight; + current.set(0); + } + + long increaseCurrent() { + return current.addAndGet(weight); + } + + void sel(int total) { + current.addAndGet(-1L * total); + } + + long getLastUpdate() { + return lastUpdate; + } + + void setLastUpdate(long lastUpdate) { + this.lastUpdate = lastUpdate; + } + + } + + + @Override + public Host doSelect(Collection source) { + + List hosts = new ArrayList<>(source); + String key = hosts.get(0).getWorkGroup(); + ConcurrentMap map = workGroupWeightMap.get(key); + if (map == null) { + workGroupWeightMap.putIfAbsent(key, new ConcurrentHashMap<>()); + map = workGroupWeightMap.get(key); + } + + int totalWeight = 0; + long maxCurrent = Long.MIN_VALUE; + long now = System.currentTimeMillis(); + Host selectedHost = null; + WeightedRoundRobin selectWeightRoundRobin = null; + + for (Host host : hosts) { + String workGroupHost = host.getWorkGroup() + host.getAddress(); + WeightedRoundRobin weightedRoundRobin = map.get(workGroupHost); + int weight = host.getWeight(); + if (weight < 0) { + weight = 0; + } + + if (weightedRoundRobin == null) { + weightedRoundRobin = new WeightedRoundRobin(); + // set weight + weightedRoundRobin.setWeight(weight); + map.putIfAbsent(workGroupHost, weightedRoundRobin); + weightedRoundRobin = map.get(workGroupHost); + } + if (weight != weightedRoundRobin.getWeight()) { + weightedRoundRobin.setWeight(weight); + } + + long cur = weightedRoundRobin.increaseCurrent(); + weightedRoundRobin.setLastUpdate(now); + if (cur > maxCurrent) { + maxCurrent = cur; + selectedHost = host; + selectWeightRoundRobin = weightedRoundRobin; + } + + totalWeight += weight; + } + + + if (!updateLock.get() && hosts.size() != map.size() && updateLock.compareAndSet(false, true)) { + try { + ConcurrentMap newMap = new ConcurrentHashMap<>(map); + newMap.entrySet().removeIf(item -> now - item.getValue().getLastUpdate() > RECYCLE_PERIOD); + workGroupWeightMap.put(key, newMap); + } finally { + updateLock.set(false); + } + } + + if (selectedHost != null) { + selectWeightRoundRobin.sel(totalWeight); + return selectedHost; + } + + return hosts.get(0); + } +} From e2472846c63c791c2f9e92baec00ee64cb1b0b8a Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 12 Jan 2021 14:21:05 +0800 Subject: [PATCH 03/68] rpc --- .../java/org/apache/dolphinscheduler/remote/rpc/MainTest.java | 1 - .../remote/rpc/{filter => }/directory/Directory.java | 2 +- .../dolphinscheduler/remote/rpc/filter/SelectorFilter.java | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/{filter => }/directory/Directory.java (95%) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 3071b4e0b4..fbf499eb48 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -1,6 +1,5 @@ package org.apache.dolphinscheduler.remote.rpc; -import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java index c090e0ebb3..7c4907d6dc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/directory/Directory.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java @@ -1,4 +1,4 @@ -package org.apache.dolphinscheduler.remote.rpc.filter.directory; +package org.apache.dolphinscheduler.remote.rpc.directory; import org.apache.dolphinscheduler.remote.rpc.filter.SelectorFilter; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java index dea5a0f652..30aa32e9f2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java @@ -3,11 +3,10 @@ package org.apache.dolphinscheduler.remote.rpc.filter; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.filter.directory.Directory; import org.apache.dolphinscheduler.remote.rpc.selector.RandomSelector; import org.apache.dolphinscheduler.remote.utils.Host; +im -import java.nio.channels.Selector; import java.util.ArrayList; import java.util.List; From 2327da77b147324c711859394d6cb2d40796d19f Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 12 Jan 2021 19:21:38 +0800 Subject: [PATCH 04/68] rpc --- dolphinscheduler-remote/pom.xml | 13 +++++ .../remote/NettyRemotingClient.java | 2 - .../remote/NettyRemotingServer.java | 1 - .../remote/decoder/NettyDecoder.java | 44 ++++++++++++++++ .../remote/decoder/NettyEncoder.java | 32 ++++++++++++ .../remote/handler/NettyClientHandler.java | 2 - .../remote/handler/NettyServerHandler.java | 2 - .../remote/serialize/ProtoStuffUtils.java | 52 +++++++++++++++++++ 8 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index e40b7e1274..27f0923017 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -51,6 +51,19 @@ net.bytebuddy byte-buddy + + + + io.protostuff + protostuff-core + 1.7.2 + + + + io.protostuff + protostuff-runtime + 1.7.2 + junit junit diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index c1aea90393..0092b682e8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.remote; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; -import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.exceptions.RemotingTimeoutException; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 867cf4dc56..56b31cd0b5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.remote; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; -import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.handler.NettyServerHandler; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java new file mode 100644 index 0000000000..f212b77389 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java @@ -0,0 +1,44 @@ +package org.apache.dolphinscheduler.remote.decoder; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; + +import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; + +import java.util.List; + +/** + * @author jiangli + * @date 2021-01-12 18:53 + */ +public class NettyDecoder extends ByteToMessageDecoder { + + private Class genericClass; + + // 构造函数传入向反序列化的class + public NettyDecoder(Class genericClass) { + this.genericClass = genericClass; + } + + @Override + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { + if (byteBuf.readableBytes() < 4) { + return; + } + byteBuf.markReaderIndex(); + int dataLength = byteBuf.readInt(); + if (dataLength < 0) { + channelHandlerContext.close(); + } + if (byteBuf.readableBytes() < dataLength) { + byteBuf.resetReaderIndex(); + } + //将ByteBuf转换为byte[] + byte[] data = new byte[dataLength]; + byteBuf.readBytes(data); + //将data转换成object + Object obj = ProtoStuffUtils.deserialize(data, genericClass); + list.add(obj); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java new file mode 100644 index 0000000000..3ba5781fb4 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java @@ -0,0 +1,32 @@ +package org.apache.dolphinscheduler.remote.decoder; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; + +import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; + +/** + * @author jiangli + * @date 2021-01-12 18:52 + */ +public class NettyEncoder extends MessageToByteEncoder { + + + private Class genericClass; + + // 构造函数传入向反序列化的class + public NettyEncoder(Class genericClass) { + this.genericClass = genericClass; + } + @Override + protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { + + if (genericClass.isInstance(o)) { + byte[] data = ProtoStuffUtils.serialize(o); + byteBuf.writeInt(data.length); + byteBuf.writeBytes(data); + } + + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index a988acfe17..701c3b9a09 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.remote.handler; import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 09e41e9b54..0e34f99996 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -18,8 +18,6 @@ package org.apache.dolphinscheduler.remote.handler; import org.apache.dolphinscheduler.remote.NettyRemotingServer; -import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; import org.apache.dolphinscheduler.remote.utils.Pair; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java new file mode 100644 index 0000000000..4face03f90 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -0,0 +1,52 @@ +package org.apache.dolphinscheduler.remote.serialize; + +import io.protostuff.LinkedBuffer; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author jiangli + * @date 2021-01-12 18:56 + */ +public class ProtoStuffUtils { + + private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); + + private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); + + @SuppressWarnings("unchecked") + public static byte[] serialize(T obj) { + Class clazz = (Class) obj.getClass(); + Schema schema = getSchema(clazz); + byte[] data; + try { + data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + } finally { + buffer.clear(); + } + return data; + } + + @SuppressWarnings("unchecked") + private static Schema getSchema(Class clazz) { + Schema schema = (Schema) schemaCache.get(clazz); + if (schema == null) { + schema = RuntimeSchema.getSchema(clazz); + if (schema == null) { + schemaCache.put(clazz, schema); + } + } + return schema; + } + + public static T deserialize(byte[] bytes, Class clazz) { + Schema schema = getSchema(clazz); + T obj = schema.newMessage(); + ProtostuffIOUtil.mergeFrom(bytes, obj, schema); + return obj; + } +} From c78f5e270a1ede1b59fda232f7bd3f82577acdb0 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 13 Jan 2021 09:01:29 +0800 Subject: [PATCH 05/68] rpc --- .../org/apache/dolphinscheduler/remote/NettyRemotingClient.java | 2 ++ .../org/apache/dolphinscheduler/remote/NettyRemotingServer.java | 1 + .../dolphinscheduler/remote/handler/NettyClientHandler.java | 2 ++ .../dolphinscheduler/remote/handler/NettyServerHandler.java | 2 ++ 4 files changed, 7 insertions(+) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java index 0092b682e8..c1aea90393 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingClient.java @@ -19,6 +19,8 @@ package org.apache.dolphinscheduler.remote; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.exceptions.RemotingTimeoutException; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index 56b31cd0b5..867cf4dc56 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.remote; import org.apache.dolphinscheduler.remote.codec.NettyDecoder; import org.apache.dolphinscheduler.remote.codec.NettyEncoder; +import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.handler.NettyServerHandler; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java index 701c3b9a09..a988acfe17 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyClientHandler.java @@ -18,6 +18,8 @@ package org.apache.dolphinscheduler.remote.handler; import org.apache.dolphinscheduler.remote.NettyRemotingClient; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java index 0e34f99996..09e41e9b54 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/handler/NettyServerHandler.java @@ -18,6 +18,8 @@ package org.apache.dolphinscheduler.remote.handler; import org.apache.dolphinscheduler.remote.NettyRemotingServer; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor; import org.apache.dolphinscheduler.remote.utils.ChannelUtils; import org.apache.dolphinscheduler.remote.utils.Pair; From 3bcb892dd5a61ec33b8f79902cd9f3a324582572 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 13 Jan 2021 14:00:38 +0800 Subject: [PATCH 06/68] rpc --- .../remote/rpc/remote/NettyChannel.java | 8 ++++ .../remote/rpc/remote/NettyClientHandler.java | 40 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java new file mode 100644 index 0000000000..a38f7df4a5 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java @@ -0,0 +1,8 @@ +package org.apache.dolphinscheduler.remote.rpc.remote; + +/** + * @author jiangli + * @date 2021-01-13 13:51 + */ +public class NettyChannel { +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java new file mode 100644 index 0000000000..69189ca32a --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -0,0 +1,40 @@ +package org.apache.dolphinscheduler.remote.rpc.remote; + + +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; + +/** + * @author jiangli + * @date 2021-01-13 13:33 + */ +@ChannelHandler.Sharable +public class NettyClientHandler extends ChannelInboundHandlerAdapter { + + + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + super.channelRegistered(ctx); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + super.channelRead(ctx, msg); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + super.userEventTriggered(ctx, evt); + } +} From d2f136ba09cb022e5a454197790014e81774587f Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 13 Jan 2021 14:26:28 +0800 Subject: [PATCH 07/68] rpc --- .../remote/rpc/remote/NettyClientHandler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 69189ca32a..76b5f313b5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -5,6 +5,10 @@ import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import org.apache.dolphinscheduler.remote.utils.ChannelUtils; + +import java.net.InetSocketAddress; + /** * @author jiangli * @date 2021-01-13 13:33 @@ -20,12 +24,15 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { - super.channelActive(ctx); + + ctx.channel().close(); } @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { - super.channelInactive(ctx); + InetSocketAddress address =(InetSocketAddress) ctx.channel().remoteAddress(); + ctx.channel().close(); + //todo connectManage.removeChannel(ctx.channel()); } @Override From 74a4de43b9b4ac1c0ad7d38fd9f4fc829848315c Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Thu, 14 Jan 2021 10:57:41 +0800 Subject: [PATCH 08/68] rpc --- .../remote/config/NettyServerConfig.java | 2 +- .../remote/decoder/NettyDecoder.java | 1 + .../remote/decoder/NettyEncoder.java | 6 +- .../dolphinscheduler/remote/rpc/MainTest.java | 24 +- .../remote/rpc/client/RpcRequestTable.java | 28 +++ .../remote/rpc/filter/SelectorFilter.java | 3 +- .../remote/rpc/future/RpcFuture.java | 52 +++++ .../remote/rpc/remote/NettyChannel.java | 8 - .../remote/rpc/remote/NettyClient.java | 214 ++++++++++++++++++ .../remote/rpc/remote/NettyClientHandler.java | 56 +++-- .../remote/rpc/remote/NettyServer.java | 187 +++++++++++++++ .../remote/rpc/remote/NettyServerHandler.java | 61 +++++ 12 files changed, 610 insertions(+), 32 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java index 4ec8a0f7a7..cdee158d7c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java @@ -56,7 +56,7 @@ public class NettyServerConfig { /** * listen port */ - private int listenPort = 12346; + private int listenPort = 12366; public int getListenPort() { return listenPort; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java index f212b77389..e8b31e9b7c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java @@ -23,6 +23,7 @@ public class NettyDecoder extends ByteToMessageDecoder { @Override protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { + System.out.println("decoder"); if (byteBuf.readableBytes() < 4) { return; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java index 3ba5781fb4..7a4dc6dc27 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java @@ -4,13 +4,14 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.MessageToByteEncoder; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; /** * @author jiangli * @date 2021-01-12 18:52 */ -public class NettyEncoder extends MessageToByteEncoder { +public class NettyEncoder extends MessageToByteEncoder { private Class genericClass; @@ -21,12 +22,13 @@ public class NettyEncoder extends MessageToByteEncoder { } @Override protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { - + System.out.println("encsss"); if (genericClass.isInstance(o)) { byte[] data = ProtoStuffUtils.serialize(o); byteBuf.writeInt(data.length); byteBuf.writeBytes(data); } + } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index fbf499eb48..204d9d4150 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -1,6 +1,12 @@ package org.apache.dolphinscheduler.remote.rpc; -import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; + +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; +import org.apache.dolphinscheduler.remote.utils.Host; /** * @author jiangli @@ -9,12 +15,18 @@ import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; public class MainTest { public static void main(String[] args) throws Exception { + NettyServer nettyServer=new NettyServer(new NettyServerConfig()); - RpcClient rpcClient = new RpcClient(); - IUserService userService = rpcClient.create(IUserService.class); - for (int i = 0; i < 100; i++) { - userService.say(); - } + NettyClient nettyClient=new NettyClient(new NettyClientConfig()); + + Host host=new Host("127.0.0.1",12366); + RpcRequest rpcRequest=new RpcRequest(); + rpcRequest.setRequestId("988"); + rpcRequest.setClassName("kris"); + rpcRequest.setMethodName("ll"); + + + nettyClient.sendMsg(host,rpcRequest); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java new file mode 100644 index 0000000000..677a2997e0 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java @@ -0,0 +1,28 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author jiangli + * @date 2021-01-14 10:42 + */ +public class RpcRequestTable { + + // key: requestId value: RpcFuture + private static ConcurrentHashMap processingRpc = new ConcurrentHashMap<>(); + + public static void put(String requestId,RpcFuture rpcFuture){ + processingRpc.put(requestId,rpcFuture); + } + + public static RpcFuture get(String requestId){ + return processingRpc.get(requestId); + } + + public static void remove(String requestId){ + processingRpc.remove(requestId); + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java index 30aa32e9f2..8e8214a59f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java @@ -3,9 +3,10 @@ package org.apache.dolphinscheduler.remote.rpc.filter; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.directory.Directory; import org.apache.dolphinscheduler.remote.rpc.selector.RandomSelector; import org.apache.dolphinscheduler.remote.utils.Host; -im + import java.util.ArrayList; import java.util.List; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java new file mode 100644 index 0000000000..516e05b917 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java @@ -0,0 +1,52 @@ +package org.apache.dolphinscheduler.remote.rpc.future; + +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * @author jiangli + * @date 2021-01-14 09:24 + */ +public class RpcFuture implements Future { + + private CountDownLatch latch = new CountDownLatch(1); + + private RpcResponse response; + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public Object get() throws InterruptedException, ExecutionException { + boolean b = latch.await(5,TimeUnit.SECONDS); + return response.getResult(); + } + + @Override + public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + boolean b = latch.await(timeout,unit); + return response.getResult(); + } + + public void done(RpcResponse response){ + this.response = response; + latch.countDown(); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java deleted file mode 100644 index a38f7df4a5..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyChannel.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.remote; - -/** - * @author jiangli - * @date 2021-01-13 13:51 - */ -public class NettyChannel { -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java new file mode 100644 index 0000000000..a4c74fdc41 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java @@ -0,0 +1,214 @@ +package org.apache.dolphinscheduler.remote.rpc.remote; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.CharsetUtil; + +import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; + +import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; +import org.apache.dolphinscheduler.remote.future.ResponseFuture; +import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.remote.utils.NettyUtils; + +import java.net.InetSocketAddress; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author jiangli + * @date 2021-01-13 19:31 + */ +public class NettyClient { + + private final Logger logger = LoggerFactory.getLogger(NettyClient.class); + + /** + * worker group + */ + private final EventLoopGroup workerGroup; + + /** + * client config + */ + private final NettyClientConfig clientConfig; + + + /** + * client bootstrap + */ + private final Bootstrap bootstrap = new Bootstrap(); + + /** + * started flag + */ + private final AtomicBoolean isStarted = new AtomicBoolean(false); + + /** + * channels + */ + private final ConcurrentHashMap channels = new ConcurrentHashMap(128); + + /** + * get channel + */ + public Channel getChannel(Host host) { + Channel channel = channels.get(host); + if (channel != null && channel.isActive()) { + return channel; + } + return createChannel(host, true); + } + + /** + * create channel + * + * @param host host + * @param isSync sync flag + * @return channel + */ + public Channel createChannel(Host host, boolean isSync) { + ChannelFuture future; + try { + synchronized (bootstrap) { + future = bootstrap.connect(new InetSocketAddress(host.getIp(), host.getPort())); + } + if (isSync) { + future.sync(); + } + if (future.isSuccess()) { + Channel channel = future.channel(); + channels.put(host, channel); + return channel; + } + } catch (Exception ex) { + logger.warn(String.format("connect to %s error", host), ex); + } + return null; + } + + /** + * client init + * + * @param clientConfig client config + */ + public NettyClient(final NettyClientConfig clientConfig) { + this.clientConfig = clientConfig; + if (NettyUtils.useEpoll()) { + this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); + } + }); + } else { + this.workerGroup = new NioEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClient_%d", this.threadIndex.incrementAndGet())); + } + }); + } + this.start(); + + } + + /** + * start + */ + private void start() { + + this.bootstrap + .group(this.workerGroup) + .channel(NettyUtils.getSocketChannelClass()) + .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) + .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) + .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) + .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) + .handler(new LoggingHandler(LogLevel.DEBUG)) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) { + ch.pipeline() + .addLast(new NettyEncoder(RpcRequest.class)) //OUT - 1 + .addLast(new NettyDecoder(RpcResponse.class)) + .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + + .addLast(new NettyClientHandler()); + } + }); + + isStarted.compareAndSet(false, true); + System.out.println("netty client start"); + } + + public void sendMsg(Host host, RpcRequest request) { + Channel channel = getChannel(host); + assert channel != null; + // ctx.writeAndFlush(Unpooled.copiedBuffer + RpcFuture future = new RpcFuture(); + RpcRequestTable.put(request.getRequestId(), future); + + channel.writeAndFlush(request); + // System.out.println(); + // channel.writeAndFlush( ProtoStuffUtils.serialize(request)); + + } + + + /** + * close + */ + public void close() { + if (isStarted.compareAndSet(true, false)) { + try { + closeChannels(); + if (workerGroup != null) { + this.workerGroup.shutdownGracefully(); + } + } catch (Exception ex) { + logger.error("netty client close exception", ex); + } + logger.info("netty client closed"); + } + } + + /** + * close channels + */ + private void closeChannels() { + for (Channel channel : this.channels.values()) { + channel.close(); + } + this.channels.clear(); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 76b5f313b5..4708c1096a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -1,14 +1,24 @@ package org.apache.dolphinscheduler.remote.rpc.remote; +import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; -import org.apache.dolphinscheduler.remote.utils.ChannelUtils; +import org.apache.dolphinscheduler.remote.NettyRemotingClient; +import org.apache.dolphinscheduler.remote.command.Command; +import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; import java.net.InetSocketAddress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * @author jiangli * @date 2021-01-13 13:33 @@ -17,31 +27,49 @@ import java.net.InetSocketAddress; public class NettyClientHandler extends ChannelInboundHandlerAdapter { + private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); + @Override - public void channelRegistered(ChannelHandlerContext ctx) throws Exception { - super.channelRegistered(ctx); + public void channelActive(ChannelHandlerContext ctx) { + + // ctx.channel().close(); } @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - + public void channelInactive(ChannelHandlerContext ctx) { + System.out.println("client 关闭channel"); + InetSocketAddress address = (InetSocketAddress) ctx.channel().remoteAddress(); ctx.channel().close(); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) throws Exception { - InetSocketAddress address =(InetSocketAddress) ctx.channel().remoteAddress(); - ctx.channel().close(); - //todo connectManage.removeChannel(ctx.channel()); + //todo connectManage.removeChannel(ctx.channel()); } @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - super.channelRead(ctx, msg); + System.out.println("收到消息"); + RpcResponse rsp = (RpcResponse) msg; + System.out.println(rsp); } @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - super.userEventTriggered(ctx, evt); + + if (evt instanceof IdleStateEvent){ + IdleStateEvent event = (IdleStateEvent)evt; + if (event.state()== IdleState.ALL_IDLE){ + RpcRequest request = new RpcRequest(); + request.setMethodName("heartBeat"); + ctx.channel().writeAndFlush(request); + logger.info("已超过30秒未与RPC服务器进行读写操作!将发送心跳消息..."); + } + }else{ + super.userEventTriggered(ctx,evt); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + System.out.println("exceptionCaught"); + logger.error("exceptionCaught : {}", cause.getMessage(), cause); + ctx.channel().close(); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java new file mode 100644 index 0000000000..41495f11c8 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java @@ -0,0 +1,187 @@ +package org.apache.dolphinscheduler.remote.rpc.remote; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.handler.timeout.IdleStateHandler; + +import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; + +import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; + +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.remote.utils.NettyUtils; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author jiangli + * @date 2021-01-13 19:32 + */ +public class NettyServer { + + private static final Logger logger =LoggerFactory.getLogger(NettyServer.class); + + /** + * boss group + */ + private final EventLoopGroup bossGroup; + + /** + * worker group + */ + private final EventLoopGroup workGroup; + + /** + * server config + */ + private final NettyServerConfig serverConfig; + + /** + * server bootstrap + */ + private final ServerBootstrap serverBootstrap = new ServerBootstrap(); + + /** + * started flag + */ + private final AtomicBoolean isStarted = new AtomicBoolean(false); + + /** + * server init + * + * @param serverConfig server config + */ + public NettyServer(final NettyServerConfig serverConfig) { + this.serverConfig = serverConfig; + if (NettyUtils.useEpoll()) { + this.bossGroup = new EpollEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); + } + }); + + this.workGroup = new EpollEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + } + }); + } else { + this.bossGroup = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerBossThread_%d", this.threadIndex.incrementAndGet())); + } + }); + + this.workGroup = new NioEventLoopGroup(serverConfig.getWorkerThread(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerWorkerThread_%d", this.threadIndex.incrementAndGet())); + } + }); + } + this.start(); + } + + + /** + * server start + */ + public void start() { + if (isStarted.compareAndSet(false, true)) { + this.serverBootstrap + .group(this.bossGroup, this.workGroup) + .channel(NettyUtils.getServerSocketChannelClass()) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) + .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) + .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) + .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) + .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) + .handler(new LoggingHandler(LogLevel.DEBUG)) + .childHandler(new ChannelInitializer() { + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + initNettyChannel(ch); + } + }); + + ChannelFuture future; + try { + future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); + } catch (Exception e) { + //logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + } + if (future.isSuccess()) { + // logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); + } else if (future.cause() != null) { + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause()); + } else { + throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); + } + } + System.out.println("netty ser ver start"); + } + + /** + * init netty channel + * + * @param ch socket channel + */ + private void initNettyChannel(SocketChannel ch) { + ch.pipeline() + .addLast(new NettyDecoder(RpcRequest.class)) + .addLast(new NettyEncoder(RpcResponse.class)) + .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("handler", new NettyServerHandler()); + } + + + public void close() { + if (isStarted.compareAndSet(true, false)) { + try { + if (bossGroup != null) { + this.bossGroup.shutdownGracefully(); + } + if (workGroup != null) { + this.workGroup.shutdownGracefully(); + } + + } catch (Exception ex) { + logger.error("netty server close exception", ex); + } + logger.info("netty server closed"); + } + } + + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java new file mode 100644 index 0000000000..9ee9ff2197 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -0,0 +1,61 @@ +package org.apache.dolphinscheduler.remote.rpc.remote; + +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleStateEvent; + +import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author jiangli + * @date 2021-01-13 19:20 + */ +public class NettyServerHandler extends ChannelInboundHandlerAdapter { + + private static final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + super.channelRegistered(ctx); + } + + + @Override + public void channelInactive(ChannelHandlerContext ctx){ + logger.info("channel close"); + ctx.channel().close(); + } + + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + System.out.println("客户端连接成功!"+ctx.channel().remoteAddress()); + logger.info("客户端连接成功!"+ctx.channel().remoteAddress()); + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + logger.info("server read msg"); + System.out.println("收到消息"); + RpcRequest req= (RpcRequest) msg; + System.out.println(req.getRequestId()); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + ctx.channel().close(); + } else { + super.userEventTriggered(ctx, evt); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + System.out.println("exceptionCaught"); + logger.error("exceptionCaught : {}", cause.getMessage(), cause); + ctx.channel().close(); + } +} From ed164fe3257d6f6e358a1b581fdc3f36f508fd27 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Thu, 14 Jan 2021 14:57:02 +0800 Subject: [PATCH 09/68] rpc --- .../remote/rpc/IUserService.java | 2 +- .../dolphinscheduler/remote/rpc/MainTest.java | 15 +++++++----- .../remote/rpc/UserService.java | 4 ++-- .../rpc/client/ConsumerInterceptor.java | 6 +++++ .../remote/rpc/client/RpcClient.java | 5 ++++ .../remote/rpc/remote/NettyServerHandler.java | 23 ++++++++++++++++++- 6 files changed, 45 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java index e834995028..23ae0c2586 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java @@ -6,5 +6,5 @@ package org.apache.dolphinscheduler.remote.rpc; */ public interface IUserService { - String say(); + String say(String sb); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 204d9d4150..eab927161b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -3,6 +3,8 @@ package org.apache.dolphinscheduler.remote.rpc; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; +import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; @@ -17,16 +19,17 @@ public class MainTest { public static void main(String[] args) throws Exception { NettyServer nettyServer=new NettyServer(new NettyServerConfig()); - NettyClient nettyClient=new NettyClient(new NettyClientConfig()); + // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); Host host=new Host("127.0.0.1",12366); - RpcRequest rpcRequest=new RpcRequest(); - rpcRequest.setRequestId("988"); - rpcRequest.setClassName("kris"); - rpcRequest.setMethodName("ll"); + + IRpcClient rpcClient=new RpcClient(); + IUserService userService= rpcClient.create(IUserService.class); + userService.say("calvin"); - nettyClient.sendMsg(host,rpcRequest); + + // nettyClient.sendMsg(host,rpcRequest); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index bf1e6623e3..0dfc08ad2c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -6,7 +6,7 @@ package org.apache.dolphinscheduler.remote.rpc; */ public class UserService implements IUserService{ @Override - public String say() { - return null; + public String say(String s) { + return "krris"+s; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index e34db2dd9c..e8821f7ab9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -4,9 +4,12 @@ import net.bytebuddy.implementation.bind.annotation.AllArguments; import net.bytebuddy.implementation.bind.annotation.Origin; import net.bytebuddy.implementation.bind.annotation.RuntimeType; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.filter.FilterChain; +import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.remote.utils.Host; import java.lang.reflect.Method; import java.util.UUID; @@ -32,6 +35,9 @@ public class ConsumerInterceptor { RpcRequest request = buildReq(args, method); //todo System.out.println(invoker.invoke(request)); + NettyClient nettyClient = new NettyClient(new NettyClientConfig()); + Host host = new Host("127.0.0.1", 12366); + nettyClient.sendMsg(host, request); return null; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java index 83b64fcc31..b9efdf8f9f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java @@ -4,6 +4,9 @@ import net.bytebuddy.ByteBuddy; import net.bytebuddy.implementation.MethodDelegation; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; +import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; + import java.util.concurrent.ConcurrentHashMap; /** @@ -15,6 +18,8 @@ public class RpcClient implements IRpcClient{ private ConcurrentHashMap classMap=new ConcurrentHashMap<>(); + + @Override public T create(Class clazz) throws Exception { if(!classMap.containsKey(clazz.getName())){ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index 9ee9ff2197..f5e68a38bd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -5,6 +5,10 @@ import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleStateEvent; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,11 +40,28 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) { + public void channelRead(ChannelHandlerContext ctx, Object msg) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { logger.info("server read msg"); System.out.println("收到消息"); RpcRequest req= (RpcRequest) msg; System.out.println(req.getRequestId()); + RpcResponse response=new RpcResponse(); + response.setMsg("llll"); + response.setRequestId(req.getRequestId()); + + Class handlerClass = req.getClass(); + System.out.println(req.getMethodName()); + System.out.println(req.getClassName()); + String methodName = req.getMethodName(); + Class[] parameterTypes = req.getParameterTypes(); + Object[] parameters = req.getParameters(); + + // JDK reflect + Method method = handlerClass.getMethod(methodName, parameterTypes); + method.setAccessible(true); + Object result = method.invoke(req.getClassName(), parameters); + response.setResult(result); + ctx.writeAndFlush(response); } @Override From 985a4fc99cacf6ff03f25ee91ebd2af8eddaa716 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Thu, 14 Jan 2021 17:04:55 +0800 Subject: [PATCH 10/68] rpc --- .../remote/config/NettyServerConfig.java | 2 +- .../dolphinscheduler/remote/rpc/MainTest.java | 2 +- .../rpc/client/ConsumerInterceptor.java | 2 +- .../remote/rpc/remote/NettyClientHandler.java | 2 +- .../remote/rpc/remote/NettyServerHandler.java | 31 ++++++++++++------- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java index cdee158d7c..b0fd3893ed 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java @@ -56,7 +56,7 @@ public class NettyServerConfig { /** * listen port */ - private int listenPort = 12366; + private int listenPort = 12336; public int getListenPort() { return listenPort; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index eab927161b..315f49390c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -24,7 +24,7 @@ public class MainTest { Host host=new Host("127.0.0.1",12366); IRpcClient rpcClient=new RpcClient(); - IUserService userService= rpcClient.create(IUserService.class); + IUserService userService= rpcClient.create(UserService.class); userService.say("calvin"); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index e8821f7ab9..a60d062087 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -36,7 +36,7 @@ public class ConsumerInterceptor { //todo System.out.println(invoker.invoke(request)); NettyClient nettyClient = new NettyClient(new NettyClientConfig()); - Host host = new Host("127.0.0.1", 12366); + Host host = new Host("127.0.0.1", 12336); nettyClient.sendMsg(host, request); return null; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 4708c1096a..e02cb3c925 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -47,7 +47,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { System.out.println("收到消息"); RpcResponse rsp = (RpcResponse) msg; - System.out.println(rsp); + System.out.println(rsp.getResult().toString()); } @Override diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index f5e68a38bd..b7feb22585 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -40,7 +40,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + public void channelRead(ChannelHandlerContext ctx, Object msg) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ClassNotFoundException, InstantiationException { logger.info("server read msg"); System.out.println("收到消息"); RpcRequest req= (RpcRequest) msg; @@ -49,17 +49,26 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { response.setMsg("llll"); response.setRequestId(req.getRequestId()); - Class handlerClass = req.getClass(); - System.out.println(req.getMethodName()); - System.out.println(req.getClassName()); - String methodName = req.getMethodName(); - Class[] parameterTypes = req.getParameterTypes(); - Object[] parameters = req.getParameters(); - // JDK reflect - Method method = handlerClass.getMethod(methodName, parameterTypes); - method.setAccessible(true); - Object result = method.invoke(req.getClassName(), parameters); + String classname=req.getClassName(); + //获得服务端要调用的方法名称 + String methodName=req.getMethodName(); + //获得服务端要调用方法的参数类型 + Class[] parameterTypes=req.getParameterTypes(); + //获得服务端要调用方法的每一个参数的值 + Object[] arguments=req.getParameters(); + + //创建类 + Class serviceClass=Class.forName(classname); + //创建对象 + Object object = serviceClass.newInstance(); + //获得该类的对应的方法 + Method method=serviceClass.getMethod(methodName, parameterTypes); + + //该对象调用指定方法 + Object result=method.invoke(object, arguments); + + response.setResult(result); ctx.writeAndFlush(response); } From 0db19065157325fb834f616ebd3043cf9cc30bb6 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Fri, 15 Jan 2021 06:27:25 +0800 Subject: [PATCH 11/68] rpc test --- .../dolphinscheduler/remote/rpc/MainTest.java | 16 ++++++++-------- .../remote/rpc/client/ConsumerInterceptor.java | 4 ++-- .../remote/rpc/remote/NettyClient.java | 15 ++++++++++----- .../remote/rpc/remote/NettyClientHandler.java | 7 +++++++ 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 315f49390c..9cb9e5dc78 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -17,19 +17,19 @@ import org.apache.dolphinscheduler.remote.utils.Host; public class MainTest { public static void main(String[] args) throws Exception { - NettyServer nettyServer=new NettyServer(new NettyServerConfig()); + NettyServer nettyServer = new NettyServer(new NettyServerConfig()); - // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); + // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); - Host host=new Host("127.0.0.1",12366); + Host host = new Host("127.0.0.1", 12366); - IRpcClient rpcClient=new RpcClient(); - IUserService userService= rpcClient.create(UserService.class); - userService.say("calvin"); + IRpcClient rpcClient = new RpcClient(); + IUserService userService = rpcClient.create(UserService.class); + String result = userService.say("calvin"); + System.out.println("我是你爸爸吧"+result); - - // nettyClient.sendMsg(host,rpcRequest); + // nettyClient.sendMsg(host,rpcRequest); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index a60d062087..3058cf0285 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -37,8 +37,8 @@ public class ConsumerInterceptor { System.out.println(invoker.invoke(request)); NettyClient nettyClient = new NettyClient(new NettyClientConfig()); Host host = new Host("127.0.0.1", 12336); - nettyClient.sendMsg(host, request); - return null; + return nettyClient.sendMsg(host, request); + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java index a4c74fdc41..9fb923d893 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.remote.utils.NettyUtils; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -171,16 +172,20 @@ public class NettyClient { System.out.println("netty client start"); } - public void sendMsg(Host host, RpcRequest request) { + public Object sendMsg(Host host, RpcRequest request) { Channel channel = getChannel(host); assert channel != null; - // ctx.writeAndFlush(Unpooled.copiedBuffer RpcFuture future = new RpcFuture(); RpcRequestTable.put(request.getRequestId(), future); - channel.writeAndFlush(request); - // System.out.println(); - // channel.writeAndFlush( ProtoStuffUtils.serialize(request)); + Object result = null; + try { + result=future.get(); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + return result; + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index e02cb3c925..2c6202bca7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -11,8 +11,10 @@ import io.netty.handler.timeout.IdleStateEvent; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; import java.net.InetSocketAddress; @@ -47,6 +49,11 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { System.out.println("收到消息"); RpcResponse rsp = (RpcResponse) msg; + RpcFuture rpcFuture= RpcRequestTable.get(rsp.getRequestId()); + if(null!=rpcFuture){ + RpcRequestTable.remove(rsp.getRequestId()); + rpcFuture.done(rsp); + } System.out.println(rsp.getResult().toString()); } From 9d3c37e7b844e5ec4f01b48204b456ee2fee265b Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Fri, 15 Jan 2021 06:47:02 +0800 Subject: [PATCH 12/68] rpc --- .../java/org/apache/dolphinscheduler/remote/rpc/MainTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 9cb9e5dc78..1718152f3a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -24,7 +24,7 @@ public class MainTest { Host host = new Host("127.0.0.1", 12366); IRpcClient rpcClient = new RpcClient(); - IUserService userService = rpcClient.create(UserService.class); + UserService userService = rpcClient.create(UserService.class); String result = userService.say("calvin"); System.out.println("我是你爸爸吧"+result); From e950b65945eb28051e5cd364c7137b21146a15b3 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sat, 16 Jan 2021 22:22:08 +0800 Subject: [PATCH 13/68] rpc --- .../remote/decoder/NettyDecoder.java | 6 +- .../remote/decoder/NettyEncoder.java | 6 +- .../remote/rpc/IUserService.java | 3 + .../dolphinscheduler/remote/rpc/MainTest.java | 6 +- .../remote/rpc/UserCallback.java | 15 +++++ .../remote/rpc/UserService.java | 3 + .../dolphinscheduler/remote/rpc/base/Rpc.java | 25 ++++++++ .../remote/rpc/client/ConsumerConfig.java | 61 +++++++++++++++++++ .../rpc/client/ConsumerConfigCache.java | 19 ++++++ .../rpc/client/ConsumerInterceptor.java | 40 ++++++++++-- .../remote/rpc/client/ConsumerInvoker.java | 3 +- .../remote/rpc/client/IRpcClient.java | 3 +- .../remote/rpc/client/RpcClient.java | 7 +-- .../remote/rpc/client/RpcRequestCache.java | 29 +++++++++ .../remote/rpc/client/RpcRequestTable.java | 17 +++--- .../rpc/common/AbstractRpcCallBack.java | 10 +++ .../rpc/common/ConsumerConfigConstants.java | 14 +++++ .../remote/rpc/common/RpcRequest.java | 3 +- .../remote/rpc/future/RpcFuture.java | 3 +- .../remote/rpc/remote/NettyClient.java | 22 ++++--- .../remote/rpc/remote/NettyClientHandler.java | 47 +++++--------- .../remote/rpc/remote/NettyServer.java | 3 +- .../remote/rpc/remote/NettyServerHandler.java | 54 ++++++++-------- .../remote/serialize/ProtoStuffUtils.java | 3 +- 24 files changed, 291 insertions(+), 111 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java index e8b31e9b7c..160e5f50ff 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java @@ -9,21 +9,19 @@ import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; import java.util.List; /** - * @author jiangli - * @date 2021-01-12 18:53 + * NettyDecoder */ public class NettyDecoder extends ByteToMessageDecoder { private Class genericClass; - // 构造函数传入向反序列化的class + public NettyDecoder(Class genericClass) { this.genericClass = genericClass; } @Override protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { - System.out.println("decoder"); if (byteBuf.readableBytes() < 4) { return; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java index 7a4dc6dc27..c381b3fdc4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java @@ -11,24 +11,22 @@ import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; * @author jiangli * @date 2021-01-12 18:52 */ -public class NettyEncoder extends MessageToByteEncoder { +public class NettyEncoder extends MessageToByteEncoder { private Class genericClass; - // 构造函数传入向反序列化的class public NettyEncoder(Class genericClass) { this.genericClass = genericClass; } + @Override protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { - System.out.println("encsss"); if (genericClass.isInstance(o)) { byte[] data = ProtoStuffUtils.serialize(o); byteBuf.writeInt(data.length); byteBuf.writeBytes(data); } - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java index 23ae0c2586..87bf405920 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java @@ -1,10 +1,13 @@ package org.apache.dolphinscheduler.remote.rpc; +import org.apache.dolphinscheduler.remote.rpc.base.Rpc; + /** * @author jiangli * @date 2021-01-11 21:05 */ public interface IUserService { + @Rpc(async = true,callback = UserCallback.class) String say(String sb); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 1718152f3a..df5f8d0257 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -1,12 +1,9 @@ package org.apache.dolphinscheduler.remote.rpc; -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; import org.apache.dolphinscheduler.remote.utils.Host; @@ -26,8 +23,7 @@ public class MainTest { IRpcClient rpcClient = new RpcClient(); UserService userService = rpcClient.create(UserService.class); String result = userService.say("calvin"); - System.out.println("我是你爸爸吧"+result); - + System.out.println( "异步回掉成功"+result); // nettyClient.sendMsg(host,rpcRequest); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java new file mode 100644 index 0000000000..4bd60996cb --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java @@ -0,0 +1,15 @@ +package org.apache.dolphinscheduler.remote.rpc; + +import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; + +/** + * @author jiangli + * @date 2021-01-15 07:32 + */ +public class UserCallback extends AbstractRpcCallBack { + @Override + public void run(Object object) { + String msg= (String) object; + System.out.println("我是异步回调"+msg); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index 0dfc08ad2c..8485d10f56 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -1,11 +1,14 @@ package org.apache.dolphinscheduler.remote.rpc; +import org.apache.dolphinscheduler.remote.rpc.base.Rpc; + /** * @author jiangli * @date 2021-01-11 21:05 */ public class UserService implements IUserService{ @Override + @Rpc(async = true,callback = UserCallback.class,retries = 9999,isOneway = false) public String say(String s) { return "krris"+s; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java new file mode 100644 index 0000000000..5ad64f52f4 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java @@ -0,0 +1,25 @@ +package org.apache.dolphinscheduler.remote.rpc.base; + +import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Rpc + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Rpc { + + int retries() default 3; + + boolean async() default false; + + boolean isOneway() default true; + + Class callback() default AbstractRpcCallBack.class; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java new file mode 100644 index 0000000000..bb046f435e --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java @@ -0,0 +1,61 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; +import org.apache.dolphinscheduler.remote.rpc.common.ConsumerConfigConstants; + +/** + * ConsumerConfig + */ +public class ConsumerConfig { + + private Class callBackClass; + + private String serviceName; + + private Boolean async = ConsumerConfigConstants.DEFAULT_SYNC; + + private Boolean isOneway = ConsumerConfigConstants.DEFAULT_IS_ONEWAY; + + private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; + + + public Class getCallBackClass() { + return callBackClass; + } + + public void setCallBackClass(Class callBackClass) { + this.callBackClass = callBackClass; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public Boolean getAsync() { + return async; + } + + public void setAsync(Boolean async) { + this.async = async; + } + + public Boolean getOneway() { + return isOneway; + } + + public void setOneway(Boolean oneway) { + isOneway = oneway; + } + + public Integer getRetries() { + return retries; + } + + public void setRetries(Integer retries) { + this.retries = retries; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java new file mode 100644 index 0000000000..898a1c0827 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java @@ -0,0 +1,19 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import java.util.concurrent.ConcurrentHashMap; + +/** + * ConsumerConfigCache + */ +public class ConsumerConfigCache { + + private static ConcurrentHashMap consumerMap=new ConcurrentHashMap<>(); + + public static ConsumerConfig getConfigByServersName(String serviceName){ + return consumerMap.get(serviceName); + } + + public static void putConfig(String serviceName,ConsumerConfig consumerConfig){ + consumerMap.putIfAbsent(serviceName,consumerConfig); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index 3058cf0285..ccc0fb358d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -6,6 +6,7 @@ import net.bytebuddy.implementation.bind.annotation.RuntimeType; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.rpc.Invoker; +import org.apache.dolphinscheduler.remote.rpc.base.Rpc; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.filter.FilterChain; import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; @@ -13,6 +14,7 @@ import org.apache.dolphinscheduler.remote.utils.Host; import java.lang.reflect.Method; import java.util.UUID; +import java.util.function.Consumer; /** * ConsumerInterceptor @@ -24,6 +26,8 @@ public class ConsumerInterceptor { private FilterChain filterChain; + private NettyClient nettyClient=new NettyClient(new NettyClientConfig()); + public ConsumerInterceptor(Invoker invoker) { this.filterChain = new FilterChain(invoker); this.invoker = this.filterChain.buildFilterChain(); @@ -33,13 +37,19 @@ public class ConsumerInterceptor { @RuntimeType public Object intercept(@AllArguments Object[] args, @Origin Method method) throws Throwable { RpcRequest request = buildReq(args, method); - //todo - System.out.println(invoker.invoke(request)); - NettyClient nettyClient = new NettyClient(new NettyClientConfig()); + + + String serviceName = method.getDeclaringClass().getName() + method; + ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); + if (null == consumerConfig) { + consumerConfig = cacheServiceConfig(method, serviceName); + } + boolean async = consumerConfig.getAsync(); + + //load balance Host host = new Host("127.0.0.1", 12336); - return nettyClient.sendMsg(host, request); - + return nettyClient.sendMsg(host, request, async); } private RpcRequest buildReq(Object[] args, Method method) { @@ -48,8 +58,28 @@ public class ConsumerInterceptor { request.setClassName(method.getDeclaringClass().getName()); request.setMethodName(method.getName()); request.setParameterTypes(method.getParameterTypes()); + request.setParameters(args); + + String serviceName = method.getDeclaringClass().getName() + method; + return request; } + private ConsumerConfig cacheServiceConfig(Method method, String serviceName) { + ConsumerConfig consumerConfig = new ConsumerConfig(); + consumerConfig.setServiceName(serviceName); + boolean annotationPresent = method.isAnnotationPresent(Rpc.class); + if (annotationPresent) { + Rpc rpc = method.getAnnotation(Rpc.class); + consumerConfig.setAsync(rpc.async()); + consumerConfig.setCallBackClass(rpc.callback()); + consumerConfig.setRetries(rpc.retries()); + consumerConfig.setOneway(rpc.isOneway()); + } + ConsumerConfigCache.putConfig(serviceName, consumerConfig); + + return consumerConfig; + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java index 5ef800ec5d..cc0e86a2a9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java @@ -5,8 +5,7 @@ import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; /** - * @author jiangli - * @date 2021-01-09 15:27 + * ConsumerInvoker */ public class ConsumerInvoker implements Invoker { @Override diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java index 0b384cbe62..e0e538c044 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java @@ -1,8 +1,7 @@ package org.apache.dolphinscheduler.remote.rpc.client; /** - * @author jiangli - * @date 2021-01-09 10:58 + * IRpcClient */ public interface IRpcClient { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java index b9efdf8f9f..5fa8f45f51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java @@ -5,21 +5,18 @@ import net.bytebuddy.implementation.MethodDelegation; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.rpc.base.Rpc; import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import java.util.concurrent.ConcurrentHashMap; /** - * @author jiangli - * @date 2021-01-09 10:59 + * RpcClient */ public class RpcClient implements IRpcClient{ private ConcurrentHashMap classMap=new ConcurrentHashMap<>(); - - - @Override public T create(Class clazz) throws Exception { if(!classMap.containsKey(clazz.getName())){ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java new file mode 100644 index 0000000000..5e75322ac7 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java @@ -0,0 +1,29 @@ +package org.apache.dolphinscheduler.remote.rpc.client; + +import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; + +/** + * RpcRequestCache + */ +public class RpcRequestCache { + + private RpcFuture rpcFuture; + + private String serviceName; + + public RpcFuture getRpcFuture() { + return rpcFuture; + } + + public void setRpcFuture(RpcFuture rpcFuture) { + this.rpcFuture = rpcFuture; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java index 677a2997e0..845f832526 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java @@ -5,24 +5,23 @@ import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; import java.util.concurrent.ConcurrentHashMap; /** - * @author jiangli - * @date 2021-01-14 10:42 + * RpcRequestTable */ public class RpcRequestTable { - // key: requestId value: RpcFuture - private static ConcurrentHashMap processingRpc = new ConcurrentHashMap<>(); - public static void put(String requestId,RpcFuture rpcFuture){ - processingRpc.put(requestId,rpcFuture); + private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); + + public static void put(String requestId,RpcRequestCache rpcRequestCache){ + requestMap.put(requestId,rpcRequestCache); } - public static RpcFuture get(String requestId){ - return processingRpc.get(requestId); + public static RpcRequestCache get(String requestId){ + return requestMap.get(requestId); } public static void remove(String requestId){ - processingRpc.remove(requestId); + requestMap.remove(requestId); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java new file mode 100644 index 0000000000..3e632c87f6 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java @@ -0,0 +1,10 @@ +package org.apache.dolphinscheduler.remote.rpc.common; + +/** + * AbstractRpcCallBack + */ +public abstract class AbstractRpcCallBack { + + public abstract void run(Object object); + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java new file mode 100644 index 0000000000..c844cd7b7c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java @@ -0,0 +1,14 @@ +package org.apache.dolphinscheduler.remote.rpc.common; + +/** + * ConsumerConfigConstants + */ +public class ConsumerConfigConstants { + + + public static final Boolean DEFAULT_SYNC = false; + + public static final Boolean DEFAULT_IS_ONEWAY = false; + + public static final Integer DEFAULT_RETRIES = 3; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java index 1e0eb2caf2..833ad0c73b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java @@ -1,8 +1,7 @@ package org.apache.dolphinscheduler.remote.rpc.common; /** - * @author jiangli - * @date 2021-01-09 13:21 + * RpcRequest */ public class RpcRequest { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java index 516e05b917..9a72801365 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java @@ -9,8 +9,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; /** - * @author jiangli - * @date 2021-01-14 09:24 + * RpcFuture */ public class RpcFuture implements Future { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java index 9fb923d893..c8a2317878 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java @@ -20,6 +20,7 @@ import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; import org.apache.dolphinscheduler.remote.future.ResponseFuture; +import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; @@ -41,8 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * @author jiangli - * @date 2021-01-13 19:31 + * NettyClient */ public class NettyClient { @@ -172,24 +172,30 @@ public class NettyClient { System.out.println("netty client start"); } - public Object sendMsg(Host host, RpcRequest request) { + public Object sendMsg(Host host, RpcRequest request, Boolean async) { + + System.out.println("这个不是异步"+async); Channel channel = getChannel(host); assert channel != null; + RpcRequestCache rpcRequestCache = new RpcRequestCache(); + rpcRequestCache.setServiceName(request.getClassName() + request.getMethodName()); RpcFuture future = new RpcFuture(); - RpcRequestTable.put(request.getRequestId(), future); + rpcRequestCache.setRpcFuture(future); + RpcRequestTable.put(request.getRequestId(), rpcRequestCache); channel.writeAndFlush(request); + Object result = null; + if (async) { + return true; + } try { - result=future.get(); + result = future.get(); } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } return result; - - } - /** * close */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 2c6202bca7..5e965e6063 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -1,16 +1,12 @@ package org.apache.dolphinscheduler.remote.rpc.remote; - -import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleState; import io.netty.handler.timeout.IdleStateEvent; -import org.apache.dolphinscheduler.remote.NettyRemotingClient; -import org.apache.dolphinscheduler.remote.command.Command; -import org.apache.dolphinscheduler.remote.command.CommandType; +import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; @@ -22,8 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * @author jiangli - * @date 2021-01-13 13:33 + * NettyClientHandler */ @ChannelHandler.Sharable public class NettyClientHandler extends ChannelInboundHandlerAdapter { @@ -31,45 +26,35 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); - @Override - public void channelActive(ChannelHandlerContext ctx) { - - // ctx.channel().close(); - } - @Override public void channelInactive(ChannelHandlerContext ctx) { - System.out.println("client 关闭channel"); InetSocketAddress address = (InetSocketAddress) ctx.channel().remoteAddress(); ctx.channel().close(); - //todo connectManage.removeChannel(ctx.channel()); } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - System.out.println("收到消息"); + public void channelRead(ChannelHandlerContext ctx, Object msg) { RpcResponse rsp = (RpcResponse) msg; - RpcFuture rpcFuture= RpcRequestTable.get(rsp.getRequestId()); - if(null!=rpcFuture){ + RpcRequestCache rpcRequest = RpcRequestTable.get(rsp.getRequestId()); + if (null != rpcRequest) { + RpcFuture future = rpcRequest.getRpcFuture(); RpcRequestTable.remove(rsp.getRequestId()); - rpcFuture.done(rsp); + future.done(rsp); } - System.out.println(rsp.getResult().toString()); } @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { - if (evt instanceof IdleStateEvent){ - IdleStateEvent event = (IdleStateEvent)evt; - if (event.state()== IdleState.ALL_IDLE){ - RpcRequest request = new RpcRequest(); - request.setMethodName("heartBeat"); - ctx.channel().writeAndFlush(request); - logger.info("已超过30秒未与RPC服务器进行读写操作!将发送心跳消息..."); - } - }else{ - super.userEventTriggered(ctx,evt); + if (evt instanceof IdleStateEvent) { + IdleStateEvent event = (IdleStateEvent) evt; + RpcRequest request = new RpcRequest(); + request.setMethodName("heart"); + ctx.channel().writeAndFlush(request); + logger.info("已超过30秒未与RPC服务器进行读写操作!将发送心跳消息..."); + + } else { + super.userEventTriggered(ctx, evt); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java index 41495f11c8..ed88345945 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java @@ -31,8 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * @author jiangli - * @date 2021-01-13 19:32 + * NettyServer */ public class NettyServer { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index b7feb22585..47bc3e70aa 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -14,12 +14,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * @author jiangli - * @date 2021-01-13 19:20 + * NettyServerHandler */ public class NettyServerHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); + @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { super.channelRegistered(ctx); @@ -27,47 +27,45 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { @Override - public void channelInactive(ChannelHandlerContext ctx){ + public void channelInactive(ChannelHandlerContext ctx) { logger.info("channel close"); - ctx.channel().close(); + ctx.channel().close(); } @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - System.out.println("客户端连接成功!"+ctx.channel().remoteAddress()); - logger.info("客户端连接成功!"+ctx.channel().remoteAddress()); + public void channelActive(ChannelHandlerContext ctx) { + logger.info("client connect success !" + ctx.channel().remoteAddress()); } @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ClassNotFoundException, InstantiationException { - logger.info("server read msg"); - System.out.println("收到消息"); - RpcRequest req= (RpcRequest) msg; - System.out.println(req.getRequestId()); - RpcResponse response=new RpcResponse(); - response.setMsg("llll"); + + RpcRequest req = (RpcRequest) msg; + + RpcResponse response = new RpcResponse(); + if(req.getMethodName().equals("heart")){ + logger.info("接受心跳消息!..."); + return; + } response.setRequestId(req.getRequestId()); - String classname=req.getClassName(); - //获得服务端要调用的方法名称 - String methodName=req.getMethodName(); - //获得服务端要调用方法的参数类型 - Class[] parameterTypes=req.getParameterTypes(); - //获得服务端要调用方法的每一个参数的值 - Object[] arguments=req.getParameters(); + String classname = req.getClassName(); + + String methodName = req.getMethodName(); + + Class[] parameterTypes = req.getParameterTypes(); + + Object[] arguments = req.getParameters(); + + Class serviceClass = Class.forName(classname); - //创建类 - Class serviceClass=Class.forName(classname); - //创建对象 Object object = serviceClass.newInstance(); - //获得该类的对应的方法 - Method method=serviceClass.getMethod(methodName, parameterTypes); - //该对象调用指定方法 - Object result=method.invoke(object, arguments); + Method method = serviceClass.getMethod(methodName, parameterTypes); + Object result = method.invoke(object, arguments); response.setResult(result); ctx.writeAndFlush(response); @@ -83,7 +81,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { System.out.println("exceptionCaught"); logger.error("exceptionCaught : {}", cause.getMessage(), cause); ctx.channel().close(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index 4face03f90..0277273ef5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -9,8 +9,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** - * @author jiangli - * @date 2021-01-12 18:56 + * ProtoStuffUtils */ public class ProtoStuffUtils { From 4ae52cb9b05619ec2edaacd77f6bff634b1b8878 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 17 Jan 2021 22:11:24 +0800 Subject: [PATCH 14/68] rpc --- .../remote/config/NettyServerConfig.java | 2 +- .../remote/rpc/IUserService.java | 4 +- .../dolphinscheduler/remote/rpc/MainTest.java | 14 +++- .../remote/rpc/UserCallback.java | 4 +- .../remote/rpc/UserService.java | 13 ++-- .../rpc/client/ConsumerConfigCache.java | 8 +-- .../rpc/client/ConsumerInterceptor.java | 40 +++++++---- .../remote/rpc/client/IRpcClient.java | 4 +- .../remote/rpc/client/RpcClient.java | 26 +++---- .../remote/rpc/client/RpcRequestTable.java | 2 - .../remote/rpc/future/RpcFuture.java | 8 +-- .../remote/rpc/remote/NettyClient.java | 67 +++++++++++-------- .../remote/rpc/remote/NettyClientHandler.java | 33 ++++++++- .../remote/rpc/remote/NettyServerHandler.java | 29 +++++--- .../remote/rpc/selector/RandomSelector.java | 3 +- 15 files changed, 163 insertions(+), 94 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java index b0fd3893ed..7bac0361f4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java @@ -56,7 +56,7 @@ public class NettyServerConfig { /** * listen port */ - private int listenPort = 12336; + private int listenPort = 12636; public int getListenPort() { return listenPort; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java index 87bf405920..66ef806f17 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java @@ -9,5 +9,7 @@ import org.apache.dolphinscheduler.remote.rpc.base.Rpc; public interface IUserService { @Rpc(async = true,callback = UserCallback.class) - String say(String sb); + Boolean say(String sb); + + String hi(int num); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index df5f8d0257..f295917ca4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -18,13 +18,21 @@ public class MainTest { // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); - Host host = new Host("127.0.0.1", 12366); + Host host = new Host("127.0.0.1", 12636); IRpcClient rpcClient = new RpcClient(); - UserService userService = rpcClient.create(UserService.class); - String result = userService.say("calvin"); + IUserService userService = rpcClient.create(UserService.class,host); + boolean result = userService.say("calvin"); System.out.println( "异步回掉成功"+result); + System.out.println(userService.hi(10)); + System.out.println(userService.hi(188888888)); + + UserService user = rpcClient.create(UserService.class,host); + System.out.println(user.hi(99999)); + System.out.println(user.hi(998888888)); + // UserCallback.class.newInstance().run("lllll"); + // nettyClient.sendMsg(host,rpcRequest); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java index 4bd60996cb..fe712dfbe3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java @@ -9,7 +9,7 @@ import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; public class UserCallback extends AbstractRpcCallBack { @Override public void run(Object object) { - String msg= (String) object; - System.out.println("我是异步回调"+msg); + Boolean msg= (Boolean) object; + System.out.println("我是异步回调handle Kris"+msg); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index 8485d10f56..6614e401e3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -6,10 +6,15 @@ import org.apache.dolphinscheduler.remote.rpc.base.Rpc; * @author jiangli * @date 2021-01-11 21:05 */ -public class UserService implements IUserService{ +public class UserService implements IUserService { @Override - @Rpc(async = true,callback = UserCallback.class,retries = 9999,isOneway = false) - public String say(String s) { - return "krris"+s; + @Rpc(async = true, callback = UserCallback.class, retries = 9999, isOneway = false) + public Boolean say(String s) { + return true; + } + + @Override + public String hi(int num) { + return "this world has " + num + "sun"; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java index 898a1c0827..4cd99a153b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java @@ -7,13 +7,13 @@ import java.util.concurrent.ConcurrentHashMap; */ public class ConsumerConfigCache { - private static ConcurrentHashMap consumerMap=new ConcurrentHashMap<>(); + private static ConcurrentHashMap consumerMap = new ConcurrentHashMap<>(); - public static ConsumerConfig getConfigByServersName(String serviceName){ + public static ConsumerConfig getConfigByServersName(String serviceName) { return consumerMap.get(serviceName); } - public static void putConfig(String serviceName,ConsumerConfig consumerConfig){ - consumerMap.putIfAbsent(serviceName,consumerConfig); + public static void putConfig(String serviceName, ConsumerConfig consumerConfig) { + consumerMap.putIfAbsent(serviceName, consumerConfig); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index ccc0fb358d..f98b67c504 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -1,20 +1,20 @@ package org.apache.dolphinscheduler.remote.rpc.client; -import net.bytebuddy.implementation.bind.annotation.AllArguments; -import net.bytebuddy.implementation.bind.annotation.Origin; -import net.bytebuddy.implementation.bind.annotation.RuntimeType; - -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; +import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.base.Rpc; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; import org.apache.dolphinscheduler.remote.rpc.filter.FilterChain; import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import org.apache.dolphinscheduler.remote.utils.Host; import java.lang.reflect.Method; import java.util.UUID; -import java.util.function.Consumer; + +import net.bytebuddy.implementation.bind.annotation.AllArguments; +import net.bytebuddy.implementation.bind.annotation.Origin; +import net.bytebuddy.implementation.bind.annotation.RuntimeType; /** * ConsumerInterceptor @@ -26,30 +26,41 @@ public class ConsumerInterceptor { private FilterChain filterChain; - private NettyClient nettyClient=new NettyClient(new NettyClientConfig()); + private Host host; - public ConsumerInterceptor(Invoker invoker) { + private NettyClient nettyClient = NettyClient.getInstance(); + + public ConsumerInterceptor(Invoker invoker, Host host) { this.filterChain = new FilterChain(invoker); this.invoker = this.filterChain.buildFilterChain(); + this.host = host; } @RuntimeType - public Object intercept(@AllArguments Object[] args, @Origin Method method) throws Throwable { + public Object intercept(@AllArguments Object[] args, @Origin Method method) throws RemotingException { RpcRequest request = buildReq(args, method); - String serviceName = method.getDeclaringClass().getName() + method; + String serviceName = method.getDeclaringClass().getName() + method.getName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (null == consumerConfig) { consumerConfig = cacheServiceConfig(method, serviceName); } boolean async = consumerConfig.getAsync(); - //load balance - Host host = new Host("127.0.0.1", 12336); + int retries = consumerConfig.getRetries(); + + while (retries-- > 0) { + RpcResponse rsp = (RpcResponse) nettyClient.sendMsg(host, request, async); + //success + if (null != rsp && rsp.getStatus() == 0) { + return rsp.getResult(); + } + } + // execute fail + throw new RemotingException("send msg error"); - return nettyClient.sendMsg(host, request, async); } private RpcRequest buildReq(Object[] args, Method method) { @@ -61,7 +72,7 @@ public class ConsumerInterceptor { request.setParameters(args); - String serviceName = method.getDeclaringClass().getName() + method; + String serviceName = method.getDeclaringClass().getName(); return request; } @@ -77,6 +88,7 @@ public class ConsumerInterceptor { consumerConfig.setRetries(rpc.retries()); consumerConfig.setOneway(rpc.isOneway()); } + ConsumerConfigCache.putConfig(serviceName, consumerConfig); return consumerConfig; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java index e0e538c044..abdc5d7289 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java @@ -1,11 +1,13 @@ package org.apache.dolphinscheduler.remote.rpc.client; +import org.apache.dolphinscheduler.remote.utils.Host; + /** * IRpcClient */ public interface IRpcClient { - T create(Class clazz) throws Exception; + T create(Class clazz, Host host) throws Exception; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java index 5fa8f45f51..1009750cb9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java @@ -1,14 +1,13 @@ package org.apache.dolphinscheduler.remote.rpc.client; +import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; + +import org.apache.dolphinscheduler.remote.utils.Host; + +import java.util.concurrent.ConcurrentHashMap; + import net.bytebuddy.ByteBuddy; import net.bytebuddy.implementation.MethodDelegation; -import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; - -import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.rpc.base.Rpc; -import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; - -import java.util.concurrent.ConcurrentHashMap; /** * RpcClient @@ -18,18 +17,19 @@ public class RpcClient implements IRpcClient{ private ConcurrentHashMap classMap=new ConcurrentHashMap<>(); @Override - public T create(Class clazz) throws Exception { - if(!classMap.containsKey(clazz.getName())){ + public T create(Class clazz,Host host) throws Exception { + // if(!classMap.containsKey(clazz.getName())){ T proxy = new ByteBuddy() .subclass(clazz) - .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(new ConsumerInvoker()))) + .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(new ConsumerInvoker(),host))) .make() .load(getClass().getClassLoader()) .getLoaded() .getDeclaredConstructor().newInstance(); - classMap.putIfAbsent(clazz.getName(),proxy); - } - return (T) classMap.get(clazz.getName()); + // classMap.putIfAbsent(clazz.getName(),proxy); + return proxy; + // } + // return (T) classMap.get(clazz.getName()); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java index 845f832526..344a6e4a9b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java @@ -1,7 +1,5 @@ package org.apache.dolphinscheduler.remote.rpc.client; -import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; - import java.util.concurrent.ConcurrentHashMap; /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java index 9a72801365..2e2b1a734e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java @@ -33,15 +33,15 @@ public class RpcFuture implements Future { } @Override - public Object get() throws InterruptedException, ExecutionException { + public RpcResponse get() throws InterruptedException, ExecutionException { boolean b = latch.await(5,TimeUnit.SECONDS); - return response.getResult(); + return response; } @Override - public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { boolean b = latch.await(timeout,unit); - return response.getResult(); + return response; } public void done(RpcResponse response){ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java index c8a2317878..a5f13a8343 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java @@ -1,31 +1,13 @@ package org.apache.dolphinscheduler.remote.rpc.remote; -import io.netty.bootstrap.Bootstrap; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.epoll.EpollEventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import io.netty.handler.timeout.IdleStateHandler; -import io.netty.util.CharsetUtil; - -import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; - +import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; -import org.apache.dolphinscheduler.remote.future.ResponseFuture; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; -import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NettyUtils; @@ -41,11 +23,33 @@ import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.epoll.EpollEventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import io.netty.handler.timeout.IdleStateHandler; + /** * NettyClient */ public class NettyClient { + public static NettyClient getInstance() { + return NettyClient.NettyClientInner.INSTANCE; + } + + private static class NettyClientInner { + + private static final NettyClient INSTANCE = new NettyClient(new NettyClientConfig()); + } + private final Logger logger = LoggerFactory.getLogger(NettyClient.class); /** @@ -160,38 +164,43 @@ public class NettyClient { @Override public void initChannel(SocketChannel ch) { ch.pipeline() - .addLast(new NettyEncoder(RpcRequest.class)) //OUT - 1 + .addLast(new NettyEncoder(RpcRequest.class)) .addLast(new NettyDecoder(RpcResponse.class)) .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) - .addLast(new NettyClientHandler()); } }); isStarted.compareAndSet(false, true); - System.out.println("netty client start"); } - public Object sendMsg(Host host, RpcRequest request, Boolean async) { + public RpcResponse sendMsg(Host host, RpcRequest request, Boolean async) { - System.out.println("这个不是异步"+async); Channel channel = getChannel(host); assert channel != null; RpcRequestCache rpcRequestCache = new RpcRequestCache(); rpcRequestCache.setServiceName(request.getClassName() + request.getMethodName()); - RpcFuture future = new RpcFuture(); - rpcRequestCache.setRpcFuture(future); + + + RpcFuture future = null; + if (!async) { + future = new RpcFuture(); + rpcRequestCache.setRpcFuture(future); + } RpcRequestTable.put(request.getRequestId(), rpcRequestCache); channel.writeAndFlush(request); - Object result = null; + RpcResponse result = null; if (async) { - return true; + result=new RpcResponse(); + result.setStatus((byte)0); + result.setResult(true); + return result; } try { result = future.get(); } catch (InterruptedException | ExecutionException e) { - e.printStackTrace(); + logger.error("send msg error",e); } return result; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 5e965e6063..691c725577 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -3,9 +3,11 @@ package org.apache.dolphinscheduler.remote.rpc.remote; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleState; import io.netty.handler.timeout.IdleStateEvent; +import io.netty.util.concurrent.FastThreadLocalThread; +import org.apache.dolphinscheduler.remote.rpc.client.ConsumerConfig; +import org.apache.dolphinscheduler.remote.rpc.client.ConsumerConfigCache; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; @@ -36,11 +38,36 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { public void channelRead(ChannelHandlerContext ctx, Object msg) { RpcResponse rsp = (RpcResponse) msg; RpcRequestCache rpcRequest = RpcRequestTable.get(rsp.getRequestId()); - if (null != rpcRequest) { + + if (null == rpcRequest) { + logger.warn("未知响应"); + return; + } + + String serviceName = rpcRequest.getServiceName(); + ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); + if (!consumerConfig.getAsync()) { RpcFuture future = rpcRequest.getRpcFuture(); RpcRequestTable.remove(rsp.getRequestId()); future.done(rsp); + return; + } + + //async + new FastThreadLocalThread(() -> { + try { + if (rsp.getStatus() == 0) { + consumerConfig.getCallBackClass().newInstance().run(rsp.getResult()); + } else { + logger.error("xxxx fail"); + } + } catch (InstantiationException | IllegalAccessException e) { + logger.error("execute async error", e); + } + }).start(); + + } @Override @@ -51,7 +78,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { RpcRequest request = new RpcRequest(); request.setMethodName("heart"); ctx.channel().writeAndFlush(request); - logger.info("已超过30秒未与RPC服务器进行读写操作!将发送心跳消息..."); + logger.debug("send heart beat msg..."); } else { super.userEventTriggered(ctx, evt); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index 47bc3e70aa..9cd7f2a4d6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -1,18 +1,17 @@ package org.apache.dolphinscheduler.remote.rpc.remote; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; - import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleStateEvent; + /** * NettyServerHandler */ @@ -39,17 +38,19 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, ClassNotFoundException, InstantiationException { + public void channelRead(ChannelHandlerContext ctx, Object msg) { RpcRequest req = (RpcRequest) msg; RpcResponse response = new RpcResponse(); - if(req.getMethodName().equals("heart")){ + if (req.getMethodName().equals("heart")) { logger.info("接受心跳消息!..."); return; } response.setRequestId(req.getRequestId()); + response.setStatus((byte) 0); + String classname = req.getClassName(); @@ -58,14 +59,20 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { Class[] parameterTypes = req.getParameterTypes(); Object[] arguments = req.getParameters(); + Object result = null; + try { - Class serviceClass = Class.forName(classname); + Class serviceClass = Class.forName(classname); - Object object = serviceClass.newInstance(); + Object object = serviceClass.newInstance(); - Method method = serviceClass.getMethod(methodName, parameterTypes); + Method method = serviceClass.getMethod(methodName, parameterTypes); - Object result = method.invoke(object, arguments); + result = method.invoke(object, arguments); + } catch (Exception e) { + logger.error("netty server execute error",e); + response.setStatus((byte)-1); + } response.setResult(result); ctx.writeAndFlush(response); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java index b48017ebf2..0a8c1b3666 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java @@ -8,8 +8,7 @@ import java.util.List; import java.util.concurrent.ThreadLocalRandom; /** - * @author jiangli - * @date 2021-01-11 12:00 + * RandomSelector */ public class RandomSelector extends AbstractSelector { From 939ee1f2f5e88316c48333fb880871fe95ea7c8a Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 18 Jan 2021 09:20:25 +0800 Subject: [PATCH 15/68] rpc --- .../dolphinscheduler/remote/rpc/MainTest.java | 13 +++++-------- .../remote/rpc/common/RpcRequest.java | 10 ++++++++++ .../remote/rpc/remote/NettyClientHandler.java | 2 +- .../remote/rpc/remote/NettyServer.java | 6 +++--- .../remote/rpc/remote/NettyServerHandler.java | 7 ++++--- 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index f295917ca4..2296782273 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -1,7 +1,6 @@ package org.apache.dolphinscheduler.remote.rpc; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; - import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; @@ -14,6 +13,8 @@ import org.apache.dolphinscheduler.remote.utils.Host; public class MainTest { public static void main(String[] args) throws Exception { + + NettyServer nettyServer = new NettyServer(new NettyServerConfig()); // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); @@ -21,19 +22,15 @@ public class MainTest { Host host = new Host("127.0.0.1", 12636); IRpcClient rpcClient = new RpcClient(); - IUserService userService = rpcClient.create(UserService.class,host); + IUserService userService = rpcClient.create(UserService.class, host); boolean result = userService.say("calvin"); - System.out.println( "异步回掉成功"+result); + System.out.println("异步回掉成功" + result); System.out.println(userService.hi(10)); System.out.println(userService.hi(188888888)); - UserService user = rpcClient.create(UserService.class,host); + UserService user = rpcClient.create(UserService.class, host); System.out.println(user.hi(99999)); System.out.println(user.hi(998888888)); - // UserCallback.class.newInstance().run("lllll"); - - // nettyClient.sendMsg(host,rpcRequest); - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java index 833ad0c73b..2cb4b23b7f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java @@ -10,6 +10,16 @@ public class RpcRequest { private String methodName; private Class[] parameterTypes; private Object[] parameters; + // 0 hear beat,1 businness msg + private Byte eventType=1; + + public Byte getEventType() { + return eventType; + } + + public void setEventType(Byte eventType) { + this.eventType = eventType; + } public String getRequestId() { return requestId; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index 691c725577..d6c9387b5b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -76,7 +76,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { if (evt instanceof IdleStateEvent) { IdleStateEvent event = (IdleStateEvent) evt; RpcRequest request = new RpcRequest(); - request.setMethodName("heart"); + request.setEventType((byte)0); ctx.channel().writeAndFlush(request); logger.debug("send heart beat msg..."); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java index ed88345945..091a697ff3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java @@ -128,7 +128,7 @@ public class NettyServer { .childHandler(new ChannelInitializer() { @Override - protected void initChannel(SocketChannel ch) throws Exception { + protected void initChannel(SocketChannel ch){ initNettyChannel(ch); } }); @@ -137,11 +137,11 @@ public class NettyServer { try { future = serverBootstrap.bind(serverConfig.getListenPort()).sync(); } catch (Exception e) { - //logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); + logger.error("NettyRemotingServer bind fail {}, exit", e.getMessage(), e); throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } if (future.isSuccess()) { - // logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); + logger.info("NettyRemotingServer bind success at port : {}", serverConfig.getListenPort()); } else if (future.cause() != null) { throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort()), future.cause()); } else { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index 9cd7f2a4d6..0b75d37b66 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -43,7 +43,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { RpcRequest req = (RpcRequest) msg; RpcResponse response = new RpcResponse(); - if (req.getMethodName().equals("heart")) { + if (req.getEventType() == 0) { + logger.info("接受心跳消息!..."); return; } @@ -70,8 +71,8 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { result = method.invoke(object, arguments); } catch (Exception e) { - logger.error("netty server execute error",e); - response.setStatus((byte)-1); + logger.error("netty server execute error", e); + response.setStatus((byte) -1); } response.setResult(result); From 6fafcc2f0cf0233a09b4c265b3d30af1044a4668 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 18 Jan 2021 09:27:13 +0800 Subject: [PATCH 16/68] rpc --- .../dolphinscheduler/remote/rpc/IUserService.java | 15 --------------- .../dolphinscheduler/remote/rpc/MainTest.java | 2 +- .../dolphinscheduler/remote/rpc/UserService.java | 7 +++---- .../dolphinscheduler/remote/rpc/base/Rpc.java | 2 -- .../remote/rpc/client/ConsumerConfig.java | 10 ---------- .../remote/rpc/client/ConsumerInterceptor.java | 1 - .../rpc/common/ConsumerConfigConstants.java | 2 -- 7 files changed, 4 insertions(+), 35 deletions(-) delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java deleted file mode 100644 index 66ef806f17..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/IUserService.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.rpc.base.Rpc; - -/** - * @author jiangli - * @date 2021-01-11 21:05 - */ -public interface IUserService { - - @Rpc(async = true,callback = UserCallback.class) - Boolean say(String sb); - - String hi(int num); -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 2296782273..56b4de9f22 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -22,7 +22,7 @@ public class MainTest { Host host = new Host("127.0.0.1", 12636); IRpcClient rpcClient = new RpcClient(); - IUserService userService = rpcClient.create(UserService.class, host); + UserService userService = rpcClient.create(UserService.class, host); boolean result = userService.say("calvin"); System.out.println("异步回掉成功" + result); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index 6614e401e3..57382c4368 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -6,14 +6,13 @@ import org.apache.dolphinscheduler.remote.rpc.base.Rpc; * @author jiangli * @date 2021-01-11 21:05 */ -public class UserService implements IUserService { - @Override - @Rpc(async = true, callback = UserCallback.class, retries = 9999, isOneway = false) +public class UserService { + + @Rpc(async = true, callback = UserCallback.class, retries = 9999) public Boolean say(String s) { return true; } - @Override public String hi(int num) { return "this world has " + num + "sun"; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java index 5ad64f52f4..b580aef3dd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java @@ -18,8 +18,6 @@ public @interface Rpc { boolean async() default false; - boolean isOneway() default true; - Class callback() default AbstractRpcCallBack.class; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java index bb046f435e..639e022188 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java @@ -14,8 +14,6 @@ public class ConsumerConfig { private Boolean async = ConsumerConfigConstants.DEFAULT_SYNC; - private Boolean isOneway = ConsumerConfigConstants.DEFAULT_IS_ONEWAY; - private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; @@ -43,14 +41,6 @@ public class ConsumerConfig { this.async = async; } - public Boolean getOneway() { - return isOneway; - } - - public void setOneway(Boolean oneway) { - isOneway = oneway; - } - public Integer getRetries() { return retries; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index f98b67c504..cde981b09f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -86,7 +86,6 @@ public class ConsumerInterceptor { consumerConfig.setAsync(rpc.async()); consumerConfig.setCallBackClass(rpc.callback()); consumerConfig.setRetries(rpc.retries()); - consumerConfig.setOneway(rpc.isOneway()); } ConsumerConfigCache.putConfig(serviceName, consumerConfig); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java index c844cd7b7c..3a0a5542b0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java @@ -8,7 +8,5 @@ public class ConsumerConfigConstants { public static final Boolean DEFAULT_SYNC = false; - public static final Boolean DEFAULT_IS_ONEWAY = false; - public static final Integer DEFAULT_RETRIES = 3; } From 21cec6254722636c4c0fe1ab6a1f321a222c03ba Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 20 Jan 2021 17:37:23 +0800 Subject: [PATCH 17/68] test --- .../remote/rpc/MainServerTest.java | 15 +++++++++++++++ .../dolphinscheduler/remote/rpc/MainTest.java | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java new file mode 100644 index 0000000000..88e0f615f1 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java @@ -0,0 +1,15 @@ +package org.apache.dolphinscheduler.remote.rpc; + +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; + +/** + * @author jiangli + * @date 2021-01-20 14:54 + */ +public class MainServerTest { + + public static void main(String[] args) { + NettyServer nettyServer = new NettyServer(new NettyServerConfig()); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 56b4de9f22..53d56f5c51 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -15,7 +15,7 @@ public class MainTest { public static void main(String[] args) throws Exception { - NettyServer nettyServer = new NettyServer(new NettyServerConfig()); + // NettyServer nettyServer = new NettyServer(new NettyServerConfig()); // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); From ff67386ce3ab12e6c56ac3fa8b30543a320c604b Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Fri, 22 Jan 2021 10:05:01 +0800 Subject: [PATCH 18/68] update --- .../remote/rpc/UserService.java | 2 +- .../dolphinscheduler/remote/rpc/base/Rpc.java | 32 +++- .../remote/rpc/client/ConsumerConfig.java | 29 +++- .../rpc/client/ConsumerConfigCache.java | 19 ++- .../rpc/client/ConsumerInterceptor.java | 28 ++-- .../remote/rpc/client/ConsumerInvoker.java | 17 --- .../remote/rpc/client/IRpcClient.java | 17 +++ .../remote/rpc/client/RpcClient.java | 37 +++-- .../remote/rpc/client/RpcRequestCache.java | 17 +++ .../remote/rpc/client/RpcRequestTable.java | 17 +++ .../rpc/common/AbstractRpcCallBack.java | 22 +++ .../rpc/common/ConsumerConfigConstants.java | 18 ++- .../remote/rpc/common/RpcRequest.java | 19 ++- .../remote/rpc/common/RpcResponse.java | 18 +++ .../remote/rpc/directory/Directory.java | 59 -------- .../remote/rpc/filter/Filter.java | 15 -- .../remote/rpc/filter/FilterChain.java | 38 ----- .../remote/rpc/filter/FilterWrapper.java | 33 ---- .../remote/rpc/filter/LoaderFilters.java | 26 ---- .../remote/rpc/filter/SelectorFilter.java | 60 -------- .../rpc/filter/selector/HostWeight.java | 87 ----------- .../selector/LowerWeightRoundRobin.java | 56 ------- .../filter/selector/RoundRobinSelector.java | 141 ------------------ .../remote/rpc/future/RpcFuture.java | 17 +++ .../remote/rpc/remote/NettyClient.java | 17 +++ .../remote/rpc/remote/NettyClientHandler.java | 17 +++ .../remote/rpc/remote/NettyServer.java | 17 +++ .../remote/rpc/remote/NettyServerHandler.java | 17 +++ .../remote/rpc/selector/AbstractSelector.java | 29 ---- .../remote/rpc/selector/RandomSelector.java | 44 ------ .../remote/rpc/selector/Selector.java | 16 -- .../remote/serialize/ProtoStuffUtils.java | 17 +++ 32 files changed, 347 insertions(+), 651 deletions(-) delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index 57382c4368..a8f3c35adc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -8,7 +8,7 @@ import org.apache.dolphinscheduler.remote.rpc.base.Rpc; */ public class UserService { - @Rpc(async = true, callback = UserCallback.class, retries = 9999) + @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) public Boolean say(String s) { return true; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java index b580aef3dd..7ccee632b9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.base; import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; @@ -14,10 +31,23 @@ import java.lang.annotation.Target; @Retention(RetentionPolicy.RUNTIME) public @interface Rpc { + /** + * number of retries + */ int retries() default 3; boolean async() default false; - Class callback() default AbstractRpcCallBack.class; + boolean ack() default false; + + /** + * When it is asynchronous transmission, callback must be set + */ + Class serviceCallback() default AbstractRpcCallBack.class; + + Class ackCallback() default AbstractRpcCallBack.class; + + + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java index 639e022188..757bb0778b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java @@ -1,10 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; import org.apache.dolphinscheduler.remote.rpc.common.ConsumerConfigConstants; /** - * ConsumerConfig + * We will cache the consumer configuration, when the rpc call is generated, the consumer configuration will be first obtained from here */ public class ConsumerConfig { @@ -16,12 +33,12 @@ public class ConsumerConfig { private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; - public Class getCallBackClass() { return callBackClass; } - public void setCallBackClass(Class callBackClass) { + //set call back class + void setCallBackClass(Class callBackClass) { this.callBackClass = callBackClass; } @@ -37,15 +54,15 @@ public class ConsumerConfig { return async; } - public void setAsync(Boolean async) { + void setAsync(Boolean async) { this.async = async; } - public Integer getRetries() { + Integer getRetries() { return retries; } - public void setRetries(Integer retries) { + void setRetries(Integer retries) { this.retries = retries; } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java index 4cd99a153b..f8d7d570b0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import java.util.concurrent.ConcurrentHashMap; @@ -13,7 +30,7 @@ public class ConsumerConfigCache { return consumerMap.get(serviceName); } - public static void putConfig(String serviceName, ConsumerConfig consumerConfig) { + static void putConfig(String serviceName, ConsumerConfig consumerConfig) { consumerMap.putIfAbsent(serviceName, consumerConfig); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index cde981b09f..79bf01718a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; @@ -5,7 +22,6 @@ import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.base.Rpc; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.filter.FilterChain; import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import org.apache.dolphinscheduler.remote.utils.Host; @@ -24,24 +40,18 @@ public class ConsumerInterceptor { private Invoker invoker; - private FilterChain filterChain; - private Host host; private NettyClient nettyClient = NettyClient.getInstance(); - public ConsumerInterceptor(Invoker invoker, Host host) { - this.filterChain = new FilterChain(invoker); - this.invoker = this.filterChain.buildFilterChain(); + public ConsumerInterceptor(Host host) { this.host = host; } - @RuntimeType public Object intercept(@AllArguments Object[] args, @Origin Method method) throws RemotingException { RpcRequest request = buildReq(args, method); - String serviceName = method.getDeclaringClass().getName() + method.getName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (null == consumerConfig) { @@ -52,7 +62,7 @@ public class ConsumerInterceptor { int retries = consumerConfig.getRetries(); while (retries-- > 0) { - RpcResponse rsp = (RpcResponse) nettyClient.sendMsg(host, request, async); + RpcResponse rsp = nettyClient.sendMsg(host, request, async); //success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java deleted file mode 100644 index cc0e86a2a9..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInvoker.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.client; - -import org.apache.dolphinscheduler.remote.rpc.Invoker; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; - -/** - * ConsumerInvoker - */ -public class ConsumerInvoker implements Invoker { - @Override - public RpcResponse invoke(RpcRequest req) throws Throwable { - - System.out.println(req.getRequestId()+"kris"); - return null; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java index abdc5d7289..ac7574ed1a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import org.apache.dolphinscheduler.remote.utils.Host; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java index 1009750cb9..b8d1202433 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; @@ -12,24 +29,24 @@ import net.bytebuddy.implementation.MethodDelegation; /** * RpcClient */ -public class RpcClient implements IRpcClient{ +public class RpcClient implements IRpcClient { - private ConcurrentHashMap classMap=new ConcurrentHashMap<>(); + private ConcurrentHashMap classMap = new ConcurrentHashMap<>(); @Override - public T create(Class clazz,Host host) throws Exception { - // if(!classMap.containsKey(clazz.getName())){ - T proxy = new ByteBuddy() + public T create(Class clazz, Host host) throws Exception { + // if(!classMap.containsKey(clazz.getName())){ + T proxy = new ByteBuddy() .subclass(clazz) - .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(new ConsumerInvoker(),host))) + .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(host))) .make() .load(getClass().getClassLoader()) .getLoaded() .getDeclaredConstructor().newInstance(); - // classMap.putIfAbsent(clazz.getName(),proxy); - return proxy; - // } - // return (T) classMap.get(clazz.getName()); + // classMap.putIfAbsent(clazz.getName(),proxy); + return proxy; + // } + // return (T) classMap.get(clazz.getName()); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java index 5e75322ac7..153ceba293 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java index 344a6e4a9b..ae31252d9a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.client; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java index 3e632c87f6..758b8474bb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.common; /** @@ -5,6 +22,11 @@ package org.apache.dolphinscheduler.remote.rpc.common; */ public abstract class AbstractRpcCallBack { + /** + * When sending an asynchronous message, this method will be called after the response is successfully sent. + * + * @param object response + */ public abstract void run(Object object); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java index 3a0a5542b0..d23307aedf 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.common; /** @@ -5,7 +22,6 @@ package org.apache.dolphinscheduler.remote.rpc.common; */ public class ConsumerConfigConstants { - public static final Boolean DEFAULT_SYNC = false; public static final Integer DEFAULT_RETRIES = 3; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java index 2cb4b23b7f..7d2480f642 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.common; /** @@ -11,7 +28,7 @@ public class RpcRequest { private Class[] parameterTypes; private Object[] parameters; // 0 hear beat,1 businness msg - private Byte eventType=1; + private Byte eventType = 1; public Byte getEventType() { return eventType; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java index f39889bdaf..0a8c5ef799 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.common; /** @@ -9,6 +26,7 @@ public class RpcResponse { private String msg; private Object result; private Byte status; + private Integer responseType; public String getRequestId() { return requestId; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java deleted file mode 100644 index 7c4907d6dc..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/directory/Directory.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.directory; - -import org.apache.dolphinscheduler.remote.rpc.filter.SelectorFilter; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Directory - */ -public class Directory { - - - private static final Logger logger = LoggerFactory.getLogger(Directory.class); - - - private SelectorFilter selectorFilter = SelectorFilter.getInstance(); - - public static Directory getInstance() { - return Directory.DirectoryInner.INSTANCE; - } - - private static class DirectoryInner { - - private static final Directory INSTANCE = new Directory(); - } - - private Directory() { - } - - - private ConcurrentHashMap> directoryMap = new ConcurrentHashMap<>(); - - public List getDirectory(String serviceName) { - return directoryMap.get(serviceName); - } - - public boolean addServer(String serviceName, String servicePath) { - synchronized (this) { - if (directoryMap.containsKey(serviceName)) { - directoryMap.get(serviceName).add(servicePath); - return true; - } - } - directoryMap.putIfAbsent(serviceName, new ArrayList<>(Collections.singletonList(servicePath))); - return true; - } - - public boolean removeServer(String serviceName, String servicePath) { - - return true; - } - -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java deleted file mode 100644 index 30f1ff8196..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/Filter.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.filter; - -import org.apache.dolphinscheduler.remote.rpc.Invoker; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; - - -import com.amazonaws.Response; - - -public interface Filter { - - - RpcResponse filter(Invoker invoker, RpcRequest req) throws Throwable; -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java deleted file mode 100644 index abb0f3a5bf..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterChain.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.filter; - -import org.apache.dolphinscheduler.remote.rpc.Invoker; - -import java.util.List; - -/** - * FilterChain - */ -public class FilterChain { - - - private List filters; - - private Invoker invoker; - - - public FilterChain(List filters, Invoker invoker) { - this.filters = filters; - this.invoker = invoker; - } - - public FilterChain(Invoker invoker) { - this(LoaderFilters.create().getFilters(), invoker); - } - - public Invoker buildFilterChain() { - // 最后一个 - Invoker last = invoker; - - for (int i = filters.size() - 1; i >= 0; i--) { - last = new FilterWrapper(filters.get(i), last); - } - // 第一个 - return last; - - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java deleted file mode 100644 index e390f1a735..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/FilterWrapper.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.filter; - -import org.apache.dolphinscheduler.remote.rpc.Invoker; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.utils.Host; - -/** - * @author jiangli - * @date 2021-01-11 11:48 - */ -public class FilterWrapper implements Invoker { - - - private Filter next; - - private Invoker invoker; - - - public FilterWrapper(Filter next, Invoker invoker) { - this.next = next; - this.invoker = invoker; - } - - @Override - public RpcResponse invoke(RpcRequest args) throws Throwable { - if (next != null) { - return next.filter(invoker, args); - } else { - return invoker.invoke(args); - } - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java deleted file mode 100644 index be80385b04..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/LoaderFilters.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.filter; - -import java.util.ArrayList; -import java.util.List; - -/** - * LoaderFilters - */ -public class LoaderFilters { - - - private List filterList = new ArrayList<>(); - - private LoaderFilters() { - } - - public static LoaderFilters create() { - - return new LoaderFilters(); - } - - public List getFilters() { - filterList.add(SelectorFilter.getInstance()); - return filterList; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java deleted file mode 100644 index 8e8214a59f..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/SelectorFilter.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.filter; - -import org.apache.dolphinscheduler.remote.rpc.Invoker; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.directory.Directory; -import org.apache.dolphinscheduler.remote.rpc.selector.RandomSelector; -import org.apache.dolphinscheduler.remote.utils.Host; - - -import java.util.ArrayList; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * SelectorFilter - */ -public class SelectorFilter implements Filter { - - - private static final Logger logger = LoggerFactory.getLogger(SelectorFilter.class); - - - private SelectorFilter selectorFilter = SelectorFilter.getInstance(); - - public static SelectorFilter getInstance() { - return SelectorFilterInner.INSTANCE; - } - - - private static class SelectorFilterInner { - - private static final SelectorFilter INSTANCE = new SelectorFilter(); - } - - private SelectorFilter() { - } - - @Override - public RpcResponse filter(Invoker invoker, RpcRequest req) throws Throwable { - Directory.getInstance().addServer("default","127.0.0.1:8080"); - Directory.getInstance().addServer("default","127.0.0.2:8080"); - Directory.getInstance().addServer("default","127.0.0.3:8080"); - List hosts = Directory.getInstance().getDirectory("default"); - List candidateHosts = new ArrayList<>(hosts.size()); - hosts.forEach(node -> { - Host nodeHost = Host.of(node); - nodeHost.setWorkGroup("default"); - candidateHosts.add(nodeHost); - }); - RandomSelector randomSelector = new RandomSelector(); - System.out.println(randomSelector.doSelect(candidateHosts)); - RpcResponse rsp = new RpcResponse(); - rsp.setMsg("ms"); - return rsp; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java deleted file mode 100644 index e6352aa8d5..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/HostWeight.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.rpc.filter.selector; - -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.Host; - -/** - * host weight - */ -public class HostWeight { - - private final int CPU_FACTOR = 10; - - private final int MEMORY_FACTOR = 20; - - private final int LOAD_AVERAGE_FACTOR = 70; - - private final Host host; - - private final double weight; - - private double currentWeight; - - public HostWeight(Host host, double cpu, double memory, double loadAverage) { - this.weight = getWeight(cpu, memory, loadAverage, host); - this.host = host; - this.currentWeight = weight; - } - - public double getCurrentWeight() { - return currentWeight; - } - - public double getWeight() { - return weight; - } - - public void setCurrentWeight(double currentWeight) { - this.currentWeight = currentWeight; - } - - public Host getHost() { - return host; - } - - @Override - public String toString() { - return "HostWeight{" - + "host=" + host - + ", weight=" + weight - + ", currentWeight=" + currentWeight - + '}'; - } - - private double getWeight(double cpu, double memory, double loadAverage, Host host) { - double calculateWeight = cpu * CPU_FACTOR + memory * MEMORY_FACTOR + loadAverage * LOAD_AVERAGE_FACTOR; - return getWarmUpWeight(host, calculateWeight); - } - - /** - * If the warm-up is not over, add the weight - */ - private double getWarmUpWeight(Host host, double weight) { - long startTime = host.getStartTime(); - long uptime = System.currentTimeMillis() - startTime; - if (uptime > 0 && uptime < Constants.WARM_UP_TIME) { - return weight * Constants.WARM_UP_TIME / uptime; - } - return weight; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java deleted file mode 100644 index 115ca311d0..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/LowerWeightRoundRobin.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.rpc.filter.selector; - - -import org.apache.dolphinscheduler.remote.rpc.selector.AbstractSelector; - -import java.util.Collection; - -/** - * lower weight round robin - */ -public class LowerWeightRoundRobin extends AbstractSelector { - - /** - * select - * - * @param sources sources - * @return HostWeight - */ - @Override - public HostWeight doSelect(Collection sources) { - double totalWeight = 0; - double lowWeight = 0; - HostWeight lowerNode = null; - for (HostWeight hostWeight : sources) { - totalWeight += hostWeight.getWeight(); - hostWeight.setCurrentWeight(hostWeight.getCurrentWeight() + hostWeight.getWeight()); - if (lowerNode == null || lowWeight > hostWeight.getCurrentWeight()) { - lowerNode = hostWeight; - lowWeight = hostWeight.getCurrentWeight(); - } - } - lowerNode.setCurrentWeight(lowerNode.getCurrentWeight() + totalWeight); - return lowerNode; - - } -} - - - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java deleted file mode 100644 index 5859b4c96a..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/filter/selector/RoundRobinSelector.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.remote.rpc.filter.selector; - -import org.apache.dolphinscheduler.remote.rpc.selector.AbstractSelector; -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - - -/** - * Smooth Weight Round Robin - */ -public class RoundRobinSelector extends AbstractSelector { - - private ConcurrentMap> workGroupWeightMap = new ConcurrentHashMap<>(); - - private static final int RECYCLE_PERIOD = 100000; - - private AtomicBoolean updateLock = new AtomicBoolean(); - - protected static class WeightedRoundRobin { - private int weight; - private AtomicLong current = new AtomicLong(0); - private long lastUpdate; - - int getWeight() { - return weight; - } - - void setWeight(int weight) { - this.weight = weight; - current.set(0); - } - - long increaseCurrent() { - return current.addAndGet(weight); - } - - void sel(int total) { - current.addAndGet(-1L * total); - } - - long getLastUpdate() { - return lastUpdate; - } - - void setLastUpdate(long lastUpdate) { - this.lastUpdate = lastUpdate; - } - - } - - - @Override - public Host doSelect(Collection source) { - - List hosts = new ArrayList<>(source); - String key = hosts.get(0).getWorkGroup(); - ConcurrentMap map = workGroupWeightMap.get(key); - if (map == null) { - workGroupWeightMap.putIfAbsent(key, new ConcurrentHashMap<>()); - map = workGroupWeightMap.get(key); - } - - int totalWeight = 0; - long maxCurrent = Long.MIN_VALUE; - long now = System.currentTimeMillis(); - Host selectedHost = null; - WeightedRoundRobin selectWeightRoundRobin = null; - - for (Host host : hosts) { - String workGroupHost = host.getWorkGroup() + host.getAddress(); - WeightedRoundRobin weightedRoundRobin = map.get(workGroupHost); - int weight = host.getWeight(); - if (weight < 0) { - weight = 0; - } - - if (weightedRoundRobin == null) { - weightedRoundRobin = new WeightedRoundRobin(); - // set weight - weightedRoundRobin.setWeight(weight); - map.putIfAbsent(workGroupHost, weightedRoundRobin); - weightedRoundRobin = map.get(workGroupHost); - } - if (weight != weightedRoundRobin.getWeight()) { - weightedRoundRobin.setWeight(weight); - } - - long cur = weightedRoundRobin.increaseCurrent(); - weightedRoundRobin.setLastUpdate(now); - if (cur > maxCurrent) { - maxCurrent = cur; - selectedHost = host; - selectWeightRoundRobin = weightedRoundRobin; - } - - totalWeight += weight; - } - - - if (!updateLock.get() && hosts.size() != map.size() && updateLock.compareAndSet(false, true)) { - try { - ConcurrentMap newMap = new ConcurrentHashMap<>(map); - newMap.entrySet().removeIf(item -> now - item.getValue().getLastUpdate() > RECYCLE_PERIOD); - workGroupWeightMap.put(key, newMap); - } finally { - updateLock.set(false); - } - } - - if (selectedHost != null) { - selectWeightRoundRobin.sel(totalWeight); - return selectedHost; - } - - return hosts.get(0); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java index 2e2b1a734e..60f25294bb 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.future; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java index a5f13a8343..0da43ce0dc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index d6c9387b5b..da1b3c47f6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.remote; import io.netty.channel.ChannelHandler; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java index 091a697ff3..eca355a483 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.remote; import io.netty.bootstrap.ServerBootstrap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index 0b75d37b66..e429af3ce3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.rpc.remote; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java deleted file mode 100644 index 8e8d68ae39..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/AbstractSelector.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.selector; - -import org.apache.dolphinscheduler.common.utils.CollectionUtils; - -import java.util.Collection; - -/** - * AbstractSelector - */ -public abstract class AbstractSelector implements Selector{ - @Override - public T select(Collection source) { - - if (CollectionUtils.isEmpty(source)) { - throw new IllegalArgumentException("Empty source."); - } - - /** - * if only one , return directly - */ - if (source.size() == 1) { - return (T)source.toArray()[0]; - } - return doSelect(source); - } - - protected abstract T doSelect(Collection source); - -} \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java deleted file mode 100644 index 0a8c1b3666..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/RandomSelector.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.selector; - -import org.apache.dolphinscheduler.remote.utils.Host; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -/** - * RandomSelector - */ -public class RandomSelector extends AbstractSelector { - - @Override - public Host doSelect(final Collection source) { - - List hosts = new ArrayList<>(source); - int size = hosts.size(); - int[] weights = new int[size]; - int totalWeight = 0; - int index = 0; - - for (Host host : hosts) { - totalWeight += host.getWeight(); - weights[index] = host.getWeight(); - index++; - } - - if (totalWeight > 0) { - int offset = ThreadLocalRandom.current().nextInt(totalWeight); - - for (int i = 0; i < size; i++) { - offset -= weights[i]; - if (offset < 0) { - return hosts.get(i); - } - } - } - return hosts.get(ThreadLocalRandom.current().nextInt(size)); - } - -} - diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java deleted file mode 100644 index d511294496..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/selector/Selector.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc.selector; - -import java.util.Collection; - -/** - * Selector - */ -public interface Selector { - - /** - * select - * @param source source - * @return T - */ - T select(Collection source); -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index 0277273ef5..96ed34d619 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.remote.serialize; import io.protostuff.LinkedBuffer; From 230f3a30d21219b1961d3561264192be0142bd61 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 24 Jan 2021 21:59:25 +0800 Subject: [PATCH 19/68] add rpc config --- dolphinscheduler-remote/pom.xml | 10 +++ .../dolphinscheduler/remote/rpc/MainTest.java | 7 +- .../remote/rpc/UserService.java | 6 +- .../remote/rpc/base/RpcService.java | 29 ++++++++ .../remote/rpc/client/ConsumerConfig.java | 21 ++++-- .../rpc/client/ConsumerInterceptor.java | 11 ++- .../remote/rpc/common/RequestEventType.java | 42 ++++++++++++ .../remote/rpc/common/ResponseEventType.java | 41 +++++++++++ .../remote/rpc/common/RpcRequest.java | 14 +++- .../remote/rpc/common/RpcResponse.java | 6 +- .../remote/rpc/config/ServiceBean.java | 68 +++++++++++++++++++ .../remote/rpc/remote/NettyClientHandler.java | 12 +++- .../remote/rpc/remote/NettyServerHandler.java | 12 +++- pom.xml | 1 + 14 files changed, 258 insertions(+), 22 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 27f0923017..708fb94a5b 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -69,6 +69,16 @@ junit test + + org.reflections + reflections + 0.9.11 + + + + com.google.guava + guava + diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java index 53d56f5c51..4115184f22 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java @@ -22,15 +22,18 @@ public class MainTest { Host host = new Host("127.0.0.1", 12636); IRpcClient rpcClient = new RpcClient(); - UserService userService = rpcClient.create(UserService.class, host); + IUserService userService = rpcClient.create(IUserService.class, host); boolean result = userService.say("calvin"); System.out.println("异步回掉成功" + result); System.out.println(userService.hi(10)); System.out.println(userService.hi(188888888)); - UserService user = rpcClient.create(UserService.class, host); + IUserService user = rpcClient.create(IUserService.class, host); System.out.println(user.hi(99999)); System.out.println(user.hi(998888888)); + + System.out.println(IUserService.class.getSimpleName()); + System.out.println(UserService.class.getSimpleName()); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java index a8f3c35adc..65734074fe 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java @@ -1,18 +1,22 @@ package org.apache.dolphinscheduler.remote.rpc; import org.apache.dolphinscheduler.remote.rpc.base.Rpc; +import org.apache.dolphinscheduler.remote.rpc.base.RpcService; /** * @author jiangli * @date 2021-01-11 21:05 */ -public class UserService { +@RpcService("IUserService") +public class UserService implements IUserService{ @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) + @Override public Boolean say(String s) { return true; } + @Override public String hi(int num) { return "this world has " + num + "sun"; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java new file mode 100644 index 0000000000..4f2407413b --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.base; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface RpcService { + String value() default ""; +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java index 757bb0778b..897eb688cc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java @@ -25,7 +25,9 @@ import org.apache.dolphinscheduler.remote.rpc.common.ConsumerConfigConstants; */ public class ConsumerConfig { - private Class callBackClass; + private Class serviceCallBackClass; + + private Class ackCallBackClass; private String serviceName; @@ -33,13 +35,20 @@ public class ConsumerConfig { private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; - public Class getCallBackClass() { - return callBackClass; + public Class getServiceCallBackClass() { + return serviceCallBackClass; } - //set call back class - void setCallBackClass(Class callBackClass) { - this.callBackClass = callBackClass; + public void setServiceCallBackClass(Class serviceCallBackClass) { + this.serviceCallBackClass = serviceCallBackClass; + } + + public Class getAckCallBackClass() { + return ackCallBackClass; + } + + public void setAckCallBackClass(Class ackCallBackClass) { + this.ackCallBackClass = ackCallBackClass; } public String getServiceName() { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java index 79bf01718a..2b90c429b3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.remote.rpc.client; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.rpc.Invoker; import org.apache.dolphinscheduler.remote.rpc.base.Rpc; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; @@ -37,9 +36,6 @@ import net.bytebuddy.implementation.bind.annotation.RuntimeType; */ public class ConsumerInterceptor { - private Invoker invoker; - - private Host host; private NettyClient nettyClient = NettyClient.getInstance(); @@ -52,7 +48,7 @@ public class ConsumerInterceptor { public Object intercept(@AllArguments Object[] args, @Origin Method method) throws RemotingException { RpcRequest request = buildReq(args, method); - String serviceName = method.getDeclaringClass().getName() + method.getName(); + String serviceName = method.getDeclaringClass().getSimpleName() + method.getName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (null == consumerConfig) { consumerConfig = cacheServiceConfig(method, serviceName); @@ -76,7 +72,7 @@ public class ConsumerInterceptor { private RpcRequest buildReq(Object[] args, Method method) { RpcRequest request = new RpcRequest(); request.setRequestId(UUID.randomUUID().toString()); - request.setClassName(method.getDeclaringClass().getName()); + request.setClassName(method.getDeclaringClass().getSimpleName()); request.setMethodName(method.getName()); request.setParameterTypes(method.getParameterTypes()); @@ -94,7 +90,8 @@ public class ConsumerInterceptor { if (annotationPresent) { Rpc rpc = method.getAnnotation(Rpc.class); consumerConfig.setAsync(rpc.async()); - consumerConfig.setCallBackClass(rpc.callback()); + consumerConfig.setServiceCallBackClass(rpc.serviceCallback()); + consumerConfig.setAckCallBackClass(rpc.ackCallback()); consumerConfig.setRetries(rpc.retries()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java new file mode 100644 index 0000000000..c2f8a8f1dd --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.common; + +public enum RequestEventType { + + HEARTBEAT((byte)1,"heartbeat"), + BUSINESS((byte)2,"business request"); + + + private Byte type; + + private String description; + + RequestEventType(Byte type, String description) { + this.type = type; + this.description = description; + } + + public Byte getType() { + return type; + } + + public String getDescription() { + return description; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java new file mode 100644 index 0000000000..4c0d72181c --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java @@ -0,0 +1,41 @@ +package org.apache.dolphinscheduler.remote.rpc.common;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public enum ResponseEventType { + + ACK((byte) 1, "ack"), + BUSINESS_RSP((byte) 2, "business response"); + + private Byte type; + + private String description; + + ResponseEventType(Byte type, String description) { + this.type = type; + this.description = description; + } + + + public Byte getType() { + return type; + } + + public String getDescription() { + return description; + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java index 7d2480f642..3c8732c61f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java @@ -27,9 +27,13 @@ public class RpcRequest { private String methodName; private Class[] parameterTypes; private Object[] parameters; - // 0 hear beat,1 businness msg + /** + * @see RequestEventType + */ private Byte eventType = 1; + private Boolean ack; + public Byte getEventType() { return eventType; } @@ -77,4 +81,12 @@ public class RpcRequest { public void setParameters(Object[] parameters) { this.parameters = parameters; } + + public Boolean getAck() { + return ack; + } + + public void setAck(Boolean ack) { + this.ack = ack; + } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java index 0a8c5ef799..9e6db6ec11 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java @@ -26,7 +26,11 @@ public class RpcResponse { private String msg; private Object result; private Byte status; - private Integer responseType; + + /** + * @see ResponseEventType + */ + private Byte responseType; public String getRequestId() { return requestId; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java new file mode 100644 index 0000000000..a0fe93c5be --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.remote.rpc.config; + +import org.apache.dolphinscheduler.remote.rpc.IUserService; +import org.apache.dolphinscheduler.remote.rpc.base.RpcService; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicBoolean; + + +import org.reflections.Reflections; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class ServiceBean { + + private static final Logger logger = LoggerFactory.getLogger(ServiceBean.class); + + private static Map serviceMap = new HashMap<>(); + + private static AtomicBoolean initialized = new AtomicBoolean(false); + + private static synchronized void init() { + Reflections f = new Reflections("org/apache/dolphinscheduler/remote/rpc"); + + + List> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class)); + list.forEach(rpcClass -> { + RpcService rpcService = rpcClass.getAnnotation(RpcService.class); + serviceMap.put(rpcService.value(), rpcClass); + }); + } + + public static void main(String[] args) { + init(); + } + + public static Class getServiceClass(String className) { + if (initialized.get()) { + return (Class) serviceMap.get(className); + } else { + init(); + } + return (Class) serviceMap.get(className); + } + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java index da1b3c47f6..97d4bb0444 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; +import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; import org.slf4j.Logger; @@ -75,7 +76,12 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { new FastThreadLocalThread(() -> { try { if (rsp.getStatus() == 0) { - consumerConfig.getCallBackClass().newInstance().run(rsp.getResult()); + try { + consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); + } catch (InvocationTargetException | NoSuchMethodException e) { + logger.error("rpc call back error",e); + } + } else { logger.error("xxxx fail"); } @@ -108,4 +114,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { logger.error("exceptionCaught : {}", cause.getMessage(), cause); ctx.channel().close(); } + + private void executeAsyncHandler(){ + + } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index e429af3ce3..f5ab779928 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -17,10 +17,16 @@ package org.apache.dolphinscheduler.remote.rpc.remote; +import org.apache.dolphinscheduler.remote.rpc.IUserService; +import org.apache.dolphinscheduler.remote.rpc.base.RpcService; import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.remote.rpc.config.ServiceBean; import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,11 +68,12 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { RpcResponse response = new RpcResponse(); if (req.getEventType() == 0) { - logger.info("接受心跳消息!..."); + logger.info("accept heartbeat msg"); return; } response.setRequestId(req.getRequestId()); + response.setStatus((byte) 0); @@ -79,8 +86,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { Object[] arguments = req.getParameters(); Object result = null; try { - - Class serviceClass = Class.forName(classname); + Class serviceClass = ServiceBean.getServiceClass(classname); Object object = serviceClass.newInstance(); diff --git a/pom.xml b/pom.xml index 63776bdb03..af4210dce4 100644 --- a/pom.xml +++ b/pom.xml @@ -1101,5 +1101,6 @@ dolphinscheduler-service dolphinscheduler-spi dolphinscheduler-microbench + dolphinscheduler-remote-connfig From a2d245ddc1cb3eabb6fa51eab407ddfae5c6cb8f Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 24 Jan 2021 22:04:27 +0800 Subject: [PATCH 20/68] add rpc config --- .../dolphinscheduler/remote/rpc/remote/NettyServerHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java index f5ab779928..9556ec1891 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java @@ -71,6 +71,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { logger.info("accept heartbeat msg"); return; } + //todo 使用业务线程池去处理 不要占用netty的资源 response.setRequestId(req.getRequestId()); From 9f58eb7ea55b149da808d22598493684cb5cc5cf Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 21:40:12 +0800 Subject: [PATCH 21/68] add rpc config --- .../remote/config/NettyServerConfig.java | 2 +- .../remote/decoder/NettyEncoder.java | 1 - .../dolphinscheduler/remote/rpc/Invoker.java | 12 --- .../remote/rpc/MainServerTest.java | 15 ---- .../dolphinscheduler/remote/rpc/MainTest.java | 39 --------- .../remote/rpc/UserCallback.java | 15 ---- .../remote/rpc/UserService.java | 23 ------ .../{remote => }/rpc/base/Rpc.java | 4 +- .../{remote => }/rpc/base/RpcService.java | 2 +- .../rpc/client/ConsumerConfig.java | 10 +-- .../rpc/client/ConsumerConfigCache.java | 2 +- .../rpc/client/ConsumerInterceptor.java | 10 +-- .../{remote => }/rpc/client/IRpcClient.java | 2 +- .../{remote => }/rpc/client/RpcClient.java | 14 +--- .../rpc/client/RpcRequestCache.java | 4 +- .../rpc/client/RpcRequestTable.java | 2 +- .../rpc/common/AbstractRpcCallBack.java | 2 +- .../rpc/common/ConsumerConfigConstants.java | 2 +- .../rpc/common/RequestEventType.java | 2 +- .../rpc/common/ResponseEventType.java | 3 +- .../{remote => }/rpc/common/RpcRequest.java | 2 +- .../{remote => }/rpc/common/RpcResponse.java | 2 +- .../rpc/common/ThreadPoolManager.java | 45 +++++++++++ .../{remote => }/rpc/config/ServiceBean.java | 19 ++--- .../{remote => }/rpc/future/RpcFuture.java | 4 +- .../{remote => }/rpc/remote/NettyClient.java | 63 +++++++-------- .../rpc/remote/NettyClientHandler.java | 51 ++++++------ .../{remote => }/rpc/remote/NettyServer.java | 80 +++++++++---------- .../rpc/remote/NettyServerHandler.java | 29 +++---- .../apache/dolphinscheduler/rpc/RpcTest.java | 63 +++++++++++++++ .../dolphinscheduler/rpc/UserCallback.java | 30 +++++++ .../dolphinscheduler/rpc/UserService.java | 20 +++++ pom.xml | 1 + 33 files changed, 301 insertions(+), 274 deletions(-) delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/base/Rpc.java (92%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/base/RpcService.java (95%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/ConsumerConfig.java (83%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/ConsumerConfigCache.java (95%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/ConsumerInterceptor.java (92%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/IRpcClient.java (94%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/RpcClient.java (77%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/RpcRequestCache.java (91%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/client/RpcRequestTable.java (96%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/AbstractRpcCallBack.java (95%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/ConsumerConfigConstants.java (94%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/RequestEventType.java (95%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/ResponseEventType.java (95%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/RpcRequest.java (97%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/common/RpcResponse.java (96%) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/config/ServiceBean.java (86%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/future/RpcFuture.java (94%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/remote/NettyClient.java (76%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/remote/NettyClientHandler.java (74%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/remote/NettyServer.java (79%) rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote => }/rpc/remote/NettyServerHandler.java (84%) create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java index 7bac0361f4..4ec8a0f7a7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/config/NettyServerConfig.java @@ -56,7 +56,7 @@ public class NettyServerConfig { /** * listen port */ - private int listenPort = 12636; + private int listenPort = 12346; public int getListenPort() { return listenPort; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java index c381b3fdc4..b732a50680 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java @@ -4,7 +4,6 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.MessageToByteEncoder; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java deleted file mode 100644 index e93dfa4e66..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/Invoker.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; - -/** - * Invoker - */ -public interface Invoker { - - RpcResponse invoke(RpcRequest req) throws Throwable; -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java deleted file mode 100644 index 88e0f615f1..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainServerTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; - -/** - * @author jiangli - * @date 2021-01-20 14:54 - */ -public class MainServerTest { - - public static void main(String[] args) { - NettyServer nettyServer = new NettyServer(new NettyServerConfig()); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java deleted file mode 100644 index 4115184f22..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/MainTest.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.rpc.client.IRpcClient; -import org.apache.dolphinscheduler.remote.rpc.client.RpcClient; -import org.apache.dolphinscheduler.remote.rpc.remote.NettyServer; -import org.apache.dolphinscheduler.remote.utils.Host; - -/** - * @author jiangli - * @date 2021-01-11 21:06 - */ -public class MainTest { - - public static void main(String[] args) throws Exception { - - - // NettyServer nettyServer = new NettyServer(new NettyServerConfig()); - - // NettyClient nettyClient=new NettyClient(new NettyClientConfig()); - - Host host = new Host("127.0.0.1", 12636); - - IRpcClient rpcClient = new RpcClient(); - IUserService userService = rpcClient.create(IUserService.class, host); - boolean result = userService.say("calvin"); - System.out.println("异步回掉成功" + result); - - System.out.println(userService.hi(10)); - System.out.println(userService.hi(188888888)); - - IUserService user = rpcClient.create(IUserService.class, host); - System.out.println(user.hi(99999)); - System.out.println(user.hi(998888888)); - - System.out.println(IUserService.class.getSimpleName()); - System.out.println(UserService.class.getSimpleName()); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java deleted file mode 100644 index fe712dfbe3..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserCallback.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; - -/** - * @author jiangli - * @date 2021-01-15 07:32 - */ -public class UserCallback extends AbstractRpcCallBack { - @Override - public void run(Object object) { - Boolean msg= (Boolean) object; - System.out.println("我是异步回调handle Kris"+msg); - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java deleted file mode 100644 index 65734074fe..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/UserService.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.apache.dolphinscheduler.remote.rpc; - -import org.apache.dolphinscheduler.remote.rpc.base.Rpc; -import org.apache.dolphinscheduler.remote.rpc.base.RpcService; - -/** - * @author jiangli - * @date 2021-01-11 21:05 - */ -@RpcService("IUserService") -public class UserService implements IUserService{ - - @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) - @Override - public Boolean say(String s) { - return true; - } - - @Override - public String hi(int num) { - return "this world has " + num + "sun"; - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java similarity index 92% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java index 7ccee632b9..1fc6ca3627 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.base; +package org.apache.dolphinscheduler.rpc.base; -import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; +import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java index 4f2407413b..5ebb2cfe7a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/base/RpcService.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/RpcService.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.base; +package org.apache.dolphinscheduler.rpc.base; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java similarity index 83% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java index 897eb688cc..10d6cd5c4b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java @@ -15,10 +15,10 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; -import org.apache.dolphinscheduler.remote.rpc.common.AbstractRpcCallBack; -import org.apache.dolphinscheduler.remote.rpc.common.ConsumerConfigConstants; +import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; +import org.apache.dolphinscheduler.rpc.common.ConsumerConfigConstants; /** * We will cache the consumer configuration, when the rpc call is generated, the consumer configuration will be first obtained from here @@ -39,7 +39,7 @@ public class ConsumerConfig { return serviceCallBackClass; } - public void setServiceCallBackClass(Class serviceCallBackClass) { + void setServiceCallBackClass(Class serviceCallBackClass) { this.serviceCallBackClass = serviceCallBackClass; } @@ -47,7 +47,7 @@ public class ConsumerConfig { return ackCallBackClass; } - public void setAckCallBackClass(Class ackCallBackClass) { + void setAckCallBackClass(Class ackCallBackClass) { this.ackCallBackClass = ackCallBackClass; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java index f8d7d570b0..a407079fc5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerConfigCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java similarity index 92% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 2b90c429b3..f976d0b1d6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -15,14 +15,14 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; -import org.apache.dolphinscheduler.remote.rpc.base.Rpc; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.remote.NettyClient; import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.rpc.base.Rpc; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.remote.NettyClient; import java.lang.reflect.Method; import java.util.UUID; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java similarity index 94% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java index ac7574ed1a..979a979e88 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/IRpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; import org.apache.dolphinscheduler.remote.utils.Host; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java similarity index 77% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java index b8d1202433..02f99b4f62 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcClient.java @@ -15,14 +15,12 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; import static net.bytebuddy.matcher.ElementMatchers.isDeclaredBy; import org.apache.dolphinscheduler.remote.utils.Host; -import java.util.concurrent.ConcurrentHashMap; - import net.bytebuddy.ByteBuddy; import net.bytebuddy.implementation.MethodDelegation; @@ -31,22 +29,14 @@ import net.bytebuddy.implementation.MethodDelegation; */ public class RpcClient implements IRpcClient { - private ConcurrentHashMap classMap = new ConcurrentHashMap<>(); - @Override public T create(Class clazz, Host host) throws Exception { - // if(!classMap.containsKey(clazz.getName())){ - T proxy = new ByteBuddy() + return new ByteBuddy() .subclass(clazz) .method(isDeclaredBy(clazz)).intercept(MethodDelegation.to(new ConsumerInterceptor(host))) .make() .load(getClass().getClassLoader()) .getLoaded() .getDeclaredConstructor().newInstance(); - - // classMap.putIfAbsent(clazz.getName(),proxy); - return proxy; - // } - // return (T) classMap.get(clazz.getName()); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java similarity index 91% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java index 153ceba293..14864d6c8b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; -import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.rpc.future.RpcFuture; /** * RpcRequestCache diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java index ae31252d9a..f3cdff4f77 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.client; +package org.apache.dolphinscheduler.rpc.client; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java index 758b8474bb..2b106a96e4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/AbstractRpcCallBack.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/AbstractRpcCallBack.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.common; +package org.apache.dolphinscheduler.rpc.common; /** * AbstractRpcCallBack diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java similarity index 94% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java index d23307aedf..478c10432c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ConsumerConfigConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.common; +package org.apache.dolphinscheduler.rpc.common; /** * ConsumerConfigConstants diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java index c2f8a8f1dd..558c88067b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RequestEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RequestEventType.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.common; +package org.apache.dolphinscheduler.rpc.common; public enum RequestEventType { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java similarity index 95% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java index 4c0d72181c..66ca157e23 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/ResponseEventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ResponseEventType.java @@ -1,4 +1,4 @@ -package org.apache.dolphinscheduler.remote.rpc.common;/* +package org.apache.dolphinscheduler.rpc.common;/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -29,7 +29,6 @@ public enum ResponseEventType { this.description = description; } - public Byte getType() { return type; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java similarity index 97% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java index 3c8732c61f..a22c8f8a93 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.common; +package org.apache.dolphinscheduler.rpc.common; /** * RpcRequest diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java similarity index 96% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java index 9e6db6ec11..6aafc039aa 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/common/RpcResponse.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcResponse.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.common; +package org.apache.dolphinscheduler.rpc.common; /** * RpcResponse diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java new file mode 100644 index 0000000000..cb4c33a23a --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.common; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor.DiscardPolicy; +import java.util.concurrent.TimeUnit; + +public enum ThreadPoolManager { + + INSTANCE; + + ExecutorService executorService; + + ThreadPoolManager() { + int SIZE_WORK_QUEUE = 200; + long KEEP_ALIVE_TIME = 60; + int CORE_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; + int MAXI_MUM_POOL_SIZE = CORE_POOL_SIZE * 4; + executorService = new ThreadPoolExecutor(CORE_POOL_SIZE, MAXI_MUM_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(SIZE_WORK_QUEUE), + new DiscardPolicy()); + } + + public void addExecuteTask(Runnable task) { + executorService.submit(task); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java similarity index 86% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index a0fe93c5be..f51f35b3e6 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -15,16 +15,14 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.config; +package org.apache.dolphinscheduler.rpc.config; -import org.apache.dolphinscheduler.remote.rpc.IUserService; -import org.apache.dolphinscheduler.remote.rpc.base.RpcService; +import org.apache.dolphinscheduler.rpc.base.RpcService; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.ServiceLoader; import java.util.concurrent.atomic.AtomicBoolean; @@ -32,7 +30,9 @@ import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +/** + * ServiceBean find all rpcService + */ public class ServiceBean { private static final Logger logger = LoggerFactory.getLogger(ServiceBean.class); @@ -42,9 +42,8 @@ public class ServiceBean { private static AtomicBoolean initialized = new AtomicBoolean(false); private static synchronized void init() { - Reflections f = new Reflections("org/apache/dolphinscheduler/remote/rpc"); - - + // todo config + Reflections f = new Reflections("org/apache/dolphinscheduler/rpc"); List> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class)); list.forEach(rpcClass -> { RpcService rpcService = rpcClass.getAnnotation(RpcService.class); @@ -52,10 +51,6 @@ public class ServiceBean { }); } - public static void main(String[] args) { - init(); - } - public static Class getServiceClass(String className) { if (initialized.get()) { return (Class) serviceMap.get(className); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java similarity index 94% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 60f25294bb..8cf1dfaa22 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.future; +package org.apache.dolphinscheduler.rpc.future; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java similarity index 76% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 0da43ce0dc..ed7aaa05ef 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.remote; +package org.apache.dolphinscheduler.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; -import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; -import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; +import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.future.RpcFuture; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NettyUtils; @@ -98,7 +98,7 @@ public class NettyClient { /** * get channel */ - public Channel getChannel(Host host) { + private Channel getChannel(Host host) { Channel channel = channels.get(host); if (channel != null && channel.isActive()) { return channel; @@ -138,7 +138,7 @@ public class NettyClient { * * @param clientConfig client config */ - public NettyClient(final NettyClientConfig clientConfig) { + private NettyClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; if (NettyUtils.useEpoll()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), new ThreadFactory() { @@ -169,24 +169,24 @@ public class NettyClient { private void start() { this.bootstrap - .group(this.workerGroup) - .channel(NettyUtils.getSocketChannelClass()) - .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) - .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) - .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) - .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) - .handler(new LoggingHandler(LogLevel.DEBUG)) - .handler(new ChannelInitializer() { - @Override - public void initChannel(SocketChannel ch) { - ch.pipeline() - .addLast(new NettyEncoder(RpcRequest.class)) - .addLast(new NettyDecoder(RpcResponse.class)) - .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) - .addLast(new NettyClientHandler()); - } - }); + .group(this.workerGroup) + .channel(NettyUtils.getSocketChannelClass()) + .option(ChannelOption.SO_KEEPALIVE, clientConfig.isSoKeepalive()) + .option(ChannelOption.TCP_NODELAY, clientConfig.isTcpNoDelay()) + .option(ChannelOption.SO_SNDBUF, clientConfig.getSendBufferSize()) + .option(ChannelOption.SO_RCVBUF, clientConfig.getReceiveBufferSize()) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, clientConfig.getConnectTimeoutMillis()) + .handler(new LoggingHandler(LogLevel.DEBUG)) + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) { + ch.pipeline() + .addLast(new NettyEncoder(RpcRequest.class)) + .addLast(new NettyDecoder(RpcResponse.class)) + .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) + .addLast(new NettyClientHandler()); + } + }); isStarted.compareAndSet(false, true); } @@ -196,9 +196,8 @@ public class NettyClient { Channel channel = getChannel(host); assert channel != null; RpcRequestCache rpcRequestCache = new RpcRequestCache(); - rpcRequestCache.setServiceName(request.getClassName() + request.getMethodName()); - - + String serviceName = request.getClassName() + request.getMethodName(); + rpcRequestCache.setServiceName(serviceName); RpcFuture future = null; if (!async) { future = new RpcFuture(); @@ -209,15 +208,15 @@ public class NettyClient { RpcResponse result = null; if (async) { - result=new RpcResponse(); - result.setStatus((byte)0); + result = new RpcResponse(); + result.setStatus((byte) 0); result.setResult(true); return result; } try { result = future.get(); } catch (InterruptedException | ExecutionException e) { - logger.error("send msg error",e); + logger.error("send msg error，service name is {}", serviceName, e); } return result; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java similarity index 74% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index 97d4bb0444..ec4f97255a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -15,21 +15,16 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.remote; +package org.apache.dolphinscheduler.rpc.remote; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelInboundHandlerAdapter; -import io.netty.handler.timeout.IdleStateEvent; -import io.netty.util.concurrent.FastThreadLocalThread; - -import org.apache.dolphinscheduler.remote.rpc.client.ConsumerConfig; -import org.apache.dolphinscheduler.remote.rpc.client.ConsumerConfigCache; -import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestCache; -import org.apache.dolphinscheduler.remote.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.rpc.client.ConsumerConfig; +import org.apache.dolphinscheduler.rpc.client.ConsumerConfigCache; +import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; +import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; +import org.apache.dolphinscheduler.rpc.future.RpcFuture; import java.lang.reflect.InvocationTargetException; import java.net.InetSocketAddress; @@ -37,6 +32,12 @@ import java.net.InetSocketAddress; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.util.concurrent.FastThreadLocalThread; + /** * NettyClientHandler */ @@ -46,6 +47,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); + private final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; + @Override public void channelInactive(ChannelHandlerContext ctx) { InetSocketAddress address = (InetSocketAddress) ctx.channel().remoteAddress(); @@ -58,10 +61,13 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { RpcRequestCache rpcRequest = RpcRequestTable.get(rsp.getRequestId()); if (null == rpcRequest) { - logger.warn("未知响应"); + logger.warn("rpc read error,this request does not exist"); return; } + threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest)); + } + private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest) { String serviceName = rpcRequest.getServiceName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (!consumerConfig.getAsync()) { @@ -71,7 +77,6 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { return; } - //async new FastThreadLocalThread(() -> { try { @@ -79,18 +84,16 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { try { consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); } catch (InvocationTargetException | NoSuchMethodException e) { - logger.error("rpc call back error",e); + logger.error("rpc call back error, serviceName {} ", serviceName, e); } } else { - logger.error("xxxx fail"); + logger.error("rpc response error ,serviceName {}", serviceName); } } catch (InstantiationException | IllegalAccessException e) { - logger.error("execute async error", e); + logger.error("execute async error,serviceName {}", serviceName, e); } }).start(); - - } @Override @@ -99,7 +102,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { if (evt instanceof IdleStateEvent) { IdleStateEvent event = (IdleStateEvent) evt; RpcRequest request = new RpcRequest(); - request.setEventType((byte)0); + request.setEventType((byte) 0); ctx.channel().writeAndFlush(request); logger.debug("send heart beat msg..."); @@ -110,12 +113,8 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - System.out.println("exceptionCaught"); logger.error("exceptionCaught : {}", cause.getMessage(), cause); ctx.channel().close(); } - private void executeAsyncHandler(){ - - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java similarity index 79% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index eca355a483..cbba950c23 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -15,7 +15,23 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.remote; +package org.apache.dolphinscheduler.rpc.remote; + +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; +import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; +import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.remote.utils.NettyUtils; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; @@ -29,30 +45,12 @@ import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import io.netty.handler.timeout.IdleStateHandler; -import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; - -import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; - -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.utils.Constants; -import org.apache.dolphinscheduler.remote.utils.NettyUtils; - -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * NettyServer */ public class NettyServer { - private static final Logger logger =LoggerFactory.getLogger(NettyServer.class); + private static final Logger logger = LoggerFactory.getLogger(NettyServer.class); /** * boss group @@ -126,29 +124,28 @@ public class NettyServer { this.start(); } - /** * server start */ public void start() { if (isStarted.compareAndSet(false, true)) { this.serverBootstrap - .group(this.bossGroup, this.workGroup) - .channel(NettyUtils.getServerSocketChannelClass()) - .option(ChannelOption.SO_REUSEADDR, true) - .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) - .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) - .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) - .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) - .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) - .handler(new LoggingHandler(LogLevel.DEBUG)) - .childHandler(new ChannelInitializer() { + .group(this.bossGroup, this.workGroup) + .channel(NettyUtils.getServerSocketChannelClass()) + .option(ChannelOption.SO_REUSEADDR, true) + .option(ChannelOption.SO_BACKLOG, serverConfig.getSoBacklog()) + .childOption(ChannelOption.SO_KEEPALIVE, serverConfig.isSoKeepalive()) + .childOption(ChannelOption.TCP_NODELAY, serverConfig.isTcpNoDelay()) + .childOption(ChannelOption.SO_SNDBUF, serverConfig.getSendBufferSize()) + .childOption(ChannelOption.SO_RCVBUF, serverConfig.getReceiveBufferSize()) + .handler(new LoggingHandler(LogLevel.DEBUG)) + .childHandler(new ChannelInitializer() { - @Override - protected void initChannel(SocketChannel ch){ - initNettyChannel(ch); - } - }); + @Override + protected void initChannel(SocketChannel ch) { + initNettyChannel(ch); + } + }); ChannelFuture future; try { @@ -165,7 +162,6 @@ public class NettyServer { throw new RuntimeException(String.format("NettyRemotingServer bind %s fail", serverConfig.getListenPort())); } } - System.out.println("netty ser ver start"); } /** @@ -175,13 +171,12 @@ public class NettyServer { */ private void initNettyChannel(SocketChannel ch) { ch.pipeline() - .addLast(new NettyDecoder(RpcRequest.class)) - .addLast(new NettyEncoder(RpcResponse.class)) - .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) - .addLast("handler", new NettyServerHandler()); + .addLast(new NettyDecoder(RpcRequest.class)) + .addLast(new NettyEncoder(RpcResponse.class)) + .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) + .addLast("handler", new NettyServerHandler()); } - public void close() { if (isStarted.compareAndSet(true, false)) { try { @@ -199,5 +194,4 @@ public class NettyServer { } } - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java similarity index 84% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java index 9556ec1891..ae4ccaab6e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java @@ -15,18 +15,14 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.rpc.remote; +package org.apache.dolphinscheduler.rpc.remote; -import org.apache.dolphinscheduler.remote.rpc.IUserService; -import org.apache.dolphinscheduler.remote.rpc.base.RpcService; -import org.apache.dolphinscheduler.remote.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.remote.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.remote.rpc.config.ServiceBean; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; +import org.apache.dolphinscheduler.rpc.config.ServiceBean; import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; -import java.util.ServiceLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,19 +38,19 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); + private final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; + @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { super.channelRegistered(ctx); } - @Override public void channelInactive(ChannelHandlerContext ctx) { logger.info("channel close"); ctx.channel().close(); } - @Override public void channelActive(ChannelHandlerContext ctx) { logger.info("client connect success !" + ctx.channel().remoteAddress()); @@ -65,19 +61,20 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { RpcRequest req = (RpcRequest) msg; - RpcResponse response = new RpcResponse(); if (req.getEventType() == 0) { logger.info("accept heartbeat msg"); return; } - //todo 使用业务线程池去处理 不要占用netty的资源 + threadPoolManager.addExecuteTask(() -> readHandler(ctx, req)); + } + + private void readHandler(ChannelHandlerContext ctx, RpcRequest req) { + RpcResponse response = new RpcResponse(); response.setRequestId(req.getRequestId()); - response.setStatus((byte) 0); - String classname = req.getClassName(); String methodName = req.getMethodName(); @@ -95,7 +92,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { result = method.invoke(object, arguments); } catch (Exception e) { - logger.error("netty server execute error", e); + logger.error("netty server execute error,service name {}", classname + methodName, e); response.setStatus((byte) -1); } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java new file mode 100644 index 0000000000..5a10de8d73 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc; + +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.rpc.client.IRpcClient; +import org.apache.dolphinscheduler.rpc.client.RpcClient; +import org.apache.dolphinscheduler.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.rpc.remote.NettyServer; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class RpcTest { + private NettyServer nettyServer; + + private IUserService userService; + + private Host host; + + @Before + public void before() throws Exception { + nettyServer = new NettyServer(new NettyServerConfig()); + IRpcClient rpcClient = new RpcClient(); + host = new Host("127.0.0.1", 12346); + userService = rpcClient.create(IUserService.class, host); + + } + + @Test + public void sendTest() { + Integer result = userService.hi(3); + Assert.assertSame(4, result); + result = userService.hi(4); + Assert.assertSame(5, result); + userService.say("sync"); + } + + @After + public void after() { + NettyClient.getInstance().close(); + nettyServer.close(); + } + +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java new file mode 100644 index 0000000000..882cace115 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc; + +import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; + +/** + * UserCallback + */ +public class UserCallback extends AbstractRpcCallBack { + @Override + public void run(Object object) { + + } +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java new file mode 100644 index 0000000000..bd0919ebb9 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -0,0 +1,20 @@ +package org.apache.dolphinscheduler.rpc; + +import org.apache.dolphinscheduler.rpc.base.RpcService; + +/** + * UserService + */ +@RpcService("IUserService") +public class UserService implements IUserService{ + + @Override + public Boolean say(String s) { + return true; + } + + @Override + public Integer hi(int num) { + return ++num; + } +} diff --git a/pom.xml b/pom.xml index af4210dce4..7ab8117f6d 100644 --- a/pom.xml +++ b/pom.xml @@ -868,6 +868,7 @@ **/dao/entity/UdfFuncTest.java **/remote/JsonSerializerTest.java **/remote/RemoveTaskLogResponseCommandTest.java + **/rpc/RpcTest.java **/remote/RemoveTaskLogRequestCommandTest.java **/remote/NettyRemotingClientTest.java **/remote/NettyUtilTest.java From a97f78188469e836c7196733f5d0ba246b644746 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 22:08:00 +0800 Subject: [PATCH 22/68] add license --- dolphinscheduler-dist/release-docs/LICENSE | 10 +- .../licenses/LICENSE-protostuff-core.txt | 202 ++++++++++++++++++ .../licenses/LICENSE-protostuff-runtime.txt | 202 ++++++++++++++++++ .../licenses/LICENSE-reflections.txt | 13 ++ dolphinscheduler-remote/pom.xml | 17 +- .../remote/decoder/NettyEncoder.java | 31 --- .../decoder => rpc/codec}/NettyDecoder.java | 19 +- .../rpc/codec/NettyEncoder.java | 48 +++++ .../rpc/remote/NettyClient.java | 4 +- .../rpc/remote/NettyServer.java | 4 +- pom.xml | 22 ++ tools/dependencies/known-dependencies.txt | 3 + 12 files changed, 529 insertions(+), 46 deletions(-) create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote/decoder => rpc/codec}/NettyDecoder.java (57%) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 44892a242e..ac074ce4a2 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -506,9 +506,17 @@ Apache 2.0 licenses ======================================== echarts 4.1.0: https://github.com/apache/incubator-echarts Apache-2.0 remixicon 2.5.0 https://github.com/Remix-Design/remixicon Apache-2.0 + protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 + protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 ======================================== BSD licenses ======================================== - d3 3.5.17: https://github.com/d3/d3 BSD-3-Clause \ No newline at end of file + d3 3.5.17: https://github.com/d3/d3 BSD-3-Clause + + +======================================== +WTFPL License +======================================== + reflections 0.9.12: https://github.com/ronmamo/reflections WTFPL \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt new file mode 100644 index 0000000000..c3155d2478 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-reflections.txt @@ -0,0 +1,13 @@ + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index 708fb94a5b..72d2dae6f1 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -51,29 +51,28 @@ net.bytebuddy byte-buddy + - io.protostuff protostuff-core - 1.7.2 - + io.protostuff protostuff-runtime - 1.7.2 + + + org.reflections + reflections + + junit junit test - - org.reflections - reflections - 0.9.11 - com.google.guava diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java deleted file mode 100644 index b732a50680..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyEncoder.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.apache.dolphinscheduler.remote.decoder; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; - -import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; - -/** - * @author jiangli - * @date 2021-01-12 18:52 - */ -public class NettyEncoder extends MessageToByteEncoder { - - - private Class genericClass; - - public NettyEncoder(Class genericClass) { - this.genericClass = genericClass; - } - - @Override - protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { - if (genericClass.isInstance(o)) { - byte[] data = ProtoStuffUtils.serialize(o); - byteBuf.writeInt(data.length); - byteBuf.writeBytes(data); - } - - } -} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java similarity index 57% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 160e5f50ff..70af889338 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/decoder/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -1,4 +1,21 @@ -package org.apache.dolphinscheduler.remote.decoder; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.codec; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java new file mode 100644 index 0000000000..ae9f1c9722 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.codec; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; + +import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; + +/** + * @author jiangli + * @date 2021-01-12 18:52 + */ +public class NettyEncoder extends MessageToByteEncoder { + + + private Class genericClass; + + public NettyEncoder(Class genericClass) { + this.genericClass = genericClass; + } + + @Override + protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { + if (genericClass.isInstance(o)) { + byte[] data = ProtoStuffUtils.serialize(o); + byteBuf.writeInt(data.length); + byteBuf.writeBytes(data); + } + + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index ed7aaa05ef..5ed44fb7d7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; -import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; +import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; +import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; import org.apache.dolphinscheduler.rpc.common.RpcRequest; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index cbba950c23..52b6b4e422 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -18,8 +18,8 @@ package org.apache.dolphinscheduler.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.remote.decoder.NettyDecoder; -import org.apache.dolphinscheduler.remote.decoder.NettyEncoder; +import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; +import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.NettyUtils; import org.apache.dolphinscheduler.rpc.common.RpcRequest; diff --git a/pom.xml b/pom.xml index 7ab8117f6d..7369ce0aa5 100644 --- a/pom.xml +++ b/pom.xml @@ -120,6 +120,8 @@ 2.0.0 0.184 ${dep.airlift.version} + 1.7.2 + 0.9.12 @@ -314,6 +316,26 @@ ${jackson.version} + + + + io.protostuff + protostuff-core + ${protostuff.version} + + + + io.protostuff + protostuff-runtime + ${protostuff.version} + + + + org.reflections + reflections + ${reflections.version} + + junit junit diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 188db804f1..67a12e05fb 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -162,8 +162,11 @@ poi-3.17.jar postgresql-42.1.4.jar presto-jdbc-0.238.1.jar protobuf-java-2.5.0.jar +protostuff-core-1.7.2.jar +protostuff-runtime-1.7.2.jar quartz-2.3.0.jar quartz-jobs-2.3.0.jar +reflections-0.9.11.jar slf4j-api-1.7.5.jar snakeyaml-1.23.jar snappy-0.2.jar From 4325410abc3d8913d4d513d937f79150aa751518 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 22:12:08 +0800 Subject: [PATCH 23/68] add license --- .../dolphinscheduler/rpc/UserService.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java index bd0919ebb9..025735c1d3 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.dolphinscheduler.rpc; import org.apache.dolphinscheduler.rpc.base.RpcService; From 958d2fd66dce4d670ab02cd45a2ec9be4b51e1c7 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 22:25:36 +0800 Subject: [PATCH 24/68] add license --- dolphinscheduler-dist/release-docs/LICENSE | 2 +- pom.xml | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index ac074ce4a2..3dc7d36b8e 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -224,7 +224,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. avro 1.7.4: https://github.com/apache/avro, Apache 2.0 aws-sdk-java 1.7.4: https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk/1.7.4, Apache 2.0 bonecp 0.8.0.RELEASE: https://github.com/wwadge/bonecp, Apache 2.0 - byte-buddy 1.9.10: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.10, Apache 2.0 + byte-buddy 1.9.16: https://mvnrepository.com/artifact/net.bytebuddy/byte-buddy/1.9.16, Apache 2.0 classmate 1.4.0: https://mvnrepository.com/artifact/com.fasterxml/classmate/1.4.0, Apache 2.0 clickhouse-jdbc 0.1.52: https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc/0.1.52, Apache 2.0 commons-beanutils 1.7.0 https://mvnrepository.com/artifact/commons-beanutils/commons-beanutils/1.7.0, Apache 2.0 diff --git a/pom.xml b/pom.xml index 7369ce0aa5..7a13ea4ea3 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,7 @@ ${dep.airlift.version} 1.7.2 0.9.12 + 1.9.16 @@ -330,6 +331,12 @@ ${protostuff.version} + + net.bytebuddy + byte-buddy + ${byte-buddy.version} + + org.reflections reflections @@ -1124,6 +1131,5 @@ dolphinscheduler-service dolphinscheduler-spi dolphinscheduler-microbench - dolphinscheduler-remote-connfig From d1f4e6a270478de29a447fbe1e25e7241fda3c36 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 22:34:25 +0800 Subject: [PATCH 25/68] add IUserService Interface --- .../dolphinscheduler/rpc/IUserService.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java new file mode 100644 index 0000000000..66015bae11 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc; + +import org.apache.dolphinscheduler.rpc.base.Rpc; + +/** + * IUserService + */ +public interface IUserService { + + @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) + Boolean say(String s); + + Integer hi(int num); +} From ea97ed17819c309d572368a2abf79fe28169f723 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 23:30:05 +0800 Subject: [PATCH 26/68] fix bug and add license --- dolphinscheduler-dist/release-docs/LICENSE | 4 +- .../licenses/LICENSE-javassist.txt | 469 ++++++++++++++++++ .../licenses/LICENSE-protostuff-api.txt | 202 ++++++++ .../LICENSE-protostuff-collectionschema.txt | 202 ++++++++ .../remote/serialize/ProtoStuffUtils.java | 2 +- .../apache/dolphinscheduler/rpc/base/Rpc.java | 2 + .../rpc/codec/NettyDecoder.java | 2 - .../rpc/future/RpcFuture.java | 6 +- .../rpc/remote/NettyClient.java | 1 + tools/dependencies/known-dependencies.txt | 5 +- 10 files changed, 887 insertions(+), 8 deletions(-) create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 3dc7d36b8e..03d34efa5d 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -460,7 +460,7 @@ The following components are provided under a MPL 1.1 license. See project link The text of each license is also included at licenses/LICENSE-[project].txt. jamon-runtime 2.3.1: https://mvnrepository.com/artifact/org.jamon/jamon-runtime/2.3.1, MPL-1.1 - + javassist 3.26.0-GA: https://github.com/jboss-javassist/javassist, MPL-1.1 ======================================================================== Public Domain licenses @@ -508,6 +508,8 @@ Apache 2.0 licenses remixicon 2.5.0 https://github.com/Remix-Design/remixicon Apache-2.0 protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 + protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 + protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 ======================================== diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt new file mode 100644 index 0000000000..8b7c1537ba --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-javassist.txt @@ -0,0 +1,469 @@ + MOZILLA PUBLIC LICENSE + Version 1.1 + + --------------- + +1. Definitions. + + 1.0.1. "Commercial Use" means distribution or otherwise making the + Covered Code available to a third party. + + 1.1. "Contributor" means each entity that creates or contributes to + the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Code, prior Modifications used by a Contributor, and the Modifications + made by that particular Contributor. + + 1.3. "Covered Code" means the Original Code or Modifications or the + combination of the Original Code and Modifications, in each case + including portions thereof. + + 1.4. "Electronic Distribution Mechanism" means a mechanism generally + accepted in the software development community for the electronic + transfer of data. + + 1.5. "Executable" means Covered Code in any form other than Source + Code. + + 1.6. "Initial Developer" means the individual or entity identified + as the Initial Developer in the Source Code notice required by Exhibit + A. + + 1.7. "Larger Work" means a work which combines Covered Code or + portions thereof with code not governed by the terms of this License. + + 1.8. "License" means this document. + + 1.8.1. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means any addition to or deletion from the + substance or structure of either the Original Code or any previous + Modifications. When Covered Code is released as a series of files, a + Modification is: + A. Any addition to or deletion from the contents of a file + containing Original Code or previous Modifications. + + B. Any new file that contains any part of the Original Code or + previous Modifications. + + 1.10. "Original Code" means Source Code of computer software code + which is described in the Source Code notice required by Exhibit A as + Original Code, and which, at the time of its release under this + License is not already Covered Code governed by this License. + + 1.10.1. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.11. "Source Code" means the preferred form of the Covered Code for + making modifications to it, including all modules it contains, plus + any associated interface definition files, scripts used to control + compilation and installation of an Executable, or source code + differential comparisons against either the Original Code or another + well known, available Covered Code of the Contributor's choice. The + Source Code can be in a compressed or archival form, provided the + appropriate decompression or de-archiving software is widely available + for no charge. + + 1.12. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, this + License or a future version of this License issued under Section 6.1. + For legal entities, "You" includes any entity which controls, is + controlled by, or is under common control with You. For purposes of + this definition, "control" means (a) the power, direct or indirect, + to cause the direction or management of such entity, whether by + contract or otherwise, or (b) ownership of more than fifty percent + (50%) of the outstanding shares or beneficial ownership of such + entity. + +2. Source Code License. + + 2.1. The Initial Developer Grant. + The Initial Developer hereby grants You a world-wide, royalty-free, + non-exclusive license, subject to third party intellectual property + claims: + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Code (or portions thereof) with or without Modifications, and/or + as part of a Larger Work; and + + (b) under Patents Claims infringed by the making, using or + selling of Original Code, to make, have made, use, practice, + sell, and offer for sale, and/or otherwise dispose of the + Original Code (or portions thereof). + + (c) the licenses granted in this Section 2.1(a) and (b) are + effective on the date Initial Developer first distributes + Original Code under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: 1) for code that You delete from the Original Code; 2) + separate from the Original Code; or 3) for infringements caused + by: i) the modification of the Original Code or ii) the + combination of the Original Code with other software or devices. + + 2.2. Contributor Grant. + Subject to third party intellectual property claims, each Contributor + hereby grants You a world-wide, royalty-free, non-exclusive license + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor, to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof) either on an + unmodified basis, with other Modifications, as Covered Code + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or + selling of Modifications made by that Contributor either alone + and/or in combination with its Contributor Version (or portions + of such combination), to make, use, sell, offer for sale, have + made, and/or otherwise dispose of: 1) Modifications made by that + Contributor (or portions thereof); and 2) the combination of + Modifications made by that Contributor with its Contributor + Version (or portions of such combination). + + (c) the licenses granted in Sections 2.2(a) and 2.2(b) are + effective on the date Contributor first makes Commercial Use of + the Covered Code. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: 1) for any code that Contributor has deleted from the + Contributor Version; 2) separate from the Contributor Version; + 3) for infringements caused by: i) third party modifications of + Contributor Version or ii) the combination of Modifications made + by that Contributor with other software (except as part of the + Contributor Version) or other devices; or 4) under Patent Claims + infringed by Covered Code in the absence of Modifications made by + that Contributor. + +3. Distribution Obligations. + + 3.1. Application of License. + The Modifications which You create or to which You contribute are + governed by the terms of this License, including without limitation + Section 2.2. The Source Code version of Covered Code may be + distributed only under the terms of this License or a future version + of this License released under Section 6.1, and You must include a + copy of this License with every copy of the Source Code You + distribute. You may not offer or impose any terms on any Source Code + version that alters or restricts the applicable version of this + License or the recipients' rights hereunder. However, You may include + an additional document offering the additional rights described in + Section 3.5. + + 3.2. Availability of Source Code. + Any Modification which You create or to which You contribute must be + made available in Source Code form under the terms of this License + either on the same media as an Executable version or via an accepted + Electronic Distribution Mechanism to anyone to whom you made an + Executable version available; and if made available via Electronic + Distribution Mechanism, must remain available for at least twelve (12) + months after the date it initially became available, or at least six + (6) months after a subsequent version of that particular Modification + has been made available to such recipients. You are responsible for + ensuring that the Source Code version remains available even if the + Electronic Distribution Mechanism is maintained by a third party. + + 3.3. Description of Modifications. + You must cause all Covered Code to which You contribute to contain a + file documenting the changes You made to create that Covered Code and + the date of any change. You must include a prominent statement that + the Modification is derived, directly or indirectly, from Original + Code provided by the Initial Developer and including the name of the + Initial Developer in (a) the Source Code, and (b) in any notice in an + Executable version or related documentation in which You describe the + origin or ownership of the Covered Code. + + 3.4. Intellectual Property Matters + (a) Third Party Claims. + If Contributor has knowledge that a license under a third party's + intellectual property rights is required to exercise the rights + granted by such Contributor under Sections 2.1 or 2.2, + Contributor must include a text file with the Source Code + distribution titled "LEGAL" which describes the claim and the + party making the claim in sufficient detail that a recipient will + know whom to contact. If Contributor obtains such knowledge after + the Modification is made available as described in Section 3.2, + Contributor shall promptly modify the LEGAL file in all copies + Contributor makes available thereafter and shall take other steps + (such as notifying appropriate mailing lists or newsgroups) + reasonably calculated to inform those who received the Covered + Code that new knowledge has been obtained. + + (b) Contributor APIs. + If Contributor's Modifications include an application programming + interface and Contributor has knowledge of patent licenses which + are reasonably necessary to implement that API, Contributor must + also include this information in the LEGAL file. + + (c) Representations. + Contributor represents that, except as disclosed pursuant to + Section 3.4(a) above, Contributor believes that Contributor's + Modifications are Contributor's original creation(s) and/or + Contributor has sufficient rights to grant the rights conveyed by + this License. + + 3.5. Required Notices. + You must duplicate the notice in Exhibit A in each file of the Source + Code. If it is not possible to put such notice in a particular Source + Code file due to its structure, then You must include such notice in a + location (such as a relevant directory) where a user would be likely + to look for such a notice. If You created one or more Modification(s) + You may add your name as a Contributor to the notice described in + Exhibit A. You must also duplicate this License in any documentation + for the Source Code where You describe recipients' rights or ownership + rights relating to Covered Code. You may choose to offer, and to + charge a fee for, warranty, support, indemnity or liability + obligations to one or more recipients of Covered Code. However, You + may do so only on Your own behalf, and not on behalf of the Initial + Developer or any Contributor. You must make it absolutely clear than + any such warranty, support, indemnity or liability obligation is + offered by You alone, and You hereby agree to indemnify the Initial + Developer and every Contributor for any liability incurred by the + Initial Developer or such Contributor as a result of warranty, + support, indemnity or liability terms You offer. + + 3.6. Distribution of Executable Versions. + You may distribute Covered Code in Executable form only if the + requirements of Section 3.1-3.5 have been met for that Covered Code, + and if You include a notice stating that the Source Code version of + the Covered Code is available under the terms of this License, + including a description of how and where You have fulfilled the + obligations of Section 3.2. The notice must be conspicuously included + in any notice in an Executable version, related documentation or + collateral in which You describe recipients' rights relating to the + Covered Code. You may distribute the Executable version of Covered + Code or ownership rights under a license of Your choice, which may + contain terms different from this License, provided that You are in + compliance with the terms of this License and that the license for the + Executable version does not attempt to limit or alter the recipient's + rights in the Source Code version from the rights set forth in this + License. If You distribute the Executable version under a different + license You must make it absolutely clear that any terms which differ + from this License are offered by You alone, not by the Initial + Developer or any Contributor. You hereby agree to indemnify the + Initial Developer and every Contributor for any liability incurred by + the Initial Developer or such Contributor as a result of any such + terms You offer. + + 3.7. Larger Works. + You may create a Larger Work by combining Covered Code with other code + not governed by the terms of this License and distribute the Larger + Work as a single product. In such a case, You must make sure the + requirements of this License are fulfilled for the Covered Code. + +4. Inability to Comply Due to Statute or Regulation. + + If it is impossible for You to comply with any of the terms of this + License with respect to some or all of the Covered Code due to + statute, judicial order, or regulation then You must: (a) comply with + the terms of this License to the maximum extent possible; and (b) + describe the limitations and the code they affect. Such description + must be included in the LEGAL file described in Section 3.4 and must + be included with all distributions of the Source Code. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Application of this License. + + This License applies to code to which the Initial Developer has + attached the notice in Exhibit A and to related Covered Code. + +6. Versions of the License. + + 6.1. New Versions. + Netscape Communications Corporation ("Netscape") may publish revised + and/or new versions of the License from time to time. Each version + will be given a distinguishing version number. + + 6.2. Effect of New Versions. + Once Covered Code has been published under a particular version of the + License, You may always continue to use it under the terms of that + version. You may also choose to use such Covered Code under the terms + of any subsequent version of the License published by Netscape. No one + other than Netscape has the right to modify the terms applicable to + Covered Code created under this License. + + 6.3. Derivative Works. + If You create or use a modified version of this License (which you may + only do in order to apply it to code which is not already Covered Code + governed by this License), You must (a) rename Your license so that + the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape", + "MPL", "NPL" or any confusingly similar phrase do not appear in your + license (except to note that your license differs from this License) + and (b) otherwise make it clear that Your version of the license + contains terms which differ from the Mozilla Public License and + Netscape Public License. (Filling in the name of the Initial + Developer, Original Code or Contributor in the notice described in + Exhibit A shall not of themselves be deemed to be modifications of + this License.) + +7. DISCLAIMER OF WARRANTY. + + COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, + WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF + DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. + THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE + IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT, + YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE + COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER + OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF + ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +8. TERMINATION. + + 8.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to cure + such breach within 30 days of becoming aware of the breach. All + sublicenses to the Covered Code which are properly granted shall + survive any termination of this License. Provisions which, by their + nature, must remain in effect beyond the termination of this License + shall survive. + + 8.2. If You initiate litigation by asserting a patent infringement + claim (excluding declatory judgment actions) against Initial Developer + or a Contributor (the Initial Developer or Contributor against whom + You file such action is referred to as "Participant") alleging that: + + (a) such Participant's Contributor Version directly or indirectly + infringes any patent, then any and all rights granted by such + Participant to You under Sections 2.1 and/or 2.2 of this License + shall, upon 60 days notice from Participant terminate prospectively, + unless if within 60 days after receipt of notice You either: (i) + agree in writing to pay Participant a mutually agreeable reasonable + royalty for Your past and future use of Modifications made by such + Participant, or (ii) withdraw Your litigation claim with respect to + the Contributor Version against such Participant. If within 60 days + of notice, a reasonable royalty and payment arrangement are not + mutually agreed upon in writing by the parties or the litigation claim + is not withdrawn, the rights granted by Participant to You under + Sections 2.1 and/or 2.2 automatically terminate at the expiration of + the 60 day notice period specified above. + + (b) any software, hardware, or device, other than such Participant's + Contributor Version, directly or indirectly infringes any patent, then + any rights granted to You by such Participant under Sections 2.1(b) + and 2.2(b) are revoked effective as of the date You first made, used, + sold, distributed, or had made, Modifications made by that + Participant. + + 8.3. If You assert a patent infringement claim against Participant + alleging that such Participant's Contributor Version directly or + indirectly infringes any patent where such claim is resolved (such as + by license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 8.4. In the event of termination under Sections 8.1 or 8.2 above, + all end user license agreements (excluding distributors and resellers) + which have been validly granted by You or any distributor hereunder + prior to termination shall survive termination. + +9. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL + DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE, + OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR + ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY + CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, + WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER + COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN + INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF + LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY + RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW + PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE + EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO + THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +10. U.S. GOVERNMENT END USERS. + + The Covered Code is a "commercial item," as that term is defined in + 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" and "commercial computer software documentation," as such + terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 + C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), + all U.S. Government End Users acquire Covered Code with only those + rights set forth herein. + +11. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + California law provisions (except to the extent applicable law, if + any, provides otherwise), excluding its conflict-of-law provisions. + With respect to disputes in which at least one party is a citizen of, + or an entity chartered or registered to do business in the United + States of America, any litigation relating to this License shall be + subject to the jurisdiction of the Federal Courts of the Northern + District of California, with venue lying in Santa Clara County, + California, with the losing party responsible for costs, including + without limitation, court costs and reasonable attorneys' fees and + expenses. The application of the United Nations Convention on + Contracts for the International Sale of Goods is expressly excluded. + Any law or regulation which provides that the language of a contract + shall be construed against the drafter shall not apply to this + License. + +12. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +13. MULTIPLE-LICENSED CODE. + + Initial Developer may designate portions of the Covered Code as + "Multiple-Licensed". "Multiple-Licensed" means that the Initial + Developer permits you to utilize portions of the Covered Code under + Your choice of the MPL or the alternative licenses, if any, specified + by the Initial Developer in the file described in Exhibit A. + +EXHIBIT A -Mozilla Public License. + + ``The contents of this file are subject to the Mozilla Public License + Version 1.1 (the "License"); you may not use this file except in + compliance with the License. You may obtain a copy of the License at + https://www.mozilla.org/MPL/ + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + License for the specific language governing rights and limitations + under the License. + + The Original Code is ______________________________________. + + The Initial Developer of the Original Code is ________________________. + Portions created by ______________________ are Copyright (C) ______ + _______________________. All Rights Reserved. + + Contributor(s): ______________________________________. + + Alternatively, the contents of this file may be used under the terms + of the _____ license (the "[___] License"), in which case the + provisions of [______] License are applicable instead of those + above. If you wish to allow use of your version of this file only + under the terms of the [____] License and not to allow others to use + your version of this file under the MPL, indicate your decision by + deleting the provisions above and replace them with the notice and + other provisions required by the [___] License. If you do not delete + the provisions above, a recipient may use your version of this file + under either the MPL or the [___] License." + + [NOTE: The text of this Exhibit A may differ slightly from the text of + the notices in the Source Code files of the Original Code. You should + use the text of this Exhibit A rather than the text found in the + Original Code Source Code for Your Modifications.] \ No newline at end of file diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index 96ed34d619..cb04b59749 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -52,7 +52,7 @@ public class ProtoStuffUtils { Schema schema = (Schema) schemaCache.get(clazz); if (schema == null) { schema = RuntimeSchema.getSchema(clazz); - if (schema == null) { + if (schema != null) { schemaCache.put(clazz, schema); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java index 1fc6ca3627..335759b2de 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java @@ -40,6 +40,8 @@ public @interface Rpc { boolean ack() default false; + //todo It is better to set the timeout period for synchronous calls + /** * When it is asynchronous transmission, callback must be set */ diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 70af889338..a9a1466788 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -50,10 +50,8 @@ public class NettyDecoder extends ByteToMessageDecoder { if (byteBuf.readableBytes() < dataLength) { byteBuf.resetReaderIndex(); } - //将ByteBuf转换为byte[] byte[] data = new byte[dataLength]; byteBuf.readBytes(data); - //将data转换成object Object obj = ProtoStuffUtils.deserialize(data, genericClass); list.add(obj); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 8cf1dfaa22..64f2a0afd0 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -51,17 +51,17 @@ public class RpcFuture implements Future { @Override public RpcResponse get() throws InterruptedException, ExecutionException { - boolean b = latch.await(5,TimeUnit.SECONDS); + latch.await(-1, TimeUnit.SECONDS); return response; } @Override public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - boolean b = latch.await(timeout,unit); + latch.await(timeout, unit); return response; } - public void done(RpcResponse response){ + public void done(RpcResponse response) { this.response = response; latch.countDown(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 5ed44fb7d7..936f8249f4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -217,6 +217,7 @@ public class NettyClient { result = future.get(); } catch (InterruptedException | ExecutionException e) { logger.error("send msg error，service name is {}", serviceName, e); + Thread.currentThread().interrupt(); } return result; } diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 67a12e05fb..93b915ba02 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -92,6 +92,7 @@ jackson-xc-1.9.13.jar jamon-runtime-2.3.1.jar janino-3.0.16.jar java-xmlbuilder-0.4.jar +javassist-3.26.0-GA.jar javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.inject-1.jar @@ -164,9 +165,11 @@ presto-jdbc-0.238.1.jar protobuf-java-2.5.0.jar protostuff-core-1.7.2.jar protostuff-runtime-1.7.2.jar +protostuff-api-1.7.2.jar +protostuff-collectionschema-1.7.2.jar quartz-2.3.0.jar quartz-jobs-2.3.0.jar -reflections-0.9.11.jar +reflections-0.9.12.jar slf4j-api-1.7.5.jar snakeyaml-1.23.jar snappy-0.2.jar From a63c2c6cbb9d8ff58e6b4acd78e3dfd7e39319f3 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 25 Jan 2021 23:47:07 +0800 Subject: [PATCH 27/68] fix bug --- .../java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 64f2a0afd0..b824688a46 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -51,7 +51,7 @@ public class RpcFuture implements Future { @Override public RpcResponse get() throws InterruptedException, ExecutionException { - latch.await(-1, TimeUnit.SECONDS); + latch.await(5, TimeUnit.SECONDS); return response; } From 246332b076bdd917d9b7d24ff0dd27f83567f3e6 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 26 Jan 2021 00:43:54 +0800 Subject: [PATCH 28/68] fix bug --- .../remote/serialize/ProtoStuffUtils.java | 3 +++ .../rpc/future/RpcFuture.java | 21 +++++++++++++++++-- .../rpc/remote/NettyClient.java | 16 +++++++------- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index cb04b59749..189b1a4791 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -62,6 +62,9 @@ public class ProtoStuffUtils { public static T deserialize(byte[] bytes, Class clazz) { Schema schema = getSchema(clazz); T obj = schema.newMessage(); + if(null==obj){ + return null; + } ProtostuffIOUtil.mergeFrom(bytes, obj, schema); return obj; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index b824688a46..1d40c94795 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.rpc.future; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import java.util.concurrent.CountDownLatch; @@ -34,6 +35,12 @@ public class RpcFuture implements Future { private RpcResponse response; + private RpcRequest request; + + public RpcFuture(RpcRequest rpcRequest) { + this.request = rpcRequest; + } + @Override public boolean cancel(boolean mayInterruptIfRunning) { return false; @@ -51,13 +58,23 @@ public class RpcFuture implements Future { @Override public RpcResponse get() throws InterruptedException, ExecutionException { - latch.await(5, TimeUnit.SECONDS); + boolean success = latch.await(5, TimeUnit.SECONDS); + if (!success) { + throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + + ". Request class name: " + this.request.getClassName() + + ". Request method: " + this.request.getMethodName()); + } return response; } @Override public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { - latch.await(timeout, unit); + boolean success = latch.await(timeout, unit); + if (!success) { + throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + + ". Request class name: " + this.request.getClassName() + + ". Request method: " + this.request.getMethodName()); + } return response; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 936f8249f4..28db8e5f32 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -18,16 +18,16 @@ package org.apache.dolphinscheduler.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; -import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; -import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; -import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; -import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; -import org.apache.dolphinscheduler.rpc.future.RpcFuture; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.NettyUtils; +import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; +import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; +import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; +import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; +import org.apache.dolphinscheduler.rpc.common.RpcRequest; +import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.future.RpcFuture; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; @@ -200,7 +200,7 @@ public class NettyClient { rpcRequestCache.setServiceName(serviceName); RpcFuture future = null; if (!async) { - future = new RpcFuture(); + future = new RpcFuture(request); rpcRequestCache.setRpcFuture(future); } RpcRequestTable.put(request.getRequestId(), rpcRequestCache); From 967ab641dd35cee7e9b6d2b4b210077ac20e5980 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 26 Jan 2021 01:18:47 +0800 Subject: [PATCH 29/68] fix bug and code style --- .../remote/serialize/ProtoStuffUtils.java | 17 +++++------------ .../dolphinscheduler/rpc/client/IRpcClient.java | 1 - .../rpc/client/RpcRequestTable.java | 9 ++++----- .../rpc/codec/NettyDecoder.java | 9 ++++----- .../rpc/codec/NettyEncoder.java | 7 +++---- .../rpc/config/ServiceBean.java | 1 - .../rpc/remote/NettyServer.java | 4 ++-- .../dolphinscheduler/rpc/UserService.java | 2 +- 8 files changed, 19 insertions(+), 31 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index 189b1a4791..8284171fc1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.remote.serialize; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + import io.protostuff.LinkedBuffer; import io.protostuff.ProtostuffIOUtil; import io.protostuff.Schema; import io.protostuff.runtime.RuntimeSchema; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - /** * ProtoStuffUtils */ @@ -49,20 +49,13 @@ public class ProtoStuffUtils { @SuppressWarnings("unchecked") private static Schema getSchema(Class clazz) { - Schema schema = (Schema) schemaCache.get(clazz); - if (schema == null) { - schema = RuntimeSchema.getSchema(clazz); - if (schema != null) { - schemaCache.put(clazz, schema); - } - } - return schema; + return (Schema) schemaCache.computeIfAbsent(clazz, RuntimeSchema::createFrom); } public static T deserialize(byte[] bytes, Class clazz) { Schema schema = getSchema(clazz); T obj = schema.newMessage(); - if(null==obj){ + if (null == obj) { return null; } ProtostuffIOUtil.mergeFrom(bytes, obj, schema); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java index 979a979e88..609d3b1aed 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/IRpcClient.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.remote.utils.Host; */ public interface IRpcClient { - T create(Class clazz, Host host) throws Exception; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java index f3cdff4f77..5f5e32e486 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java @@ -24,18 +24,17 @@ import java.util.concurrent.ConcurrentHashMap; */ public class RpcRequestTable { - private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); - public static void put(String requestId,RpcRequestCache rpcRequestCache){ - requestMap.put(requestId,rpcRequestCache); + public static void put(String requestId, RpcRequestCache rpcRequestCache) { + requestMap.put(requestId, rpcRequestCache); } - public static RpcRequestCache get(String requestId){ + public static RpcRequestCache get(String requestId) { return requestMap.get(requestId); } - public static void remove(String requestId){ + public static void remove(String requestId) { requestMap.remove(requestId); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index a9a1466788..d44ed0408d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -17,14 +17,14 @@ package org.apache.dolphinscheduler.rpc.codec; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; - import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; import java.util.List; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; + /** * NettyDecoder */ @@ -32,7 +32,6 @@ public class NettyDecoder extends ByteToMessageDecoder { private Class genericClass; - public NettyDecoder(Class genericClass) { this.genericClass = genericClass; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index ae9f1c9722..280fefc78c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -17,15 +17,14 @@ package org.apache.dolphinscheduler.rpc.codec; +import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; + import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import io.netty.handler.codec.MessageToByteEncoder; -import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; - /** - * @author jiangli - * @date 2021-01-12 18:52 + * NettyEncoder */ public class NettyEncoder extends MessageToByteEncoder { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index f51f35b3e6..f1c9e01767 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; - import org.reflections.Reflections; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index 52b6b4e422..3ca602a4a1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -18,10 +18,10 @@ package org.apache.dolphinscheduler.rpc.remote; import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; -import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.remote.utils.Constants; import org.apache.dolphinscheduler.remote.utils.NettyUtils; +import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; +import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java index 025735c1d3..432d06dbb1 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.rpc.base.RpcService; * UserService */ @RpcService("IUserService") -public class UserService implements IUserService{ +public class UserService implements IUserService { @Override public Boolean say(String s) { From 7a976f4c8932ed2208b5e67276e6c9cd01f84e3a Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 26 Jan 2021 09:50:08 +0800 Subject: [PATCH 30/68] fix code smell --- .../remote/serialize/ProtoStuffUtils.java | 4 +++ .../rpc/client/ConsumerConfigCache.java | 4 +++ .../rpc/client/ConsumerInterceptor.java | 3 -- .../rpc/client/RpcRequestTable.java | 4 +++ .../rpc/common/ConsumerConfigConstants.java | 4 +++ .../rpc/common/ThreadPoolManager.java | 11 +++--- .../rpc/config/ServiceBean.java | 11 ++++-- .../rpc/remote/NettyClient.java | 5 +-- .../rpc/remote/NettyClientHandler.java | 35 ++++++++----------- .../rpc/remote/NettyServerHandler.java | 11 ++---- 10 files changed, 49 insertions(+), 43 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java index 8284171fc1..9014b82276 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java @@ -30,6 +30,10 @@ import io.protostuff.runtime.RuntimeSchema; */ public class ProtoStuffUtils { + private ProtoStuffUtils() { + throw new IllegalStateException("Utility class"); + } + private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java index a407079fc5..4c8c9f4a07 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfigCache.java @@ -24,6 +24,10 @@ import java.util.concurrent.ConcurrentHashMap; */ public class ConsumerConfigCache { + private ConsumerConfigCache() { + throw new IllegalStateException("Utility class"); + } + private static ConcurrentHashMap consumerMap = new ConcurrentHashMap<>(); public static ConsumerConfig getConfigByServersName(String serviceName) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index f976d0b1d6..47bb699c9f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -75,11 +75,8 @@ public class ConsumerInterceptor { request.setClassName(method.getDeclaringClass().getSimpleName()); request.setMethodName(method.getName()); request.setParameterTypes(method.getParameterTypes()); - request.setParameters(args); - String serviceName = method.getDeclaringClass().getName(); - return request; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java index 5f5e32e486..0a62a15447 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java @@ -24,6 +24,10 @@ import java.util.concurrent.ConcurrentHashMap; */ public class RpcRequestTable { + private RpcRequestTable() { + throw new IllegalStateException("Utility class"); + } + private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); public static void put(String requestId, RpcRequestCache rpcRequestCache) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java index 478c10432c..def8fe10d4 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java @@ -22,6 +22,10 @@ package org.apache.dolphinscheduler.rpc.common; */ public class ConsumerConfigConstants { + private ConsumerConfigConstants() { + throw new IllegalStateException("Utility class"); + } + public static final Boolean DEFAULT_SYNC = false; public static final Integer DEFAULT_RETRIES = 3; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java index cb4c33a23a..1ddd574e0d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ThreadPoolManager.java @@ -29,13 +29,12 @@ public enum ThreadPoolManager { ExecutorService executorService; + private static final int WORK_QUEUE_SIZE = 200; + private static final long KEEP_ALIVE_TIME = 60; + ThreadPoolManager() { - int SIZE_WORK_QUEUE = 200; - long KEEP_ALIVE_TIME = 60; - int CORE_POOL_SIZE = Runtime.getRuntime().availableProcessors() * 2; - int MAXI_MUM_POOL_SIZE = CORE_POOL_SIZE * 4; - executorService = new ThreadPoolExecutor(CORE_POOL_SIZE, MAXI_MUM_POOL_SIZE, KEEP_ALIVE_TIME, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(SIZE_WORK_QUEUE), + executorService = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 4, KEEP_ALIVE_TIME, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(WORK_QUEUE_SIZE), new DiscardPolicy()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index f1c9e01767..6369d16725 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -36,10 +36,14 @@ public class ServiceBean { private static final Logger logger = LoggerFactory.getLogger(ServiceBean.class); - private static Map serviceMap = new HashMap<>(); + private static Map serviceMap = new HashMap<>(); private static AtomicBoolean initialized = new AtomicBoolean(false); + private ServiceBean() { + throw new IllegalStateException("Utility class"); + } + private static synchronized void init() { // todo config Reflections f = new Reflections("org/apache/dolphinscheduler/rpc"); @@ -47,16 +51,17 @@ public class ServiceBean { list.forEach(rpcClass -> { RpcService rpcService = rpcClass.getAnnotation(RpcService.class); serviceMap.put(rpcService.value(), rpcClass); + logger.info("load rpc service {}", rpcService.value()); }); } public static Class getServiceClass(String className) { if (initialized.get()) { - return (Class) serviceMap.get(className); + return serviceMap.get(className); } else { init(); } - return (Class) serviceMap.get(className); + return serviceMap.get(className); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 28db8e5f32..caabe929f7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -199,7 +199,7 @@ public class NettyClient { String serviceName = request.getClassName() + request.getMethodName(); rpcRequestCache.setServiceName(serviceName); RpcFuture future = null; - if (!async) { + if (Boolean.FALSE.equals(async)) { future = new RpcFuture(request); rpcRequestCache.setRpcFuture(future); } @@ -207,13 +207,14 @@ public class NettyClient { channel.writeAndFlush(request); RpcResponse result = null; - if (async) { + if (Boolean.TRUE.equals(async)) { result = new RpcResponse(); result.setStatus((byte) 0); result.setResult(true); return result; } try { + assert future != null; result = future.get(); } catch (InterruptedException | ExecutionException e) { logger.error("send msg error，service name is {}", serviceName, e); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index ec4f97255a..d145e34395 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -21,13 +21,13 @@ import org.apache.dolphinscheduler.rpc.client.ConsumerConfig; import org.apache.dolphinscheduler.rpc.client.ConsumerConfigCache; import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; +import org.apache.dolphinscheduler.rpc.common.RequestEventType; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; import org.apache.dolphinscheduler.rpc.future.RpcFuture; import java.lang.reflect.InvocationTargetException; -import java.net.InetSocketAddress; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,7 +36,6 @@ import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.handler.timeout.IdleStateEvent; -import io.netty.util.concurrent.FastThreadLocalThread; /** * NettyClientHandler @@ -47,11 +46,10 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyClientHandler.class); - private final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; + private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; @Override public void channelInactive(ChannelHandlerContext ctx) { - InetSocketAddress address = (InetSocketAddress) ctx.channel().remoteAddress(); ctx.channel().close(); } @@ -70,7 +68,7 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest) { String serviceName = rpcRequest.getServiceName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); - if (!consumerConfig.getAsync()) { + if (Boolean.FALSE.equals(consumerConfig.getAsync())) { RpcFuture future = rpcRequest.getRpcFuture(); RpcRequestTable.remove(rsp.getRequestId()); future.done(rsp); @@ -78,31 +76,26 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { } //async - new FastThreadLocalThread(() -> { - try { - if (rsp.getStatus() == 0) { - try { - consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); - } catch (InvocationTargetException | NoSuchMethodException e) { - logger.error("rpc call back error, serviceName {} ", serviceName, e); - } - } else { - logger.error("rpc response error ,serviceName {}", serviceName); - } - } catch (InstantiationException | IllegalAccessException e) { - logger.error("execute async error,serviceName {}", serviceName, e); + if (rsp.getStatus() == 0) { + + try { + consumerConfig.getServiceCallBackClass().getDeclaredConstructor().newInstance().run(rsp.getResult()); + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { + logger.error("rpc service call back error,serviceName {},rsp {}", serviceName, rsp); } - }).start(); + } else { + logger.error("rpc response error ,serviceName {},rsp {}", serviceName, rsp); + } + } @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent) { - IdleStateEvent event = (IdleStateEvent) evt; RpcRequest request = new RpcRequest(); - request.setEventType((byte) 0); + request.setEventType(RequestEventType.HEARTBEAT.getType()); ctx.channel().writeAndFlush(request); logger.debug("send heart beat msg..."); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java index ae4ccaab6e..28586f856c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.rpc.remote; +import org.apache.dolphinscheduler.rpc.common.RequestEventType; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; @@ -38,12 +39,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { private static final Logger logger = LoggerFactory.getLogger(NettyServerHandler.class); - private final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; - - @Override - public void channelRegistered(ChannelHandlerContext ctx) throws Exception { - super.channelRegistered(ctx); - } + private static final ThreadPoolManager threadPoolManager = ThreadPoolManager.INSTANCE; @Override public void channelInactive(ChannelHandlerContext ctx) { @@ -61,7 +57,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { RpcRequest req = (RpcRequest) msg; - if (req.getEventType() == 0) { + if (req.getEventType().equals(RequestEventType.HEARTBEAT.getType())) { logger.info("accept heartbeat msg"); return; @@ -111,7 +107,6 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - System.out.println("exceptionCaught"); logger.error("exceptionCaught : {}", cause.getMessage(), cause); ctx.channel().close(); } From d1658e9118bfe1ed678fd2fce508da603a4213f5 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 26 Jan 2021 10:07:28 +0800 Subject: [PATCH 31/68] fix init rpc request error --- .../java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java index a22c8f8a93..9f862d3565 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java @@ -30,7 +30,7 @@ public class RpcRequest { /** * @see RequestEventType */ - private Byte eventType = 1; + private Byte eventType = RequestEventType.BUSINESS.getType(); private Boolean ack; From 0f6228457aa51deeb99561897553b466eeb6bbcb Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Thu, 28 Jan 2021 13:31:52 +0800 Subject: [PATCH 32/68] add rpc protocol --- .../rpc/client/ConsumerInterceptor.java | 2 +- .../rpc/codec/NettyDecoder.java | 9 +- .../rpc/codec/NettyEncoder.java | 2 +- .../rpc/protocol/MessageHeader.java | 95 +++++++++++++++++++ .../rpc/protocol/RpcProtocol.java | 41 ++++++++ .../rpc/serializer/ProtoStuffSerializer.java | 62 ++++++++++++ .../serializer}/ProtoStuffUtils.java | 2 +- .../rpc/serializer/RpcSerializer.java | 45 +++++++++ .../rpc/serializer/Serializer.java | 28 ++++++ 9 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java rename dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/{remote/serialize => rpc/serializer}/ProtoStuffUtils.java (97%) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 47bb699c9f..0f155e07c7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -40,7 +40,7 @@ public class ConsumerInterceptor { private NettyClient nettyClient = NettyClient.getInstance(); - public ConsumerInterceptor(Host host) { + ConsumerInterceptor(Host host) { this.host = host; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index d44ed0408d..3856ecbc4d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -17,7 +17,9 @@ package org.apache.dolphinscheduler.rpc.codec; -import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; +import org.apache.dolphinscheduler.rpc.serializer.ProtoStuffUtils; +import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; +import org.apache.dolphinscheduler.rpc.serializer.Serializer; import java.util.List; @@ -49,9 +51,12 @@ public class NettyDecoder extends ByteToMessageDecoder { if (byteBuf.readableBytes() < dataLength) { byteBuf.resetReaderIndex(); } + + byte serializerType=1; byte[] data = new byte[dataLength]; byteBuf.readBytes(data); - Object obj = ProtoStuffUtils.deserialize(data, genericClass); + Serializer serializer=RpcSerializer.getSerializerByType(serializerType); + Object obj = serializer.deserialize(data, genericClass); list.add(obj); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index 280fefc78c..9333a618bf 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.rpc.codec; -import org.apache.dolphinscheduler.remote.serialize.ProtoStuffUtils; +import org.apache.dolphinscheduler.rpc.serializer.ProtoStuffUtils; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java new file mode 100644 index 0000000000..f6d091855d --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.protocol; + +public class MessageHeader { + + private byte magic=(byte) 0xbabe; + + /** + * context length + */ + private int contextLength; + + /** + * context + */ + private byte[] context; + + private String requestId; + + + private byte type; + + private byte status; + + private byte serialization; + + + public int getContextLength() { + return contextLength; + } + + public void setContextLength(int contextLength) { + this.contextLength = contextLength; + } + + public byte[] getContext() { + return context; + } + + public void setContext(byte[] context) { + this.context = context; + } + + public String getRequestId() { + return requestId; + } + + public void setRequestId(String requestId) { + this.requestId = requestId; + } + + public byte getType() { + return type; + } + + public void setType(byte type) { + this.type = type; + } + + public byte getStatus() { + return status; + } + + public void setStatus(byte status) { + this.status = status; + } + + public byte getSerialization() { + return serialization; + } + + public void setSerialization(byte serialization) { + this.serialization = serialization; + } + + public byte getMagic() { + return magic; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java new file mode 100644 index 0000000000..a217ca3aec --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.protocol; + +public class RpcProtocol{ + + private MessageHeader msgHeader; + + private T body; + + public MessageHeader getMsgHeader() { + return msgHeader; + } + + public void setMsgHeader(MessageHeader msgHeader) { + this.msgHeader = msgHeader; + } + + public T getBody() { + return body; + } + + public void setBody(T body) { + this.body = body; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java new file mode 100644 index 0000000000..3cb3e0a776 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java @@ -0,0 +1,62 @@ +package org.apache.dolphinscheduler.rpc.serializer;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import io.protostuff.LinkedBuffer; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; + +public class ProtoStuffSerializer implements Serializer{ + + private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); + + private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); + + + @SuppressWarnings("unchecked") + private static Schema getSchema(Class clazz) { + return (Schema) schemaCache.computeIfAbsent(clazz, RuntimeSchema::createFrom); + } + + @Override + public byte[] serialize(T obj) throws IOException { + Class clazz = (Class) obj.getClass(); + Schema schema = getSchema(clazz); + byte[] data; + try { + data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + } finally { + buffer.clear(); + } + return data; + } + + @Override + public T deserialize(byte[] data, Class clz) throws IOException { + Schema schema = getSchema(clz); + T obj = schema.newMessage(); + if (null == obj) { + return null; + } + ProtostuffIOUtil.mergeFrom(data, obj, schema); + return obj; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java similarity index 97% rename from dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java rename to dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java index 9014b82276..ef2a8846f1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/serialize/ProtoStuffUtils.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffUtils.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.remote.serialize; +package org.apache.dolphinscheduler.rpc.serializer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java new file mode 100644 index 0000000000..1ea4ab80be --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java @@ -0,0 +1,45 @@ +package org.apache.dolphinscheduler.rpc.serializer;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashMap; + +public enum RpcSerializer { + + + PROTOSTUFF((byte) 1, new ProtoStuffSerializer()); + + byte type; + + Serializer serializer; + + RpcSerializer(byte type, Serializer serializer) { + this.type = type; + this.serializer = serializer; + } + + private static HashMap SERIALIZERS_MAP = new HashMap<>(); + + static { + for (RpcSerializer rpcSerializer : RpcSerializer.values()) { + SERIALIZERS_MAP.put(rpcSerializer.type, rpcSerializer.serializer); + } + } + + public static Serializer getSerializerByType(byte type) { + return SERIALIZERS_MAP.get(type); + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java new file mode 100644 index 0000000000..f16d951e4e --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/Serializer.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.serializer; + +import java.io.IOException; + +public interface Serializer { + + byte[] serialize(T obj) throws IOException; + + T deserialize(byte[] data, Class clz) throws IOException; + +} From 825870c3a94bc54d02bda3e7b8b459e401cd54f8 Mon Sep 17 00:00:00 2001 From: dailidong Date: Sat, 30 Jan 2021 21:48:19 +0800 Subject: [PATCH 33/68] Rename LICENSE-protostuff-api.txt to LICENSE-protostuff.txt --- .../{LICENSE-protostuff-api.txt => LICENSE-protostuff.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dolphinscheduler-dist/release-docs/licenses/{LICENSE-protostuff-api.txt => LICENSE-protostuff.txt} (100%) diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.txt similarity index 100% rename from dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt rename to dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.txt From efde946afddaec7f7ba756a80a475f262c71649f Mon Sep 17 00:00:00 2001 From: dailidong Date: Sat, 30 Jan 2021 21:49:00 +0800 Subject: [PATCH 34/68] Delete LICENSE-protostuff-collectionschema.txt --- .../LICENSE-protostuff-collectionschema.txt | 202 ------------------ 1 file changed, 202 deletions(-) delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-collectionschema.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From 3a8df41862d42498846c1e97efb2d1551ec515bb Mon Sep 17 00:00:00 2001 From: dailidong Date: Sat, 30 Jan 2021 21:49:36 +0800 Subject: [PATCH 35/68] Delete LICENSE-protostuff-core.txt --- .../licenses/LICENSE-protostuff-core.txt | 202 ------------------ 1 file changed, 202 deletions(-) delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From 3c18d639935a11a1c19766c3c31b878804e22043 Mon Sep 17 00:00:00 2001 From: dailidong Date: Sat, 30 Jan 2021 21:50:20 +0800 Subject: [PATCH 36/68] Delete LICENSE-protostuff-runtime.txt --- .../licenses/LICENSE-protostuff-runtime.txt | 202 ------------------ 1 file changed, 202 deletions(-) delete mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt deleted file mode 100644 index d645695673..0000000000 --- a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From 0a92a53f634d50ca4e277344a4c6e79f11180826 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 31 Jan 2021 22:27:51 +0800 Subject: [PATCH 37/68] add jmh test --- dolphinscheduler-microbench/pom.xml | 4 + .../microbench/common/IUserService.java | 31 ++++++++ .../microbench/common/RpcTest.java | 77 +++++++++++++++++++ .../microbench/common/UserCallback.java | 30 ++++++++ .../microbench/common/UserService.java | 37 +++++++++ .../remote/codec/NettyDecoder.java | 38 ++++----- .../rpc/config/ServiceBean.java | 5 +- .../rpc/serializer/ProtoStuffSerializer.java | 5 +- .../apache/dolphinscheduler/rpc/RpcTest.java | 1 - 9 files changed, 204 insertions(+), 24 deletions(-) create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java create mode 100644 dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java diff --git a/dolphinscheduler-microbench/pom.xml b/dolphinscheduler-microbench/pom.xml index 606ecd3c38..c0c095abe7 100644 --- a/dolphinscheduler-microbench/pom.xml +++ b/dolphinscheduler-microbench/pom.xml @@ -61,6 +61,10 @@ org.slf4j slf4j-api + + org.apache.dolphinscheduler + dolphinscheduler-remote + diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java new file mode 100644 index 0000000000..3a77aa8a0d --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/IUserService.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.microbench.common; + +import org.apache.dolphinscheduler.rpc.base.Rpc; + +/** + * IUserService + */ +public interface IUserService { + + @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) + Boolean say(String s); + + Integer hi(int num); +} diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java new file mode 100644 index 0000000000..ecc54f8f26 --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/RpcTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.microbench.common; + +import org.apache.dolphinscheduler.microbench.base.AbstractBaseBenchmark; +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.rpc.client.IRpcClient; +import org.apache.dolphinscheduler.rpc.client.RpcClient; +import org.apache.dolphinscheduler.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.rpc.remote.NettyServer; + +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 10, time = 1) +@State(Scope.Benchmark) +@BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) +public class RpcTest extends AbstractBaseBenchmark { + private NettyServer nettyServer; + + private IUserService userService; + + private Host host; + private IRpcClient rpcClient = new RpcClient(); + + @Setup + public void before() throws Exception { + nettyServer = new NettyServer(new NettyServerConfig()); + IRpcClient rpcClient = new RpcClient(); + host = new Host("127.0.0.1", 12346); + userService = rpcClient.create(IUserService.class, host); + + } + + @Benchmark + @BenchmarkMode({Mode.Throughput, Mode.AverageTime, Mode.SampleTime}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public void sendTest() throws Exception { + + userService = rpcClient.create(IUserService.class, host); + Integer result = userService.hi(1); + } + + @TearDown + public void after() { + NettyClient.getInstance().close(); + nettyServer.close(); + } + +} diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java new file mode 100644 index 0000000000..bb32093f91 --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserCallback.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.microbench.common; + +import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; + +/** + * UserCallback + */ +public class UserCallback extends AbstractRpcCallBack { + @Override + public void run(Object object) { + + } +} diff --git a/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java new file mode 100644 index 0000000000..ad09a34645 --- /dev/null +++ b/dolphinscheduler-microbench/src/main/java/org/apache/dolphinscheduler/microbench/common/UserService.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.microbench.common; + +import org.apache.dolphinscheduler.rpc.base.RpcService; + +/** + * UserService + */ +@RpcService("IUserService") +public class UserService implements IUserService { + + @Override + public Boolean say(String s) { + return true; + } + + @Override + public Integer hi(int num) { + return ++num; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java index 343e8c63dd..84b5c1f90b 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/codec/NettyDecoder.java @@ -17,26 +17,27 @@ package org.apache.dolphinscheduler.remote.codec; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ReplayingDecoder; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandContext; import org.apache.dolphinscheduler.remote.command.CommandHeader; import org.apache.dolphinscheduler.remote.command.CommandType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ReplayingDecoder; + /** - * netty decoder + * netty decoder */ public class NettyDecoder extends ReplayingDecoder { private static final Logger logger = LoggerFactory.getLogger(NettyDecoder.class); - public NettyDecoder(){ + public NettyDecoder() { super(State.MAGIC); } @@ -48,11 +49,10 @@ public class NettyDecoder extends ReplayingDecoder { * @param ctx channel handler context * @param in byte buffer * @param out out content - * @throws Exception */ @Override protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - switch (state()){ + switch (state()) { case MAGIC: checkMagic(in.readByte()); checkpoint(State.VERSION); @@ -102,13 +102,13 @@ public class NettyDecoder extends ReplayingDecoder { } /** - * get command type + * get command type + * * @param type type - * @return */ - private CommandType commandType(byte type){ - for(CommandType ct : CommandType.values()){ - if(ct.ordinal() == type){ + private CommandType commandType(byte type) { + for (CommandType ct : CommandType.values()) { + if (ct.ordinal() == type) { return ct; } } @@ -116,7 +116,8 @@ public class NettyDecoder extends ReplayingDecoder { } /** - * check magic + * check magic + * * @param magic magic */ private void checkMagic(byte magic) { @@ -126,8 +127,7 @@ public class NettyDecoder extends ReplayingDecoder { } /** - * check version - * @param version + * check version */ private void checkVersion(byte version) { if (version != Command.VERSION) { @@ -135,7 +135,7 @@ public class NettyDecoder extends ReplayingDecoder { } } - enum State{ + enum State { MAGIC, VERSION, COMMAND, diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index 6369d16725..507cacb1d9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -46,7 +46,10 @@ public class ServiceBean { private static synchronized void init() { // todo config - Reflections f = new Reflections("org/apache/dolphinscheduler/rpc"); + if(initialized.get()){ + return; + } + Reflections f = new Reflections("org/apache/dolphinscheduler/"); List> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class)); list.forEach(rpcClass -> { RpcService rpcService = rpcClass.getAnnotation(RpcService.class); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java index 3cb3e0a776..a608b08ac3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java @@ -24,20 +24,19 @@ import io.protostuff.ProtostuffIOUtil; import io.protostuff.Schema; import io.protostuff.runtime.RuntimeSchema; -public class ProtoStuffSerializer implements Serializer{ +public class ProtoStuffSerializer implements Serializer { private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); - @SuppressWarnings("unchecked") private static Schema getSchema(Class clazz) { return (Schema) schemaCache.computeIfAbsent(clazz, RuntimeSchema::createFrom); } @Override - public byte[] serialize(T obj) throws IOException { + public byte[] serialize(T obj) throws IOException { Class clazz = (Class) obj.getClass(); Schema schema = getSchema(clazz); byte[] data; diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java index 5a10de8d73..bf62e467e9 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java @@ -42,7 +42,6 @@ public class RpcTest { IRpcClient rpcClient = new RpcClient(); host = new Host("127.0.0.1", 12346); userService = rpcClient.create(IUserService.class, host); - } @Test From 93fb043be38f95ad3d92bf91087c01a9efe3f905 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Sun, 31 Jan 2021 22:35:10 +0800 Subject: [PATCH 38/68] code style --- .../apache/dolphinscheduler/rpc/codec/NettyDecoder.java | 5 ++--- .../apache/dolphinscheduler/rpc/config/ServiceBean.java | 2 +- .../dolphinscheduler/rpc/protocol/MessageHeader.java | 7 +++---- .../apache/dolphinscheduler/rpc/protocol/RpcProtocol.java | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 3856ecbc4d..3328dd70b7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.rpc.codec; -import org.apache.dolphinscheduler.rpc.serializer.ProtoStuffUtils; import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import org.apache.dolphinscheduler.rpc.serializer.Serializer; @@ -52,10 +51,10 @@ public class NettyDecoder extends ByteToMessageDecoder { byteBuf.resetReaderIndex(); } - byte serializerType=1; + byte serializerType = 1; byte[] data = new byte[dataLength]; byteBuf.readBytes(data); - Serializer serializer=RpcSerializer.getSerializerByType(serializerType); + Serializer serializer = RpcSerializer.getSerializerByType(serializerType); Object obj = serializer.deserialize(data, genericClass); list.add(obj); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index 507cacb1d9..cba4254e4e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -46,7 +46,7 @@ public class ServiceBean { private static synchronized void init() { // todo config - if(initialized.get()){ + if (initialized.get()) { return; } Reflections f = new Reflections("org/apache/dolphinscheduler/"); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java index f6d091855d..4d1168aa66 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java @@ -19,15 +19,15 @@ package org.apache.dolphinscheduler.rpc.protocol; public class MessageHeader { - private byte magic=(byte) 0xbabe; + private byte magic = (byte) 0xbabe; /** - * context length + * context length */ private int contextLength; /** - * context + * context */ private byte[] context; @@ -40,7 +40,6 @@ public class MessageHeader { private byte serialization; - public int getContextLength() { return contextLength; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java index a217ca3aec..d7023f1c01 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocol.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.rpc.protocol; -public class RpcProtocol{ +public class RpcProtocol { private MessageHeader msgHeader; From 54ca216abff454576abb142ba6885bee7776eadf Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 1 Feb 2021 11:16:06 +0800 Subject: [PATCH 39/68] rpc protocol --- .../rpc/client/ConsumerInterceptor.java | 18 +++++- .../rpc/client/RpcRequestTable.java | 15 +++-- .../rpc/common/RpcRequest.java | 10 +-- .../rpc/protocol/EventType.java | 43 +++++++++++++ .../rpc/protocol/MessageHeader.java | 61 ++++++++----------- .../rpc/remote/NettyClient.java | 6 +- .../rpc/serializer/RpcSerializer.java | 4 ++ 7 files changed, 107 insertions(+), 50 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 0f155e07c7..0ae31c9cbc 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -22,7 +22,11 @@ import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.rpc.base.Rpc; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; +import org.apache.dolphinscheduler.rpc.protocol.EventType; +import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import org.apache.dolphinscheduler.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import java.lang.reflect.Method; import java.util.UUID; @@ -57,6 +61,8 @@ public class ConsumerInterceptor { int retries = consumerConfig.getRetries(); + RpcProtocol protocol=buildProtocol(request); + while (retries-- > 0) { RpcResponse rsp = nettyClient.sendMsg(host, request, async); //success @@ -71,7 +77,6 @@ public class ConsumerInterceptor { private RpcRequest buildReq(Object[] args, Method method) { RpcRequest request = new RpcRequest(); - request.setRequestId(UUID.randomUUID().toString()); request.setClassName(method.getDeclaringClass().getSimpleName()); request.setMethodName(method.getName()); request.setParameterTypes(method.getParameterTypes()); @@ -97,4 +102,15 @@ public class ConsumerInterceptor { return consumerConfig; } + private RpcProtocol buildProtocol(RpcRequest req){ + RpcProtocol protocol=new RpcProtocol<>(); + MessageHeader header=new MessageHeader(); + header.setRequestId(RpcRequestTable.getRequestId()); + header.setEventType(EventType.REQUEST.getType()); + header.setSerialization(RpcSerializer.PROTOSTUFF.getType()); + protocol.setMsgHeader(header); + protocol.setBody(req); + return protocol; + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java index 0a62a15447..4d47522698 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestTable.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.rpc.client; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; /** * RpcRequestTable @@ -28,18 +29,24 @@ public class RpcRequestTable { throw new IllegalStateException("Utility class"); } - private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); + private static AtomicLong requestIdGen = new AtomicLong(0); - public static void put(String requestId, RpcRequestCache rpcRequestCache) { + private static ConcurrentHashMap requestMap = new ConcurrentHashMap<>(); + + public static void put(long requestId, RpcRequestCache rpcRequestCache) { requestMap.put(requestId, rpcRequestCache); } - public static RpcRequestCache get(String requestId) { + public static RpcRequestCache get(Long requestId) { return requestMap.get(requestId); } - public static void remove(String requestId) { + public static void remove(Long requestId) { requestMap.remove(requestId); } + public static long getRequestId() { + return requestIdGen.incrementAndGet(); + } + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java index 9f862d3565..877d40fd0e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/RpcRequest.java @@ -22,11 +22,11 @@ package org.apache.dolphinscheduler.rpc.common; */ public class RpcRequest { - private String requestId; private String className; private String methodName; private Class[] parameterTypes; private Object[] parameters; + /** * @see RequestEventType */ @@ -42,14 +42,6 @@ public class RpcRequest { this.eventType = eventType; } - public String getRequestId() { - return requestId; - } - - public void setRequestId(String requestId) { - this.requestId = requestId; - } - public String getClassName() { return className; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java new file mode 100644 index 0000000000..7f9ce6a4c8 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.protocol; + +public enum EventType { + + HEARTBEAT((byte)1,"heartbeat"), + REQUEST((byte)2,"business request"), + RESPONSE((byte)3,"business response"); + + + private Byte type; + + private String description; + + EventType(Byte type, String description) { + this.type = type; + this.description = description; + } + + public Byte getType() { + return type; + } + + public String getDescription() { + return description; + } +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java index 4d1168aa66..858113ac7a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java @@ -21,55 +21,44 @@ public class MessageHeader { private byte magic = (byte) 0xbabe; - /** - * context length - */ - private int contextLength; + private byte version; - /** - * context - */ - private byte[] context; + private byte eventType; - private String requestId; - - - private byte type; + private int msgLength; private byte status; + private long requestId; + private byte serialization; - public int getContextLength() { - return contextLength; + public byte getMagic() { + return magic; } - public void setContextLength(int contextLength) { - this.contextLength = contextLength; + public byte getVersion() { + return version; } - public byte[] getContext() { - return context; + public void setVersion(byte version) { + this.version = version; } - public void setContext(byte[] context) { - this.context = context; + public byte getEventType() { + return eventType; } - public String getRequestId() { - return requestId; + public void setEventType(byte eventType) { + this.eventType = eventType; } - public void setRequestId(String requestId) { - this.requestId = requestId; + public int getMsgLength() { + return msgLength; } - public byte getType() { - return type; - } - - public void setType(byte type) { - this.type = type; + public void setMsgLength(int msgLength) { + this.msgLength = msgLength; } public byte getStatus() { @@ -80,6 +69,14 @@ public class MessageHeader { this.status = status; } + public long getRequestId() { + return requestId; + } + + public void setRequestId(long requestId) { + this.requestId = requestId; + } + public byte getSerialization() { return serialization; } @@ -87,8 +84,4 @@ public class MessageHeader { public void setSerialization(byte serialization) { this.serialization = serialization; } - - public byte getMagic() { - return magic; - } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index caabe929f7..9e35b5c0c1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; @@ -191,10 +192,11 @@ public class NettyClient { isStarted.compareAndSet(false, true); } - public RpcResponse sendMsg(Host host, RpcRequest request, Boolean async) { + public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) { Channel channel = getChannel(host); assert channel != null; + RpcRequest request=protocol.getBody(); RpcRequestCache rpcRequestCache = new RpcRequestCache(); String serviceName = request.getClassName() + request.getMethodName(); rpcRequestCache.setServiceName(serviceName); @@ -203,7 +205,7 @@ public class NettyClient { future = new RpcFuture(request); rpcRequestCache.setRpcFuture(future); } - RpcRequestTable.put(request.getRequestId(), rpcRequestCache); + RpcRequestTable.put(protocol.getMsgHeader().getRequestId(), rpcRequestCache); channel.writeAndFlush(request); RpcResponse result = null; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java index 1ea4ab80be..5c29d774a9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/RpcSerializer.java @@ -31,6 +31,10 @@ public enum RpcSerializer { this.serializer = serializer; } + public byte getType() { + return type; + } + private static HashMap SERIALIZERS_MAP = new HashMap<>(); static { From 392136cd2436c11d976c812bc57fcb8b04296ba0 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 1 Feb 2021 19:44:35 +0800 Subject: [PATCH 40/68] rpc protocol --- .../remote/utils/Constants.java | 2 +- .../rpc/client/ConsumerInterceptor.java | 2 +- .../rpc/codec/NettyDecoder.java | 52 ++++++++++---- .../rpc/codec/NettyEncoder.java | 36 +++++++--- .../rpc/config/ServiceBean.java | 1 + .../rpc/future/RpcFuture.java | 8 +-- .../rpc/protocol/MessageHeader.java | 6 +- .../rpc/protocol/RpcProtocolConstants.java | 26 +++++++ .../rpc/remote/NettyClient.java | 3 +- .../rpc/remote/NettyClientHandler.java | 26 ++++--- .../rpc/remote/NettyServerHandler.java | 26 ++++--- .../rpc/serializer/ProtoStuffSerializer.java | 5 +- .../apache/dolphinscheduler/rpc/MainTest.java | 67 +++++++++++++++++++ .../apache/dolphinscheduler/rpc/Server.java | 26 +++++++ .../dolphinscheduler/rpc/UserService.java | 2 + 15 files changed, 234 insertions(+), 54 deletions(-) create mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java create mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index 866ebb6c2b..dfb2d53378 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -31,7 +31,7 @@ public class Constants { public static final int NETTY_SERVER_HEART_BEAT_TIME = 1000 * 60 * 3 + 1000; - public static final int NETTY_CLIENT_HEART_BEAT_TIME = 1000 * 60; + public static final int NETTY_CLIENT_HEART_BEAT_TIME = 1000 * 6; /** * charset diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 0ae31c9cbc..cf0d2d240e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -64,7 +64,7 @@ public class ConsumerInterceptor { RpcProtocol protocol=buildProtocol(request); while (retries-- > 0) { - RpcResponse rsp = nettyClient.sendMsg(host, request, async); + RpcResponse rsp = nettyClient.sendMsg(host, protocol, async); //success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 3328dd70b7..9e27e5bc04 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -17,6 +17,10 @@ package org.apache.dolphinscheduler.rpc.codec; +import org.apache.dolphinscheduler.rpc.protocol.EventType; +import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocolConstants; import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import org.apache.dolphinscheduler.rpc.serializer.Serializer; @@ -39,23 +43,45 @@ public class NettyDecoder extends ByteToMessageDecoder { @Override protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { - if (byteBuf.readableBytes() < 4) { + + + if (byteBuf.readableBytes() < RpcProtocolConstants.HEADER_LENGTH) { + System.out.println("llllll 长度不够"); return; } byteBuf.markReaderIndex(); - int dataLength = byteBuf.readInt(); - if (dataLength < 0) { - channelHandlerContext.close(); - } - if (byteBuf.readableBytes() < dataLength) { - byteBuf.resetReaderIndex(); - } - byte serializerType = 1; - byte[] data = new byte[dataLength]; + short magic = byteBuf.readShort(); + if (RpcProtocolConstants.MAGIC != magic) { + throw new IllegalArgumentException("magic number is illegal, " + magic); + } + byte eventType = byteBuf.readByte(); + byte version = byteBuf.readByte(); + byte serialization=byteBuf.readByte(); + + byte state = byteBuf.readByte(); + long requestId=byteBuf.readLong(); + int dataLength = byteBuf.readInt(); + + byte[] data=new byte[dataLength]; byteBuf.readBytes(data); - Serializer serializer = RpcSerializer.getSerializerByType(serializerType); - Object obj = serializer.deserialize(data, genericClass); - list.add(obj); + RpcProtocol rpcProtocol=new RpcProtocol(); + MessageHeader header=new MessageHeader(); + + header.setVersion(version); + header.setSerialization(serialization); + header.setStatus(state); + header.setRequestId(requestId); + header.setEventType(eventType); + header.setMsgLength(dataLength); + rpcProtocol.setMsgHeader(header); + if(eventType!= EventType.HEARTBEAT.getType()){ + Serializer serializer = RpcSerializer.getSerializerByType(serialization); + Object obj = serializer.deserialize(data, genericClass); + rpcProtocol.setBody(obj); + } + list.add(rpcProtocol); } + + } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index 9333a618bf..e28bc316d5 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -17,7 +17,12 @@ package org.apache.dolphinscheduler.rpc.codec; +import org.apache.dolphinscheduler.rpc.protocol.EventType; +import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import org.apache.dolphinscheduler.rpc.serializer.ProtoStuffUtils; +import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; +import org.apache.dolphinscheduler.rpc.serializer.Serializer; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; @@ -26,9 +31,7 @@ import io.netty.handler.codec.MessageToByteEncoder; /** * NettyEncoder */ -public class NettyEncoder extends MessageToByteEncoder { - - +public class NettyEncoder extends MessageToByteEncoder> { private Class genericClass; public NettyEncoder(Class genericClass) { @@ -36,12 +39,29 @@ public class NettyEncoder extends MessageToByteEncoder { } @Override - protected void encode(ChannelHandlerContext channelHandlerContext, Object o, ByteBuf byteBuf) throws Exception { - if (genericClass.isInstance(o)) { - byte[] data = ProtoStuffUtils.serialize(o); - byteBuf.writeInt(data.length); - byteBuf.writeBytes(data); + protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, ByteBuf byteBuf) throws Exception { + + + MessageHeader msgHeader = msg.getMsgHeader(); + byteBuf.writeShort(msgHeader.getMagic()); + if(msgHeader.getEventType()== EventType.HEARTBEAT.getType()){ + byteBuf.writeByte(EventType.HEARTBEAT.getType()); + System.out.println("heart beat "); + return; } + + byteBuf.writeByte(msgHeader.getEventType()); + byteBuf.writeByte(msgHeader.getVersion()); + byteBuf.writeByte(msgHeader.getSerialization()); + + byteBuf.writeByte(msgHeader.getStatus()); + byteBuf.writeLong(msgHeader.getRequestId()); + + Serializer rpcSerializer = RpcSerializer.getSerializerByType(msgHeader.getSerialization()); + + byte[] data = rpcSerializer.serialize(msg.getBody()); + byteBuf.writeInt(data.length); + byteBuf.writeBytes(data); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java index cba4254e4e..a51281d8d8 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java @@ -56,6 +56,7 @@ public class ServiceBean { serviceMap.put(rpcService.value(), rpcClass); logger.info("load rpc service {}", rpcService.value()); }); + initialized.set(true); } public static Class getServiceClass(String className) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 1d40c94795..603aaa089f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -59,22 +59,22 @@ public class RpcFuture implements Future { @Override public RpcResponse get() throws InterruptedException, ExecutionException { boolean success = latch.await(5, TimeUnit.SECONDS); - if (!success) { + /* if (!success) { throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + ". Request class name: " + this.request.getClassName() + ". Request method: " + this.request.getMethodName()); - } + }*/ return response; } @Override public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { boolean success = latch.await(timeout, unit); - if (!success) { + /* if (!success) { throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + ". Request class name: " + this.request.getClassName() + ". Request method: " + this.request.getMethodName()); - } + }*/ return response; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java index 858113ac7a..5a176d6240 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.rpc.protocol; public class MessageHeader { - private byte magic = (byte) 0xbabe; - private byte version; private byte eventType; @@ -33,7 +31,9 @@ public class MessageHeader { private byte serialization; - public byte getMagic() { + private short magic = RpcProtocolConstants.MAGIC; + + public short getMagic() { return magic; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java new file mode 100644 index 0000000000..549b29a993 --- /dev/null +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.rpc.protocol; + +public class RpcProtocolConstants { + + public static final int HEADER_LENGTH = 18; + + public static final short MAGIC = (short) 0xbabe; + +} diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 9e35b5c0c1..9a247d9b28 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -206,8 +206,7 @@ public class NettyClient { rpcRequestCache.setRpcFuture(future); } RpcRequestTable.put(protocol.getMsgHeader().getRequestId(), rpcRequestCache); - channel.writeAndFlush(request); - + channel.writeAndFlush(protocol); RpcResponse result = null; if (Boolean.TRUE.equals(async)) { result = new RpcResponse(); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index d145e34395..e363fa80be 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -26,6 +26,9 @@ import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; import org.apache.dolphinscheduler.rpc.future.RpcFuture; +import org.apache.dolphinscheduler.rpc.protocol.EventType; +import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import java.lang.reflect.InvocationTargetException; @@ -55,22 +58,26 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - RpcResponse rsp = (RpcResponse) msg; - RpcRequestCache rpcRequest = RpcRequestTable.get(rsp.getRequestId()); + System.out.println("xxxxxxxx"+msg.getClass().getSimpleName()); + RpcProtocol rpcProtocol= (RpcProtocol) msg; + + RpcResponse rsp = (RpcResponse) rpcProtocol.getBody(); + long reqId=rpcProtocol.getMsgHeader().getRequestId(); + RpcRequestCache rpcRequest = RpcRequestTable.get(reqId); if (null == rpcRequest) { logger.warn("rpc read error,this request does not exist"); return; } - threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest)); + threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest,reqId)); } - private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest) { + private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest,long reqId) { String serviceName = rpcRequest.getServiceName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (Boolean.FALSE.equals(consumerConfig.getAsync())) { RpcFuture future = rpcRequest.getRpcFuture(); - RpcRequestTable.remove(rsp.getRequestId()); + RpcRequestTable.remove(reqId); future.done(rsp); return; @@ -94,9 +101,12 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent) { - RpcRequest request = new RpcRequest(); - request.setEventType(RequestEventType.HEARTBEAT.getType()); - ctx.channel().writeAndFlush(request); + RpcProtocol rpcProtocol=new RpcProtocol(); + MessageHeader messageHeader=new MessageHeader(); + messageHeader.setEventType(EventType.HEARTBEAT.getType()); + rpcProtocol.setMsgHeader(messageHeader); + rpcProtocol.setBody(new RpcRequest()); + ctx.channel().writeAndFlush(rpcProtocol); logger.debug("send heart beat msg..."); } else { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java index 28586f856c..c19a29069d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java @@ -22,6 +22,8 @@ import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; import org.apache.dolphinscheduler.rpc.config.ServiceBean; +import org.apache.dolphinscheduler.rpc.protocol.EventType; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import java.lang.reflect.Method; @@ -54,20 +56,20 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - - RpcRequest req = (RpcRequest) msg; - - if (req.getEventType().equals(RequestEventType.HEARTBEAT.getType())) { - - logger.info("accept heartbeat msg"); + System.out.println("channel read"+msg.getClass().getSimpleName()); + RpcProtocol rpcProtocol= (RpcProtocol) msg; + if(rpcProtocol.getMsgHeader().getEventType()==EventType.HEARTBEAT.getType()){ return; } - threadPoolManager.addExecuteTask(() -> readHandler(ctx, req)); + threadPoolManager.addExecuteTask(() -> readHandler(ctx, rpcProtocol)); } - private void readHandler(ChannelHandlerContext ctx, RpcRequest req) { + private void readHandler(ChannelHandlerContext ctx, RpcProtocol protocol) { + + RpcRequest req= (RpcRequest) protocol.getBody(); RpcResponse response = new RpcResponse(); - response.setRequestId(req.getRequestId()); + + // response.setRequestId(req.getRequestId()); response.setStatus((byte) 0); @@ -93,13 +95,15 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } response.setResult(result); - ctx.writeAndFlush(response); + protocol.setBody(response); + protocol.getMsgHeader().setEventType(EventType.RESPONSE.getType()); + ctx.writeAndFlush(protocol); } @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent) { - ctx.channel().close(); + logger.debug("IdleStateEvent triggered, send heartbeat to channel " + ctx.channel()); } else { super.userEventTriggered(ctx, evt); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java index a608b08ac3..a5b9250fc1 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/serializer/ProtoStuffSerializer.java @@ -15,7 +15,6 @@ package org.apache.dolphinscheduler.rpc.serializer;/* * limitations under the License. */ -import java.io.IOException; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -36,7 +35,7 @@ public class ProtoStuffSerializer implements Serializer { } @Override - public byte[] serialize(T obj) throws IOException { + public byte[] serialize(T obj) { Class clazz = (Class) obj.getClass(); Schema schema = getSchema(clazz); byte[] data; @@ -49,7 +48,7 @@ public class ProtoStuffSerializer implements Serializer { } @Override - public T deserialize(byte[] data, Class clz) throws IOException { + public T deserialize(byte[] data, Class clz) { Schema schema = getSchema(clz); T obj = schema.newMessage(); if (null == obj) { diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java new file mode 100644 index 0000000000..59528d0cf9 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java @@ -0,0 +1,67 @@ +package org.apache.dolphinscheduler.rpc;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.remote.utils.Host; +import org.apache.dolphinscheduler.rpc.client.IRpcClient; +import org.apache.dolphinscheduler.rpc.client.RpcClient; +import org.apache.dolphinscheduler.rpc.protocol.RpcProtocolConstants; +import org.apache.dolphinscheduler.rpc.remote.NettyClient; +import org.apache.dolphinscheduler.rpc.remote.NettyServer; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class MainTest { + + + private IUserService userService; + + private Host host; + + public static void main(String[] args) throws Exception { + + IRpcClient rpcClient = new RpcClient(); + Host host = new Host("127.0.0.1", 12346); + IUserService userService = rpcClient.create(IUserService.class, host); + Integer result = userService.hi(3); + // Assert.assertSame(4, result); + result = userService.hi(4); + // Assert.assertSame(5, result); + // userService.say("sync"); + + + // NettyClient nettyClient = NettyClient.getInstance(); + // NettyClient.getInstance().close(); + // nettyServer.close(); + } + + public void sendTest() { + Integer result = userService.hi(3); + Assert.assertSame(4, result); + result = userService.hi(4); + Assert.assertSame(5, result); + userService.say("sync"); + + + NettyClient.getInstance().close(); + + } + +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java new file mode 100644 index 0000000000..b9a660aa89 --- /dev/null +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java @@ -0,0 +1,26 @@ +package org.apache.dolphinscheduler.rpc;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +import org.apache.dolphinscheduler.rpc.remote.NettyServer; + +public class Server { + + public static void main(String[] args) { + NettyServer nettyServer=new NettyServer(new NettyServerConfig()); + } +} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java index 432d06dbb1..967a42378e 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -32,6 +32,8 @@ public class UserService implements IUserService { @Override public Integer hi(int num) { + + System.out.println("hihihihi+"+num); return ++num; } } From 457807936883959f70dcc8230329311e707b33fe Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 1 Feb 2021 22:18:47 +0800 Subject: [PATCH 41/68] rpc protocol --- .../rpc/client/ConsumerInterceptor.java | 20 ++++++++++----- .../rpc/codec/NettyDecoder.java | 23 +++++++---------- .../rpc/codec/NettyEncoder.java | 24 ++++++------------ .../rpc/future/RpcFuture.java | 19 ++++++++------ .../rpc/protocol/EventType.java | 1 - .../rpc/protocol/MessageHeader.java | 20 ++++----------- .../rpc/protocol/RpcProtocolConstants.java | 2 +- .../rpc/remote/NettyClient.java | 10 ++++---- .../rpc/remote/NettyClientHandler.java | 18 +++++-------- .../rpc/remote/NettyServerHandler.java | 15 +++++------ .../apache/dolphinscheduler/rpc/MainTest.java | 25 ++++--------------- 11 files changed, 69 insertions(+), 108 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index cf0d2d240e..15281a3d2c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -29,7 +29,9 @@ import org.apache.dolphinscheduler.rpc.remote.NettyClient; import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import java.lang.reflect.Method; -import java.util.UUID; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import net.bytebuddy.implementation.bind.annotation.AllArguments; import net.bytebuddy.implementation.bind.annotation.Origin; @@ -40,6 +42,7 @@ import net.bytebuddy.implementation.bind.annotation.RuntimeType; */ public class ConsumerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(ConsumerInterceptor.class); private Host host; private NettyClient nettyClient = NettyClient.getInstance(); @@ -61,10 +64,15 @@ public class ConsumerInterceptor { int retries = consumerConfig.getRetries(); - RpcProtocol protocol=buildProtocol(request); + RpcProtocol protocol = buildProtocol(request); while (retries-- > 0) { - RpcResponse rsp = nettyClient.sendMsg(host, protocol, async); + RpcResponse rsp = null; + try { + rsp = nettyClient.sendMsg(host, protocol, async); + } catch (InterruptedException e) { + logger.warn("send msg error ", e); + } //success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); @@ -102,9 +110,9 @@ public class ConsumerInterceptor { return consumerConfig; } - private RpcProtocol buildProtocol(RpcRequest req){ - RpcProtocol protocol=new RpcProtocol<>(); - MessageHeader header=new MessageHeader(); + private RpcProtocol buildProtocol(RpcRequest req) { + RpcProtocol protocol = new RpcProtocol<>(); + MessageHeader header = new MessageHeader(); header.setRequestId(RpcRequestTable.getRequestId()); header.setEventType(EventType.REQUEST.getType()); header.setSerialization(RpcSerializer.PROTOSTUFF.getType()); diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java index 9e27e5bc04..813bf719f2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyDecoder.java @@ -43,39 +43,35 @@ public class NettyDecoder extends ByteToMessageDecoder { @Override protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf, List list) throws Exception { - - if (byteBuf.readableBytes() < RpcProtocolConstants.HEADER_LENGTH) { - System.out.println("llllll 长度不够"); return; } + byteBuf.markReaderIndex(); short magic = byteBuf.readShort(); + if (RpcProtocolConstants.MAGIC != magic) { throw new IllegalArgumentException("magic number is illegal, " + magic); } byte eventType = byteBuf.readByte(); byte version = byteBuf.readByte(); - byte serialization=byteBuf.readByte(); - - byte state = byteBuf.readByte(); - long requestId=byteBuf.readLong(); + byte serialization = byteBuf.readByte(); + long requestId = byteBuf.readLong(); int dataLength = byteBuf.readInt(); + byte[] data = new byte[dataLength]; - byte[] data=new byte[dataLength]; - byteBuf.readBytes(data); - RpcProtocol rpcProtocol=new RpcProtocol(); - MessageHeader header=new MessageHeader(); + RpcProtocol rpcProtocol = new RpcProtocol(); + MessageHeader header = new MessageHeader(); header.setVersion(version); header.setSerialization(serialization); - header.setStatus(state); header.setRequestId(requestId); header.setEventType(eventType); header.setMsgLength(dataLength); + byteBuf.readBytes(data); rpcProtocol.setMsgHeader(header); - if(eventType!= EventType.HEARTBEAT.getType()){ + if (eventType != EventType.HEARTBEAT.getType()) { Serializer serializer = RpcSerializer.getSerializerByType(serialization); Object obj = serializer.deserialize(data, genericClass); rpcProtocol.setBody(obj); @@ -83,5 +79,4 @@ public class NettyDecoder extends ByteToMessageDecoder { list.add(rpcProtocol); } - } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index e28bc316d5..4f1fd53c2a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -17,10 +17,8 @@ package org.apache.dolphinscheduler.rpc.codec; -import org.apache.dolphinscheduler.rpc.protocol.EventType; import org.apache.dolphinscheduler.rpc.protocol.MessageHeader; import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; -import org.apache.dolphinscheduler.rpc.serializer.ProtoStuffUtils; import org.apache.dolphinscheduler.rpc.serializer.RpcSerializer; import org.apache.dolphinscheduler.rpc.serializer.Serializer; @@ -40,28 +38,20 @@ public class NettyEncoder extends MessageToByteEncoder> { @Override protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, ByteBuf byteBuf) throws Exception { - - MessageHeader msgHeader = msg.getMsgHeader(); byteBuf.writeShort(msgHeader.getMagic()); - if(msgHeader.getEventType()== EventType.HEARTBEAT.getType()){ - byteBuf.writeByte(EventType.HEARTBEAT.getType()); - System.out.println("heart beat "); - return; - } - - byteBuf.writeByte(msgHeader.getEventType()); byteBuf.writeByte(msgHeader.getVersion()); byteBuf.writeByte(msgHeader.getSerialization()); - - byteBuf.writeByte(msgHeader.getStatus()); byteBuf.writeLong(msgHeader.getRequestId()); - + byte[] data = new byte[0]; + int msgLength = msgHeader.getMsgLength(); Serializer rpcSerializer = RpcSerializer.getSerializerByType(msgHeader.getSerialization()); - - byte[] data = rpcSerializer.serialize(msg.getBody()); - byteBuf.writeInt(data.length); + if (null != rpcSerializer) { + data = rpcSerializer.serialize(msg.getBody()); + msgLength = data.length; + } + byteBuf.writeInt(msgLength); byteBuf.writeBytes(data); } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 603aaa089f..73ceb25404 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -37,8 +37,11 @@ public class RpcFuture implements Future { private RpcRequest request; - public RpcFuture(RpcRequest rpcRequest) { + private long requestId; + + public RpcFuture(RpcRequest rpcRequest,long requestId) { this.request = rpcRequest; + this.requestId=requestId; } @Override @@ -57,24 +60,24 @@ public class RpcFuture implements Future { } @Override - public RpcResponse get() throws InterruptedException, ExecutionException { + public RpcResponse get() throws InterruptedException { boolean success = latch.await(5, TimeUnit.SECONDS); - /* if (!success) { - throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + if (!success) { + throw new RuntimeException("Timeout exception. Request id: " + this.requestId + ". Request class name: " + this.request.getClassName() + ". Request method: " + this.request.getMethodName()); - }*/ + } return response; } @Override public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { boolean success = latch.await(timeout, unit); - /* if (!success) { - throw new RuntimeException("Timeout exception. Request id: " + this.request.getRequestId() + if (!success) { + throw new RuntimeException("Timeout exception. Request id: " + requestId + ". Request class name: " + this.request.getClassName() + ". Request method: " + this.request.getMethodName()); - }*/ + } return response; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java index 7f9ce6a4c8..e6a85d3b8d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/EventType.java @@ -23,7 +23,6 @@ public enum EventType { REQUEST((byte)2,"business request"), RESPONSE((byte)3,"business response"); - private Byte type; private String description; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java index 5a176d6240..d16b5883d3 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/MessageHeader.java @@ -19,19 +19,17 @@ package org.apache.dolphinscheduler.rpc.protocol; public class MessageHeader { - private byte version; + private byte version = 1; private byte eventType; - private int msgLength; + private int msgLength = 0; - private byte status; + private long requestId = 0L; - private long requestId; + private byte serialization = 0; - private byte serialization; - - private short magic = RpcProtocolConstants.MAGIC; + private short magic = RpcProtocolConstants.MAGIC; public short getMagic() { return magic; @@ -61,14 +59,6 @@ public class MessageHeader { this.msgLength = msgLength; } - public byte getStatus() { - return status; - } - - public void setStatus(byte status) { - this.status = status; - } - public long getRequestId() { return requestId; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java index 549b29a993..8beee5b248 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java @@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.rpc.protocol; public class RpcProtocolConstants { - public static final int HEADER_LENGTH = 18; + public static final int HEADER_LENGTH = 17; public static final short MAGIC = (short) 0xbabe; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 9a247d9b28..2df158e454 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -32,7 +32,6 @@ import org.apache.dolphinscheduler.rpc.protocol.RpcProtocol; import java.net.InetSocketAddress; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -192,17 +191,18 @@ public class NettyClient { isStarted.compareAndSet(false, true); } - public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) { + public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) throws InterruptedException { Channel channel = getChannel(host); assert channel != null; - RpcRequest request=protocol.getBody(); + RpcRequest request = protocol.getBody(); RpcRequestCache rpcRequestCache = new RpcRequestCache(); String serviceName = request.getClassName() + request.getMethodName(); rpcRequestCache.setServiceName(serviceName); + long reqId = protocol.getMsgHeader().getRequestId(); RpcFuture future = null; if (Boolean.FALSE.equals(async)) { - future = new RpcFuture(request); + future = new RpcFuture(request, reqId); rpcRequestCache.setRpcFuture(future); } RpcRequestTable.put(protocol.getMsgHeader().getRequestId(), rpcRequestCache); @@ -217,7 +217,7 @@ public class NettyClient { try { assert future != null; result = future.get(); - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException e) { logger.error("send msg error，service name is {}", serviceName, e); Thread.currentThread().interrupt(); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index e363fa80be..6bc61cff40 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -21,8 +21,6 @@ import org.apache.dolphinscheduler.rpc.client.ConsumerConfig; import org.apache.dolphinscheduler.rpc.client.ConsumerConfigCache; import org.apache.dolphinscheduler.rpc.client.RpcRequestCache; import org.apache.dolphinscheduler.rpc.client.RpcRequestTable; -import org.apache.dolphinscheduler.rpc.common.RequestEventType; -import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; import org.apache.dolphinscheduler.rpc.future.RpcFuture; @@ -58,21 +56,20 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { - System.out.println("xxxxxxxx"+msg.getClass().getSimpleName()); - RpcProtocol rpcProtocol= (RpcProtocol) msg; + RpcProtocol rpcProtocol = (RpcProtocol) msg; RpcResponse rsp = (RpcResponse) rpcProtocol.getBody(); - long reqId=rpcProtocol.getMsgHeader().getRequestId(); + long reqId = rpcProtocol.getMsgHeader().getRequestId(); RpcRequestCache rpcRequest = RpcRequestTable.get(reqId); if (null == rpcRequest) { logger.warn("rpc read error,this request does not exist"); return; } - threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest,reqId)); + threadPoolManager.addExecuteTask(() -> readHandler(rsp, rpcRequest, reqId)); } - private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest,long reqId) { + private void readHandler(RpcResponse rsp, RpcRequestCache rpcRequest, long reqId) { String serviceName = rpcRequest.getServiceName(); ConsumerConfig consumerConfig = ConsumerConfigCache.getConfigByServersName(serviceName); if (Boolean.FALSE.equals(consumerConfig.getAsync())) { @@ -82,7 +79,6 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { return; } - //async if (rsp.getStatus() == 0) { @@ -101,14 +97,12 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { if (evt instanceof IdleStateEvent) { - RpcProtocol rpcProtocol=new RpcProtocol(); - MessageHeader messageHeader=new MessageHeader(); + RpcProtocol rpcProtocol = new RpcProtocol(); + MessageHeader messageHeader = new MessageHeader(); messageHeader.setEventType(EventType.HEARTBEAT.getType()); rpcProtocol.setMsgHeader(messageHeader); - rpcProtocol.setBody(new RpcRequest()); ctx.channel().writeAndFlush(rpcProtocol); logger.debug("send heart beat msg..."); - } else { super.userEventTriggered(ctx, evt); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java index c19a29069d..9dd4315a16 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServerHandler.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.rpc.remote; -import org.apache.dolphinscheduler.rpc.common.RequestEventType; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.common.ThreadPoolManager; @@ -55,22 +54,20 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { } @Override + @SuppressWarnings("unchecked") public void channelRead(ChannelHandlerContext ctx, Object msg) { - System.out.println("channel read"+msg.getClass().getSimpleName()); - RpcProtocol rpcProtocol= (RpcProtocol) msg; - if(rpcProtocol.getMsgHeader().getEventType()==EventType.HEARTBEAT.getType()){ + RpcProtocol rpcProtocol = (RpcProtocol) msg; + if (rpcProtocol.getMsgHeader().getEventType() == EventType.HEARTBEAT.getType()) { + logger.info("heart beat"); return; } threadPoolManager.addExecuteTask(() -> readHandler(ctx, rpcProtocol)); } private void readHandler(ChannelHandlerContext ctx, RpcProtocol protocol) { - - RpcRequest req= (RpcRequest) protocol.getBody(); + RpcRequest req = (RpcRequest) protocol.getBody(); RpcResponse response = new RpcResponse(); - // response.setRequestId(req.getRequestId()); - response.setStatus((byte) 0); String classname = req.getClassName(); @@ -90,7 +87,7 @@ public class NettyServerHandler extends ChannelInboundHandlerAdapter { result = method.invoke(object, arguments); } catch (Exception e) { - logger.error("netty server execute error,service name {}", classname + methodName, e); + logger.error("netty server execute error,service name :{} method name :{} ", classname + methodName, e); response.setStatus((byte) -1); } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java index 59528d0cf9..8a64f9988e 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java @@ -1,4 +1,4 @@ -package org.apache.dolphinscheduler.rpc;/* +/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -15,18 +15,14 @@ package org.apache.dolphinscheduler.rpc;/* * limitations under the License. */ -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; +package org.apache.dolphinscheduler.rpc; + import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.rpc.client.IRpcClient; import org.apache.dolphinscheduler.rpc.client.RpcClient; -import org.apache.dolphinscheduler.rpc.protocol.RpcProtocolConstants; import org.apache.dolphinscheduler.rpc.remote.NettyClient; -import org.apache.dolphinscheduler.rpc.remote.NettyServer; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; public class MainTest { @@ -38,18 +34,9 @@ public class MainTest { public static void main(String[] args) throws Exception { IRpcClient rpcClient = new RpcClient(); - Host host = new Host("127.0.0.1", 12346); - IUserService userService = rpcClient.create(IUserService.class, host); - Integer result = userService.hi(3); - // Assert.assertSame(4, result); - result = userService.hi(4); - // Assert.assertSame(5, result); - // userService.say("sync"); + Host host = new Host("127.0.0.1", 12346); + IUserService userService = rpcClient.create(IUserService.class, host); - - // NettyClient nettyClient = NettyClient.getInstance(); - // NettyClient.getInstance().close(); - // nettyServer.close(); } public void sendTest() { @@ -58,8 +45,6 @@ public class MainTest { result = userService.hi(4); Assert.assertSame(5, result); userService.say("sync"); - - NettyClient.getInstance().close(); } From d3a1da47a7c3fb1e5d247d76077d4e40889ceba6 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Mon, 1 Feb 2021 22:46:35 +0800 Subject: [PATCH 42/68] code style --- .../dolphinscheduler/rpc/client/ConsumerInterceptor.java | 1 + .../org/apache/dolphinscheduler/rpc/future/RpcFuture.java | 8 ++++---- .../test/java/org/apache/dolphinscheduler/rpc/Server.java | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 15281a3d2c..ad7b5d2c09 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -72,6 +72,7 @@ public class ConsumerInterceptor { rsp = nettyClient.sendMsg(host, protocol, async); } catch (InterruptedException e) { logger.warn("send msg error ", e); + Thread.currentThread().interrupt(); } //success if (null != rsp && rsp.getStatus() == 0) { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index 73ceb25404..b10824b24d 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -39,9 +39,9 @@ public class RpcFuture implements Future { private long requestId; - public RpcFuture(RpcRequest rpcRequest,long requestId) { + public RpcFuture(RpcRequest rpcRequest, long requestId) { this.request = rpcRequest; - this.requestId=requestId; + this.requestId = requestId; } @Override @@ -71,9 +71,9 @@ public class RpcFuture implements Future { } @Override - public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + public RpcResponse get(long timeout, TimeUnit unit) throws InterruptedException { boolean success = latch.await(timeout, unit); - if (!success) { + if (!success) { throw new RuntimeException("Timeout exception. Request id: " + requestId + ". Request class name: " + this.request.getClassName() + ". Request method: " + this.request.getMethodName()); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java index b9a660aa89..b9b098a17c 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java @@ -21,6 +21,6 @@ import org.apache.dolphinscheduler.rpc.remote.NettyServer; public class Server { public static void main(String[] args) { - NettyServer nettyServer=new NettyServer(new NettyServerConfig()); + NettyServer nettyServer = new NettyServer(new NettyServerConfig()); } } From 3ba03213fe8b544ba65bdd21f0b52ff0edd32557 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 2 Feb 2021 00:08:00 +0800 Subject: [PATCH 43/68] code style --- .../java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java | 3 +-- .../test/java/org/apache/dolphinscheduler/rpc/UserService.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java index b10824b24d..b782d86b0f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/future/RpcFuture.java @@ -21,10 +21,8 @@ import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; /** * RpcFuture @@ -61,6 +59,7 @@ public class RpcFuture implements Future { @Override public RpcResponse get() throws InterruptedException { + // the timeout period should be defined by the business party boolean success = latch.await(5, TimeUnit.SECONDS); if (!success) { throw new RuntimeException("Timeout exception. Request id: " + this.requestId diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java index 967a42378e..ba990d0772 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -33,7 +33,7 @@ public class UserService implements IUserService { @Override public Integer hi(int num) { - System.out.println("hihihihi+"+num); + System.out.println("hihihihi+" + num); return ++num; } } From c9daf6a42516884e7209d5cf8e561eaf88fe29c3 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 2 Feb 2021 00:25:31 +0800 Subject: [PATCH 44/68] async not needed callback --- .../org/apache/dolphinscheduler/rpc/base/Rpc.java | 2 ++ .../rpc/client/ConsumerConfig.java | 10 ++++++++++ .../rpc/client/ConsumerInterceptor.java | 4 ++++ .../rpc/common/ConsumerConfigConstants.java | 2 ++ .../rpc/remote/NettyClientHandler.java | 3 +++ .../apache/dolphinscheduler/rpc/IUserService.java | 5 ++++- .../org/apache/dolphinscheduler/rpc/RpcTest.java | 2 ++ .../apache/dolphinscheduler/rpc/UserCallback.java | 9 ++++++++- .../apache/dolphinscheduler/rpc/UserService.java | 15 ++++++++++++++- 9 files changed, 49 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java index 335759b2de..655aa2730e 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/base/Rpc.java @@ -40,6 +40,8 @@ public @interface Rpc { boolean ack() default false; + boolean callBack() default false; + //todo It is better to set the timeout period for synchronous calls /** diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java index 10d6cd5c4b..331b0fdf59 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerConfig.java @@ -33,6 +33,8 @@ public class ConsumerConfig { private Boolean async = ConsumerConfigConstants.DEFAULT_SYNC; + private Boolean callBack = ConsumerConfigConstants.DEFAULT_CALL_BACK; + private Integer retries = ConsumerConfigConstants.DEFAULT_RETRIES; public Class getServiceCallBackClass() { @@ -74,4 +76,12 @@ public class ConsumerConfig { void setRetries(Integer retries) { this.retries = retries; } + + public Boolean getCallBack() { + return callBack; + } + + public void setCallBack(Boolean callBack) { + this.callBack = callBack; + } } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index ad7b5d2c09..0c79a81abd 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.rpc.client; import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.rpc.base.Rpc; +import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; import org.apache.dolphinscheduler.rpc.common.RpcRequest; import org.apache.dolphinscheduler.rpc.common.RpcResponse; import org.apache.dolphinscheduler.rpc.protocol.EventType; @@ -102,6 +103,9 @@ public class ConsumerInterceptor { Rpc rpc = method.getAnnotation(Rpc.class); consumerConfig.setAsync(rpc.async()); consumerConfig.setServiceCallBackClass(rpc.serviceCallback()); + if (!rpc.serviceCallback().isInstance(AbstractRpcCallBack.class)) { + consumerConfig.setCallBack(true); + } consumerConfig.setAckCallBackClass(rpc.ackCallback()); consumerConfig.setRetries(rpc.retries()); } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java index def8fe10d4..e07c55488c 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/common/ConsumerConfigConstants.java @@ -29,4 +29,6 @@ public class ConsumerConfigConstants { public static final Boolean DEFAULT_SYNC = false; public static final Integer DEFAULT_RETRIES = 3; + + public static final Boolean DEFAULT_CALL_BACK = false; } diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java index 6bc61cff40..e810af2ce7 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClientHandler.java @@ -77,7 +77,10 @@ public class NettyClientHandler extends ChannelInboundHandlerAdapter { RpcRequestTable.remove(reqId); future.done(rsp); return; + } + if (Boolean.FALSE.equals(consumerConfig.getCallBack())) { + return; } if (rsp.getStatus() == 0) { diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java index 66015bae11..7beee00106 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/IUserService.java @@ -24,8 +24,11 @@ import org.apache.dolphinscheduler.rpc.base.Rpc; */ public interface IUserService { - @Rpc(async = true, serviceCallback = UserCallback.class, retries = 9999) + @Rpc(async = true, serviceCallback = UserCallback.class) Boolean say(String s); Integer hi(int num); + + @Rpc(async = true) + Boolean callBackIsFalse(String s); } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java index bf62e467e9..bd4211ea1a 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/RpcTest.java @@ -51,6 +51,8 @@ public class RpcTest { result = userService.hi(4); Assert.assertSame(5, result); userService.say("sync"); + userService.callBackIsFalse("async no call back"); + userService.hi(999999); } @After diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java index 882cace115..72c3c705ed 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserCallback.java @@ -19,12 +19,19 @@ package org.apache.dolphinscheduler.rpc; import org.apache.dolphinscheduler.rpc.common.AbstractRpcCallBack; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * UserCallback */ public class UserCallback extends AbstractRpcCallBack { + + private static final Logger logger = LoggerFactory.getLogger(UserCallback.class); + @Override public void run(Object object) { - + String msg = (String) object; + logger.debug("Kris---------------------------------userCallBack msg is {}", msg); } } diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java index ba990d0772..80fb69678a 100644 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java +++ b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/UserService.java @@ -19,21 +19,34 @@ package org.apache.dolphinscheduler.rpc; import org.apache.dolphinscheduler.rpc.base.RpcService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * UserService */ @RpcService("IUserService") public class UserService implements IUserService { + private static final Logger logger = LoggerFactory.getLogger(UserService.class); + @Override public Boolean say(String s) { + + logger.info("Kris UserService say-------------------------------Synchronous call msg{}", s); return true; } @Override public Integer hi(int num) { - System.out.println("hihihihi+" + num); + logger.info("Kris UserService hi-------------------------------async call msg{}", num); return ++num; } + + @Override + public Boolean callBackIsFalse(String s) { + logger.info("Kris UserService callBackIsFalse-------------------------------async call msg{}", s); + return null; + } } From 50dbe7d0c9a4b692eb23c8285a2f3aea57c539d1 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 2 Feb 2021 10:06:38 +0800 Subject: [PATCH 45/68] async not needed callback --- .../org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java | 5 ----- .../dolphinscheduler/rpc/protocol/RpcProtocolConstants.java | 4 ++++ .../org/apache/dolphinscheduler/rpc/remote/NettyClient.java | 4 ++-- .../org/apache/dolphinscheduler/rpc/remote/NettyServer.java | 3 +-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java index 4f1fd53c2a..58328134b9 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/codec/NettyEncoder.java @@ -30,11 +30,6 @@ import io.netty.handler.codec.MessageToByteEncoder; * NettyEncoder */ public class NettyEncoder extends MessageToByteEncoder> { - private Class genericClass; - - public NettyEncoder(Class genericClass) { - this.genericClass = genericClass; - } @Override protected void encode(ChannelHandlerContext channelHandlerContext, RpcProtocol msg, ByteBuf byteBuf) throws Exception { diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java index 8beee5b248..c8a2c570c2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/protocol/RpcProtocolConstants.java @@ -19,6 +19,10 @@ package org.apache.dolphinscheduler.rpc.protocol; public class RpcProtocolConstants { + public RpcProtocolConstants() { + throw new IllegalStateException("Utility class"); + } + public static final int HEADER_LENGTH = 17; public static final short MAGIC = (short) 0xbabe; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java index 2df158e454..9206da6ac2 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyClient.java @@ -181,7 +181,7 @@ public class NettyClient { @Override public void initChannel(SocketChannel ch) { ch.pipeline() - .addLast(new NettyEncoder(RpcRequest.class)) + .addLast(new NettyEncoder()) .addLast(new NettyDecoder(RpcResponse.class)) .addLast("client-idle-handler", new IdleStateHandler(Constants.NETTY_CLIENT_HEART_BEAT_TIME, 0, 0, TimeUnit.MILLISECONDS)) .addLast(new NettyClientHandler()); @@ -191,7 +191,7 @@ public class NettyClient { isStarted.compareAndSet(false, true); } - public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) throws InterruptedException { + public RpcResponse sendMsg(Host host, RpcProtocol protocol, Boolean async) { Channel channel = getChannel(host); assert channel != null; diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java index 3ca602a4a1..a012bfcd55 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/remote/NettyServer.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.remote.utils.NettyUtils; import org.apache.dolphinscheduler.rpc.codec.NettyDecoder; import org.apache.dolphinscheduler.rpc.codec.NettyEncoder; import org.apache.dolphinscheduler.rpc.common.RpcRequest; -import org.apache.dolphinscheduler.rpc.common.RpcResponse; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; @@ -172,7 +171,7 @@ public class NettyServer { private void initNettyChannel(SocketChannel ch) { ch.pipeline() .addLast(new NettyDecoder(RpcRequest.class)) - .addLast(new NettyEncoder(RpcResponse.class)) + .addLast(new NettyEncoder()) .addLast("server-idle-handle", new IdleStateHandler(0, 0, Constants.NETTY_SERVER_HEART_BEAT_TIME, TimeUnit.MILLISECONDS)) .addLast("handler", new NettyServerHandler()); } From 98d2406e5aac58af0a8b89d1cdd63c85095053bf Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 2 Feb 2021 10:23:40 +0800 Subject: [PATCH 46/68] async not needed callback --- .../rpc/client/ConsumerInterceptor.java | 13 ++--- .../apache/dolphinscheduler/rpc/MainTest.java | 52 ------------------- .../apache/dolphinscheduler/rpc/Server.java | 26 ---------- 3 files changed, 4 insertions(+), 87 deletions(-) delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java delete mode 100644 dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java index 0c79a81abd..c001e5a753 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/ConsumerInterceptor.java @@ -65,16 +65,11 @@ public class ConsumerInterceptor { int retries = consumerConfig.getRetries(); - RpcProtocol protocol = buildProtocol(request); + RpcProtocol protocol = buildProtocol(request); while (retries-- > 0) { - RpcResponse rsp = null; - try { - rsp = nettyClient.sendMsg(host, protocol, async); - } catch (InterruptedException e) { - logger.warn("send msg error ", e); - Thread.currentThread().interrupt(); - } + RpcResponse rsp; + rsp = nettyClient.sendMsg(host, protocol, async); //success if (null != rsp && rsp.getStatus() == 0) { return rsp.getResult(); @@ -115,7 +110,7 @@ public class ConsumerInterceptor { return consumerConfig; } - private RpcProtocol buildProtocol(RpcRequest req) { + private RpcProtocol buildProtocol(RpcRequest req) { RpcProtocol protocol = new RpcProtocol<>(); MessageHeader header = new MessageHeader(); header.setRequestId(RpcRequestTable.getRequestId()); diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java deleted file mode 100644 index 8a64f9988e..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/MainTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.rpc; - -import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.rpc.client.IRpcClient; -import org.apache.dolphinscheduler.rpc.client.RpcClient; -import org.apache.dolphinscheduler.rpc.remote.NettyClient; - -import org.junit.Assert; - -public class MainTest { - - - private IUserService userService; - - private Host host; - - public static void main(String[] args) throws Exception { - - IRpcClient rpcClient = new RpcClient(); - Host host = new Host("127.0.0.1", 12346); - IUserService userService = rpcClient.create(IUserService.class, host); - - } - - public void sendTest() { - Integer result = userService.hi(3); - Assert.assertSame(4, result); - result = userService.hi(4); - Assert.assertSame(5, result); - userService.say("sync"); - NettyClient.getInstance().close(); - - } - -} diff --git a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java b/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java deleted file mode 100644 index b9b098a17c..0000000000 --- a/dolphinscheduler-remote/src/test/java/org/apache/dolphinscheduler/rpc/Server.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.apache.dolphinscheduler.rpc;/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.dolphinscheduler.remote.config.NettyServerConfig; -import org.apache.dolphinscheduler.rpc.remote.NettyServer; - -public class Server { - - public static void main(String[] args) { - NettyServer nettyServer = new NettyServer(new NettyServerConfig()); - } -} From fba133c94cb9764041532171da7e561783d45539 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Wed, 3 Feb 2021 11:17:45 +0800 Subject: [PATCH 47/68] move license --- dolphinscheduler-dist/release-docs/LICENSE | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index 03d34efa5d..70df48e74f 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -384,6 +384,10 @@ The text of each license is also included at licenses/LICENSE-[project].txt. xml-apis 1.4.01: https://mvnrepository.com/artifact/xml-apis/xml-apis/1.4.01, Apache 2.0 and W3C zookeeper 3.4.14: https://mvnrepository.com/artifact/org.apache.zookeeper/zookeeper/3.4.14, Apache 2.0 presto-jdbc 0.238.1 https://mvnrepository.com/artifact/com.facebook.presto/presto-jdbc/0.238.1 + protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 + protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 + protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 + protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 ======================================================================== BSD licenses @@ -469,6 +473,11 @@ Public Domain licenses xz 1.0: https://mvnrepository.com/artifact/org.tukaani/xz/1.0, Public Domain aopalliance 1.0: https://mvnrepository.com/artifact/aopalliance/aopalliance/1.0, Public Domain +======================================== +WTFPL License +======================================== + reflections 0.9.12: https://github.com/ronmamo/reflections WTFPL + ======================================================================== UI related licenses @@ -506,19 +515,9 @@ Apache 2.0 licenses ======================================== echarts 4.1.0: https://github.com/apache/incubator-echarts Apache-2.0 remixicon 2.5.0 https://github.com/Remix-Design/remixicon Apache-2.0 - protostuff-core 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 - protostuff-runtime 1.7.2: https://github.com/protostuff/protostuff/protostuff-core Apache-2.0 - protostuff-api 1.7.2: https://github.com/protostuff/protostuff/protostuff-api Apache-2.0 - protostuff-collectionschema 1.7.2: https://github.com/protostuff/protostuff/protostuff-collectionschema Apache-2.0 ======================================== BSD licenses ======================================== - d3 3.5.17: https://github.com/d3/d3 BSD-3-Clause - - -======================================== -WTFPL License -======================================== - reflections 0.9.12: https://github.com/ronmamo/reflections WTFPL \ No newline at end of file + d3 3.5.17: https://github.com/d3/d3 BSD-3-Clause \ No newline at end of file From 12c1347749f9794169f70f43b6d3d14c87d67cd7 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 9 Feb 2021 12:01:30 +0800 Subject: [PATCH 48/68] rpc --- .../org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java index 14864d6c8b..2c0cf0cf8a 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/client/RpcRequestCache.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.rpc.client; import org.apache.dolphinscheduler.rpc.future.RpcFuture; /** - * RpcRequestCache + * Rpc Request Cache */ public class RpcRequestCache { From 91304e6d730311fa1d3fb9638d1b8b635ead55eb Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 9 Feb 2021 12:15:03 +0800 Subject: [PATCH 49/68] test --- tools/dependencies/check-LICENSE.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh index 0072554e29..ef2e408414 100755 --- a/tools/dependencies/check-LICENSE.sh +++ b/tools/dependencies/check-LICENSE.sh @@ -25,12 +25,10 @@ tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --stri # licenses echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt -echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" | tee all-dependencies.txt -# The prefix "dist/lib/" (9 chars) should be stripped to be ready to compare -sed -i 's/.\{9\}//' all-dependencies.txt +echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" -exec basename {} + | uniq | sort | tee all-dependencies.txt # Exclude all self modules(jars) to generate all third-party dependencies -echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | tee third-party-dependencies.txt +echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | uniq | sort | tee third-party-dependencies.txt # 1. Compare the third-party dependencies with known dependencies, expect that all third-party dependencies are KNOWN # and the exit code of the command is 0, otherwise we should add its license to LICENSE file and add the dependency to @@ -38,4 +36,4 @@ echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependenc # command in target OS is different from what we used to sort the file `known-dependencies.txt`, i.e. "sort the two file # using the same command (and default arguments)" -diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt) +diff -w -B -U0 <(sort < tools/dependencies/known-dependencies.txt) <(sort < third-party-dependencies.txt) \ No newline at end of file From 1c047e403a596cd87250f6237d0e4b9b7d960599 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 9 Feb 2021 12:46:12 +0800 Subject: [PATCH 50/68] test --- tools/dependencies/check-LICENSE.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dependencies/check-LICENSE.sh b/tools/dependencies/check-LICENSE.sh index ef2e408414..924d6d7133 100755 --- a/tools/dependencies/check-LICENSE.sh +++ b/tools/dependencies/check-LICENSE.sh @@ -25,7 +25,7 @@ tar -zxf dolphinscheduler-dist/target/apache-dolphinscheduler*-bin.tar.gz --stri # licenses echo '=== Self modules: ' && ./mvnw --batch-mode --quiet -Dexec.executable='echo' -Dexec.args='${project.artifactId}-${project.version}.jar' exec:exec | tee self-modules.txt -echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" -exec basename {} + | uniq | sort | tee all-dependencies.txt +echo '=== Distributed dependencies: ' && find dist/lib -name "*.jar" -exec basename {} \; | uniq | sort | tee all-dependencies.txt # Exclude all self modules(jars) to generate all third-party dependencies echo '=== Third party dependencies: ' && grep -vf self-modules.txt all-dependencies.txt | uniq | sort | tee third-party-dependencies.txt From d30500f261dd26f3e435af4c09111af7e24aa607 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Tue, 9 Feb 2021 13:17:24 +0800 Subject: [PATCH 51/68] test --- dolphinscheduler-dist/release-docs/LICENSE | 4 + .../licenses/LICENSE-javax.mail.txt | 759 ++++++++++++++++++ tools/dependencies/known-dependencies.txt | 4 + 3 files changed, 767 insertions(+) create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-javax.mail.txt diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE index dc196008ee..aef2f5c048 100644 --- a/dolphinscheduler-dist/release-docs/LICENSE +++ b/dolphinscheduler-dist/release-docs/LICENSE @@ -236,6 +236,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. commons-configuration 1.10: https://mvnrepository.com/artifact/commons-configuration/commons-configuration/1.10, Apache 2.0 commons-daemon 1.0.13 https://mvnrepository.com/artifact/commons-daemon/commons-daemon/1.0.13, Apache 2.0 commons-dbcp 1.4: https://github.com/apache/commons-dbcp, Apache 2.0 + commons-email 1.5: https://github.com/apache/commons-email, Apache 2.0 commons-httpclient 3.0.1: https://mvnrepository.com/artifact/commons-httpclient/commons-httpclient/3.0.1, Apache 2.0 commons-io 2.4: https://github.com/apache/commons-io, Apache 2.0 commons-lang 2.6: https://github.com/apache/commons-lang, Apache 2.0 @@ -321,6 +322,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. jpam 1.1: https://mvnrepository.com/artifact/net.sf.jpam/jpam/1.1, Apache 2.0 jsqlparser 2.1: https://github.com/JSQLParser/JSqlParser, Apache 2.0 or LGPL 2.1 jsr305 3.0.0: https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305, Apache 2.0 + jsr305 1.3.9: https://mvnrepository.com/artifact/com.google.code.findbugs/jsr305, Apache 2.0 j2objc-annotations 1.1 https://mvnrepository.com/artifact/com.google.j2objc/j2objc-annotations/1.1, Apache 2.0 libfb303 0.9.3: https://mvnrepository.com/artifact/org.apache.thrift/libfb303/0.9.3, Apache 2.0 libthrift 0.9.3: https://mvnrepository.com/artifact/org.apache.thrift/libthrift/0.9.3, Apache 2.0 @@ -416,8 +418,10 @@ CDDL licenses The following components are provided under the CDDL License. See project link for details. The text of each license is also included at licenses/LICENSE-[project].txt. + activation 1.1: https://mvnrepository.com/artifact/javax.activation/activation/1.1 CDDL 1.0 javax.activation-api 1.2.0: https://mvnrepository.com/artifact/javax.activation/javax.activation-api/1.2.0, CDDL and LGPL 2.0 javax.annotation-api 1.3.2: https://mvnrepository.com/artifact/javax.annotation/javax.annotation-api/1.3.2, CDDL + GPLv2 + javax.mail 1.6.2: https://mvnrepository.com/artifact/com.sun.mail/javax.mail/1.6.2, CDDL/GPLv2 javax.servlet-api 3.1.0: https://mvnrepository.com/artifact/javax.servlet/javax.servlet-api/3.1.0, CDDL + GPLv2 jaxb-api 2.3.1: https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api/2.3.1, CDDL 1.1 jaxb-impl 2.2.3-1: https://mvnrepository.com/artifact/com.sun.xml.bind/jaxb-impl/2.2.3-1, CDDL and GPL 1.1 diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-javax.mail.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-javax.mail.txt new file mode 100644 index 0000000000..a4c7fecdf5 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-javax.mail.txt @@ -0,0 +1,759 @@ +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.1 + +1. Definitions. + + 1.1. "Contributor" means each individual or entity that creates or + contributes to the creation of Modifications. + + 1.2. "Contributor Version" means the combination of the Original + Software, prior Modifications used by a Contributor (if any), and + the Modifications made by that particular Contributor. + + 1.3. "Covered Software" means (a) the Original Software, or (b) + Modifications, or (c) the combination of files containing Original + Software with files containing Modifications, in each case including + portions thereof. + + 1.4. "Executable" means the Covered Software in any form other than + Source Code. + + 1.5. "Initial Developer" means the individual or entity that first + makes Original Software available under this License. + + 1.6. "Larger Work" means a work which combines Covered Software or + portions thereof with code not governed by the terms of this License. + + 1.7. "License" means this document. + + 1.8. "Licensable" means having the right to grant, to the maximum + extent possible, whether at the time of the initial grant or + subsequently acquired, any and all of the rights conveyed herein. + + 1.9. "Modifications" means the Source Code and Executable form of + any of the following: + + A. Any file that results from an addition to, deletion from or + modification of the contents of a file containing Original Software + or previous Modifications; + + B. Any new file that contains any part of the Original Software or + previous Modification; or + + C. Any new file that is contributed or otherwise made available + under the terms of this License. + + 1.10. "Original Software" means the Source Code and Executable form + of computer software code that is originally released under this + License. + + 1.11. "Patent Claims" means any patent claim(s), now owned or + hereafter acquired, including without limitation, method, process, + and apparatus claims, in any patent Licensable by grantor. + + 1.12. "Source Code" means (a) the common form of computer software + code in which modifications are made and (b) associated + documentation included in or with such code. + + 1.13. "You" (or "Your") means an individual or a legal entity + exercising rights under, and complying with all of the terms of, + this License. For legal entities, "You" includes any entity which + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants. + + 2.1. The Initial Developer Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, the Initial Developer + hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Initial Developer, to use, reproduce, + modify, display, perform, sublicense and distribute the Original + Software (or portions thereof), with or without Modifications, + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of + Original Software, to make, have made, use, practice, sell, and + offer for sale, and/or otherwise dispose of the Original Software + (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on + the date Initial Developer first distributes or otherwise makes the + Original Software available to a third party under the terms of this + License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is + granted: (1) for code that You delete from the Original Software, or + (2) for infringements caused by: (i) the modification of the + Original Software, or (ii) the combination of the Original Software + with other software or devices. + + 2.2. Contributor Grant. + + Conditioned upon Your compliance with Section 3.1 below and subject + to third party intellectual property claims, each Contributor hereby + grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or + trademark) Licensable by Contributor to use, reproduce, modify, + display, perform, sublicense and distribute the Modifications + created by such Contributor (or portions thereof), either on an + unmodified basis, with other Modifications, as Covered Software + and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling + of Modifications made by that Contributor either alone and/or in + combination with its Contributor Version (or portions of such + combination), to make, use, sell, offer for sale, have made, and/or + otherwise dispose of: (1) Modifications made by that Contributor (or + portions thereof); and (2) the combination of Modifications made by + that Contributor with its Contributor Version (or portions of such + combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective + on the date Contributor first distributes or otherwise makes the + Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is + granted: (1) for any code that Contributor has deleted from the + Contributor Version; (2) for infringements caused by: (i) third + party modifications of Contributor Version, or (ii) the combination + of Modifications made by that Contributor with other software + (except as part of the Contributor Version) or other devices; or (3) + under Patent Claims infringed by Covered Software in the absence of + Modifications made by that Contributor. + +3. Distribution Obligations. + + 3.1. Availability of Source Code. + + Any Covered Software that You distribute or otherwise make available + in Executable form must also be made available in Source Code form + and that Source Code form must be distributed only under the terms + of this License. You must include a copy of this License with every + copy of the Source Code form of the Covered Software You distribute + or otherwise make available. You must inform recipients of any such + Covered Software in Executable form as to how they can obtain such + Covered Software in Source Code form in a reasonable manner on or + through a medium customarily used for software exchange. + + 3.2. Modifications. + + The Modifications that You create or to which You contribute are + governed by the terms of this License. You represent that You + believe Your Modifications are Your original creation(s) and/or You + have sufficient rights to grant the rights conveyed by this License. + + 3.3. Required Notices. + + You must include a notice in each of Your Modifications that + identifies You as the Contributor of the Modification. You may not + remove or alter any copyright, patent or trademark notices contained + within the Covered Software, or any notices of licensing or any + descriptive text giving attribution to any Contributor or the + Initial Developer. + + 3.4. Application of Additional Terms. + + You may not offer or impose any terms on any Covered Software in + Source Code form that alters or restricts the applicable version of + this License or the recipients' rights hereunder. You may choose to + offer, and to charge a fee for, warranty, support, indemnity or + liability obligations to one or more recipients of Covered Software. + However, you may do so only on Your own behalf, and not on behalf of + the Initial Developer or any Contributor. You must make it + absolutely clear that any such warranty, support, indemnity or + liability obligation is offered by You alone, and You hereby agree + to indemnify the Initial Developer and every Contributor for any + liability incurred by the Initial Developer or such Contributor as a + result of warranty, support, indemnity or liability terms You offer. + + 3.5. Distribution of Executable Versions. + + You may distribute the Executable form of the Covered Software under + the terms of this License or under the terms of a license of Your + choice, which may contain terms different from this License, + provided that You are in compliance with the terms of this License + and that the license for the Executable form does not attempt to + limit or alter the recipient's rights in the Source Code form from + the rights set forth in this License. If You distribute the Covered + Software in Executable form under a different license, You must make + it absolutely clear that any terms which differ from this License + are offered by You alone, not by the Initial Developer or + Contributor. You hereby agree to indemnify the Initial Developer and + every Contributor for any liability incurred by the Initial + Developer or such Contributor as a result of any such terms You offer. + + 3.6. Larger Works. + + You may create a Larger Work by combining Covered Software with + other code not governed by the terms of this License and distribute + the Larger Work as a single product. In such a case, You must make + sure the requirements of this License are fulfilled for the Covered + Software. + +4. Versions of the License. + + 4.1. New Versions. + + Oracle is the initial license steward and may publish revised and/or + new versions of this License from time to time. Each version will be + given a distinguishing version number. Except as provided in Section + 4.3, no one other than the license steward has the right to modify + this License. + + 4.2. Effect of New Versions. + + You may always continue to use, distribute or otherwise make the + Covered Software available under the terms of the version of the + License under which You originally received the Covered Software. If + the Initial Developer includes a notice in the Original Software + prohibiting it from being distributed or otherwise made available + under any subsequent version of the License, You must distribute and + make the Covered Software available under the terms of the version + of the License under which You originally received the Covered + Software. Otherwise, You may also choose to use, distribute or + otherwise make the Covered Software available under the terms of any + subsequent version of the License published by the license steward. + + 4.3. Modified Versions. + + When You are an Initial Developer and You want to create a new + license for Your Original Software, You may create and use a + modified version of this License if You: (a) rename the license and + remove any references to the name of the license steward (except to + note that the license differs from this License); and (b) otherwise + make it clear that the license contains terms which differ from this + License. + +5. DISCLAIMER OF WARRANTY. + + COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS, + WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, + INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE + IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR + NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF + THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE + DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY + OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, + REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN + ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS + AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + + 6.1. This License and the rights granted hereunder will terminate + automatically if You fail to comply with terms herein and fail to + cure such breach within 30 days of becoming aware of the breach. + Provisions which, by their nature, must remain in effect beyond the + termination of this License shall survive. + + 6.2. If You assert a patent infringement claim (excluding + declaratory judgment actions) against Initial Developer or a + Contributor (the Initial Developer or Contributor against whom You + assert such claim is referred to as "Participant") alleging that the + Participant Software (meaning the Contributor Version where the + Participant is a Contributor or the Original Software where the + Participant is the Initial Developer) directly or indirectly + infringes any patent, then any and all rights granted directly or + indirectly to You by such Participant, the Initial Developer (if the + Initial Developer is not the Participant) and all Contributors under + Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice + from Participant terminate prospectively and automatically at the + expiration of such 60 day notice period, unless if within such 60 + day period You withdraw Your claim with respect to the Participant + Software against such Participant either unilaterally or pursuant to + a written agreement with Participant. + + 6.3. If You assert a patent infringement claim against Participant + alleging that the Participant Software directly or indirectly + infringes any patent where such claim is resolved (such as by + license or settlement) prior to the initiation of patent + infringement litigation, then the reasonable value of the licenses + granted by such Participant under Sections 2.1 or 2.2 shall be taken + into account in determining the amount or value of any payment or + license. + + 6.4. In the event of termination under Sections 6.1 or 6.2 above, + all end user licenses that have been validly granted by You or any + distributor hereunder prior to termination (excluding licenses + granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + + UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT + (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE + INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF + COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE + TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR + CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT + LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER + FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR + LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE + POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT + APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH + PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH + LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR + LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION + AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + + The Covered Software is a "commercial item," as that term is defined + in 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer + software" (as that term is defined at 48 C.F.R. � + 252.227-7014(a)(1)) and "commercial computer software documentation" + as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent + with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 + (June 1995), all U.S. Government End Users acquire Covered Software + with only those rights set forth herein. This U.S. Government Rights + clause is in lieu of, and supersedes, any other FAR, DFAR, or other + clause or provision that addresses Government rights in computer + software under this License. + +9. MISCELLANEOUS. + + This License represents the complete agreement concerning subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. This License shall be governed by + the law of the jurisdiction specified in a notice contained within + the Original Software (except to the extent applicable law, if any, + provides otherwise), excluding such jurisdiction's conflict-of-law + provisions. Any litigation relating to this License shall be subject + to the jurisdiction of the courts located in the jurisdiction and + venue specified in a notice contained within the Original Software, + with the losing party responsible for costs, including, without + limitation, court costs and reasonable attorneys' fees and expenses. + The application of the United Nations Convention on Contracts for + the International Sale of Goods is expressly excluded. Any law or + regulation which provides that the language of a contract shall be + construed against the drafter shall not apply to this License. You + agree that You alone are responsible for compliance with the United + States export administration regulations (and the export control + laws and regulation of any other countries) when You use, distribute + or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + + As between Initial Developer and the Contributors, each party is + responsible for claims and damages arising, directly or indirectly, + out of its utilization of rights under this License and You agree to + work with Initial Developer and Contributors to distribute such + responsibility on an equitable basis. Nothing herein is intended or + shall be deemed to constitute any admission of liability. + +------------------------------------------------------------------------ + +NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION +LICENSE (CDDL) + +The code released under the CDDL shall be governed by the laws of the +State of California (excluding conflict-of-law provisions). Any +litigation relating to this License shall be subject to the jurisdiction +of the Federal Courts of the Northern District of California and the +state courts of the State of California, with venue lying in Santa Clara +County, California. + + + + The GNU General Public License (GPL) Version 2, June 1991 + +Copyright (C) 1989, 1991 Free Software Foundation, Inc. +51 Franklin Street, Fifth Floor +Boston, MA 02110-1335 +USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. + +Preamble + +The licenses for most software are designed to take away your freedom to +share and change it. By contrast, the GNU General Public License is +intended to guarantee your freedom to share and change free software--to +make sure the software is free for all its users. This General Public +License applies to most of the Free Software Foundation's software and +to any other program whose authors commit to using it. (Some other Free +Software Foundation software is covered by the GNU Library General +Public License instead.) You can apply it to your programs, too. + +When we speak of free software, we are referring to freedom, not price. +Our General Public Licenses are designed to make sure that you have the +freedom to distribute copies of free software (and charge for this +service if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs; and that you know you can do these things. + +To protect your rights, we need to make restrictions that forbid anyone +to deny you these rights or to ask you to surrender the rights. These +restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + +For example, if you distribute copies of such a program, whether gratis +or for a fee, you must give the recipients all the rights that you have. +You must make sure that they, too, receive or can get the source code. +And you must show them these terms so they know their rights. + +We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + +Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + +Finally, any free program is threatened constantly by software patents. +We wish to avoid the danger that redistributors of a free program will +individually obtain patent licenses, in effect making the program +proprietary. To prevent this, we have made it clear that any patent must +be licensed for everyone's free use or not licensed at all. + +The precise terms and conditions for copying, distribution and +modification follow. + +TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + +0. This License applies to any program or other work which contains a +notice placed by the copyright holder saying it may be distributed under +the terms of this General Public License. The "Program", below, refers +to any such program or work, and a "work based on the Program" means +either the Program or any derivative work under copyright law: that is +to say, a work containing the Program or a portion of it, either +verbatim or with modifications and/or translated into another language. +(Hereinafter, translation is included without limitation in the term +"modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of running +the Program is not restricted, and the output from the Program is +covered only if its contents constitute a work based on the Program +(independent of having been made by running the Program). Whether that +is true depends on what the Program does. + +1. You may copy and distribute verbatim copies of the Program's source +code as you receive it, in any medium, provided that you conspicuously +and appropriately publish on each copy an appropriate copyright notice +and disclaimer of warranty; keep intact all the notices that refer to +this License and to the absence of any warranty; and give any other +recipients of the Program a copy of this License along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + +2. You may modify your copy or copies of the Program or any portion of +it, thus forming a work based on the Program, and copy and distribute +such modifications or work under the terms of Section 1 above, provided +that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any part + thereof, to be licensed as a whole at no charge to all third parties + under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a notice + that there is no warranty (or else, saying that you provide a + warranty) and that users may redistribute the program under these + conditions, and telling the user how to view a copy of this License. + (Exception: if the Program itself is interactive but does not + normally print such an announcement, your work based on the Program + is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, and +can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based on +the Program, the distribution of the whole must be on the terms of this +License, whose permissions for other licensees extend to the entire +whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of a +storage or distribution medium does not bring the other work under the +scope of this License. + +3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections 1 + and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your cost + of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer to + distribute corresponding source code. (This alternative is allowed + only for noncommercial distribution and only if you received the + program in object code or executable form with such an offer, in + accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source code +means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to control +compilation and installation of the executable. However, as a special +exception, the source code distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies the +executable. + +If distribution of executable or object code is made by offering access +to copy from a designated place, then offering equivalent access to copy +the source code from the same place counts as distribution of the source +code, even though third parties are not compelled to copy the source +along with the object code. + +4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt otherwise +to copy, modify, sublicense or distribute the Program is void, and will +automatically terminate your rights under this License. However, parties +who have received copies, or rights, from you under this License will +not have their licenses terminated so long as such parties remain in +full compliance. + +5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and all +its terms and conditions for copying, distributing or modifying the +Program or works based on it. + +6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further restrictions +on the recipients' exercise of the rights granted herein. You are not +responsible for enforcing compliance by third parties to this License. + +7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot distribute +so as to satisfy simultaneously your obligations under this License and +any other pertinent obligations, then as a consequence you may not +distribute the Program at all. For example, if a patent license would +not permit royalty-free redistribution of the Program by all those who +receive copies directly or indirectly through you, then the only way you +could satisfy both it and this License would be to refrain entirely from +distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is implemented +by public license practices. Many people have made generous +contributions to the wide range of software distributed through that +system in reliance on consistent application of that system; it is up to +the author/donor to decide if he or she is willing to distribute +software through any other system and a licensee cannot impose that choice. + +This section is intended to make thoroughly clear what is believed to be +a consequence of the rest of this License. + +8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License may +add an explicit geographical distribution limitation excluding those +countries, so that distribution is permitted only in or among countries +not thus excluded. In such case, this License incorporates the +limitation as if written in the body of this License. + +9. The Free Software Foundation may publish revised and/or new +versions of the General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Program does not specify a version +number of this License, you may choose any version ever published by the +Free Software Foundation. + +10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the +author to ask for permission. For software which is copyrighted by the +Free Software Foundation, write to the Free Software Foundation; we +sometimes make exceptions for this. Our decision will be guided by the +two goals of preserving the free status of all derivatives of our free +software and of promoting the sharing and reuse of software generally. + +NO WARRANTY + +11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, +EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH +YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL +NECESSARY SERVICING, REPAIR OR CORRECTION. + +12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR +DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL +DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM +(INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED +INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF +THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR +OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +END OF TERMS AND CONDITIONS + +How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to +attach them to the start of each source file to most effectively convey +the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + One line to give the program's name and a brief idea of what it does. + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type + `show w'. This is free software, and you are welcome to redistribute + it under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the +appropriate parts of the General Public License. Of course, the commands +you use may be called something other than `show w' and `show c'; they +could even be mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + program `Gnomovision' (which makes passes at compilers) written by + James Hacker. + + signature of Ty Coon, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications +with the library. If this is what you want to do, use the GNU Library +General Public License instead of this License. + +# + +Certain source files distributed by Oracle America, Inc. and/or its +affiliates are subject to the following clarification and special +exception to the GPLv2, based on the GNU Project exception for its +Classpath libraries, known as the GNU Classpath Exception, but only +where Oracle has expressly included in the particular source file's +header the words "Oracle designates this particular file as subject to +the "Classpath" exception as provided by Oracle in the LICENSE file +that accompanied this code." + +You should also note that Oracle includes multiple, independent +programs in this software package. Some of those programs are provided +under licenses deemed incompatible with the GPLv2 by the Free Software +Foundation and others. For example, the package includes programs +licensed under the Apache License, Version 2.0. Such programs are +licensed to you under their original licenses. + +Oracle facilitates your further distribution of this package by adding +the Classpath Exception to the necessary parts of its GPLv2 code, which +permits you to use that code in combination with other independent +modules not licensed under the GPLv2. However, note that this would +not permit you to commingle code under an incompatible license with +Oracle's GPLv2 licensed code by, for example, cutting and pasting such +code into a file also containing Oracle's GPLv2 licensed code and then +distributing the result. Additionally, if you were to remove the +Classpath Exception from any of the files to which it applies and +distribute the result, you would likely be required to license some or +all of the other code in that distribution under the GPLv2 as well, and +since the GPLv2 is incompatible with the license terms of some items +included in the distribution by Oracle, removing the Classpath +Exception could therefore effectively compromise your ability to +further distribute the package. + +Proceed with caution and we recommend that you obtain the advice of a +lawyer skilled in open source matters before removing the Classpath +Exception or making modifications to this package which may +subsequently be redistributed and/or involve the use of third party +software. + +CLASSPATH EXCEPTION +Linking this library statically or dynamically with other modules is +making a combined work based on this library. Thus, the terms and +conditions of the GNU General Public License version 2 cover the whole +combination. + +As a special exception, the copyright holders of this library give you +permission to link this library with independent modules to produce an +executable, regardless of the license terms of these independent +modules, and to copy and distribute the resulting executable under +terms of your choice, provided that you also meet, for each linked +independent module, the terms and conditions of the license of that +module. An independent module is a module which is not derived from or +based on this library. If you modify this library, you may extend this +exception to your version of the library, but you are not obligated to +do so. If you do not wish to do so, delete this exception statement +from your version. diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 6afe96bd32..9b43cee2bf 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -1,4 +1,5 @@ HikariCP-3.2.0.jar +activation-1.1.jar animal-sniffer-annotations-1.14.jar aopalliance-1.0.jar apache-el-8.5.54.jar @@ -16,6 +17,7 @@ byte-buddy-1.9.16.jar checker-compat-qual-2.0.0.jar classmate-1.4.0.jar clickhouse-jdbc-0.1.52.jar +commons-email-1.5.jar commons-cli-1.2.jar commons-codec-1.11.jar commons-collections-3.2.2.jar @@ -97,6 +99,7 @@ javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.inject-1.jar javax.jdo-3.2.0-m3.jar +javax.mail-1.6.2.jar javax.servlet-api-3.1.0.jar javolution-5.5.1.jar jaxb-api-2.3.1.jar @@ -130,6 +133,7 @@ jpam-1.1.jar jsch-0.1.42.jar jsp-api-2.1.jar jsqlparser-2.1.jar +jsr305-1.3.9.jar jsr305-3.0.0.jar jta-1.1.jar jul-to-slf4j-1.7.30.jar From dc20d6a4cba67a7c43f0090ccf9dca05ae0b9561 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Fri, 26 Feb 2021 18:15:46 +0800 Subject: [PATCH 52/68] add NOTICE --- dolphinscheduler-dist/release-docs/NOTICE | 28 +++ ...tostuff.txt => LICENSE-protostuff-api.txt} | 0 .../licenses/LICENSE-protostuff-core.txt | 202 ++++++++++++++++++ .../licenses/LICENSE-protostuff-runtime.txt | 202 ++++++++++++++++++ .../LICENSE-protostuff.collectionschema.txt | 202 ++++++++++++++++++ 5 files changed, 634 insertions(+) rename dolphinscheduler-dist/release-docs/licenses/{LICENSE-protostuff.txt => LICENSE-protostuff-api.txt} (100%) create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt create mode 100644 dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt diff --git a/dolphinscheduler-dist/release-docs/NOTICE b/dolphinscheduler-dist/release-docs/NOTICE index 9be19fd975..b725de44b8 100644 --- a/dolphinscheduler-dist/release-docs/NOTICE +++ b/dolphinscheduler-dist/release-docs/NOTICE @@ -561,6 +561,34 @@ The Apache Software Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- +============================================================== + protostuff + Copyright 2009 David Yu dyuproject@gmail.com +============================================================== + +protobuf is copyright Google inc unless otherwise noted. +It is licensed under the BSD license. + +jackson-core-asl is copyright FasterXml unless otherwise noted. +It is licensed under the apache 2.0 license. + +antlr is copyright Terence Parr unless otherwise noted. +It is licensed under the BSD license. + +stringtemplate is copyright Terence Parr unless otherwise noted. +It is licensed under the BSD license. + +velocity is licensed under the apache 2.0 license. + +B64Code.java is copyright Mort Bay Consulting Pty Ltd unless otherwise noted. +It is licensed under the apache 2.0 license. + +jarjar is copyright Google inc unless otherwise noted. +It is licensed under the apache 2.0 license. + +guava is copyright Google inc unless otherwise noted. +It is licensed under the apache 2.0 license. + This product includes parquet-tools, initially developed at ARRIS, Inc. with the following copyright notice: diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt similarity index 100% rename from dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.txt rename to dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-api.txt diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-core.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff-runtime.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/dolphinscheduler-dist/release-docs/licenses/LICENSE-protostuff.collectionschema.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 3b562f557de1f9e0669f5670e3d11c671eb0339c Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Fri, 26 Feb 2021 20:28:56 +0800 Subject: [PATCH 53/68] [Fix-4886][Docker] Fix invalid volume specification in Windows --- docker/build/Dockerfile | 3 +- .../dolphinscheduler/common.properties.tpl | 2 +- ...ler_env.sh => dolphinscheduler_env.sh.tpl} | 18 ++++++------ docker/build/startup-init-conf.sh | 19 ++++++++++--- docker/docker-swarm/docker-compose.yml | 10 ++++++- docker/docker-swarm/docker-stack.yml | 18 ++++++------ docker/docker-swarm/dolphinscheduler_env.sh | 28 ------------------- 7 files changed, 45 insertions(+), 53 deletions(-) rename docker/build/conf/dolphinscheduler/env/{dolphinscheduler_env.sh => dolphinscheduler_env.sh.tpl} (72%) delete mode 100755 docker/docker-swarm/dolphinscheduler_env.sh diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile index 121970bf5a..eeea20d482 100644 --- a/docker/build/Dockerfile +++ b/docker/build/Dockerfile @@ -44,11 +44,10 @@ COPY ./startup-init-conf.sh /root/startup-init-conf.sh COPY ./startup.sh /root/startup.sh COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/ COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/ -COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/ +COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl /opt/dolphinscheduler/conf/env/ RUN dos2unix /root/checkpoint.sh && \ dos2unix /root/startup-init-conf.sh && \ dos2unix /root/startup.sh && \ - dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \ dos2unix /opt/dolphinscheduler/script/*.sh && \ dos2unix /opt/dolphinscheduler/bin/*.sh && \ rm -rf /bin/sh && \ diff --git a/docker/build/conf/dolphinscheduler/common.properties.tpl b/docker/build/conf/dolphinscheduler/common.properties.tpl index a3ccde7c61..36bc5b8de8 100644 --- a/docker/build/conf/dolphinscheduler/common.properties.tpl +++ b/docker/build/conf/dolphinscheduler/common.properties.tpl @@ -64,7 +64,7 @@ yarn.application.status.address=http://ds1:8088/ws/v1/cluster/apps/%s yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s # system env path, If you want to set your own path, you need to set this env file to an absolute path -dolphinscheduler.env.path=${DOLPHINSCHEDULER_ENV_PATH} +#dolphinscheduler.env.path=env/dolphinscheduler_env.sh development.state=false # kerberos tgt expire time, unit is hours diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl similarity index 72% rename from docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh rename to docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl index 7fd39335ae..b09e4972cd 100755 --- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh +++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh.tpl @@ -15,14 +15,14 @@ # limitations under the License. # -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/usr/bin/python -export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py +export HADOOP_HOME=$HADOOP_HOME +export HADOOP_CONF_DIR=$HADOOP_CONF_DIR +export SPARK_HOME1=$SPARK_HOME1 +export SPARK_HOME2=$SPARK_HOME2 +export PYTHON_HOME=$PYTHON_HOME +export JAVA_HOME=$JAVA_HOME +export HIVE_HOME=$HIVE_HOME +export FLINK_HOME=$FLINK_HOME +export DATAX_HOME=$DATAX_HOME export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh index 52004d7e6c..89be05eaf6 100755 --- a/docker/build/startup-init-conf.sh +++ b/docker/build/startup-init-conf.sh @@ -37,7 +37,17 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"} #============================================================================ # Common #============================================================================ -export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"} +# dolphinscheduler env +export HADOOP_HOME=${HADOOP_HOME:-"/opt/soft/hadoop"} +export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/opt/soft/hadoop/etc/hadoop"} +export SPARK_HOME1=${SPARK_HOME1:-"/opt/soft/spark1"} +export SPARK_HOME2=${SPARK_HOME2:-"/opt/soft/spark2"} +export PYTHON_HOME=${PYTHON_HOME:-"/usr/bin/python"} +export JAVA_HOME=${JAVA_HOME:-"/usr/lib/jvm/java-1.8-openjdk"} +export HIVE_HOME=${HIVE_HOME:-"/opt/soft/hive"} +export FLINK_HOME=${FLINK_HOME:-"/opt/soft/flink"} +export DATAX_HOME=${DATAX_HOME:-"/opt/soft/datax/bin/datax.py"} +# common env export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"} export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""} export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"} @@ -83,9 +93,10 @@ export ALERT_LISTEN_HOST=${ALERT_LISTEN_HOST:-"127.0.0.1"} export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"lib/plugin/alert"} echo "generate app config" -ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do +find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.tpl" | while read file; do eval "cat << EOF -$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line}) +$(cat ${file}) EOF -" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} +" > ${file%.*} done +find ${DOLPHINSCHEDULER_HOME}/conf/ -name "*.sh" -exec chmod +x {} \; diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 9c45e5b5df..a4a221c56e 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -162,6 +162,15 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + HADOOP_HOME: "/opt/soft/hadoop" + HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" + SPARK_HOME1: "/opt/soft/spark1" + SPARK_HOME2: "/opt/soft/spark2" + PYTHON_HOME: "/usr/bin/python" + JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" + HIVE_HOME: "/opt/soft/hive" + FLINK_HOME: "/opt/soft/flink" + DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql @@ -183,7 +192,6 @@ services: - dolphinscheduler-postgresql - dolphinscheduler-zookeeper volumes: - - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-resource-local:/dolphinscheduler diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index da5b8cba16..7206e4e678 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -156,6 +156,15 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + HADOOP_HOME: "/opt/soft/hadoop" + HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" + SPARK_HOME1: "/opt/soft/spark1" + SPARK_HOME2: "/opt/soft/spark2" + PYTHON_HOME: "/usr/bin/python" + JAVA_HOME: "/usr/lib/jvm/java-1.8-openjdk" + HIVE_HOME: "/opt/soft/hive" + FLINK_HOME: "/opt/soft/flink" + DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert DATABASE_HOST: dolphinscheduler-postgresql @@ -173,9 +182,6 @@ services: timeout: 5s retries: 3 start_period: 30s - configs: - - source: dolphinscheduler-worker-task-env - target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs @@ -193,8 +199,4 @@ volumes: dolphinscheduler-postgresql: dolphinscheduler-zookeeper: dolphinscheduler-worker-data: - dolphinscheduler-logs: - -configs: - dolphinscheduler-worker-task-env: - file: ./dolphinscheduler_env.sh \ No newline at end of file + dolphinscheduler-logs: \ No newline at end of file diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh deleted file mode 100755 index 7fd39335ae..0000000000 --- a/docker/docker-swarm/dolphinscheduler_env.sh +++ /dev/null @@ -1,28 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -export HADOOP_HOME=/opt/soft/hadoop -export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop -export SPARK_HOME1=/opt/soft/spark1 -export SPARK_HOME2=/opt/soft/spark2 -export PYTHON_HOME=/usr/bin/python -export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk -export HIVE_HOME=/opt/soft/hive -export FLINK_HOME=/opt/soft/flink -export DATAX_HOME=/opt/soft/datax/bin/datax.py - -export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH From 90e9d4141da08001c78b548ce6f9cb5e9599fd78 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Sat, 27 Feb 2021 01:43:20 +1000 Subject: [PATCH 54/68] chore: fix case of GitHub and JavaScript Changes are: - `Github` -> `GitHub` - `Javascript` -> `JavaScript` --- docker/build/README.md | 2 +- docker/build/README_zh_CN.md | 2 +- .../java/org/apache/dolphinscheduler/common/BrowserCommon.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index e13ff5c1e5..f1c952b4cb 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -2,7 +2,7 @@ Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. -Github URL: https://github.com/apache/incubator-dolphinscheduler +GitHub URL: https://github.com/apache/incubator-dolphinscheduler Official Website: https://dolphinscheduler.apache.org diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index d8d29efe28..993a27435e 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -2,7 +2,7 @@ 一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。 -Github URL: https://github.com/apache/incubator-dolphinscheduler +GitHub URL: https://github.com/apache/incubator-dolphinscheduler Official Website: https://dolphinscheduler.apache.org diff --git a/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java index 0740b8d323..6fafefc7da 100644 --- a/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java +++ b/e2e/src/test/java/org/apache/dolphinscheduler/common/BrowserCommon.java @@ -43,7 +43,7 @@ public class BrowserCommon { protected Actions actions; /** - * Javascript + * JavaScript */ protected JavascriptExecutor je; From 40031d8c6dd384d22639d66a21084f1dde2a8a96 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Sat, 27 Feb 2021 11:10:47 +1000 Subject: [PATCH 55/68] Fix Slack invite links --- README.md | 5 ++--- README_zh_CN.md | 10 +--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 248b9d77ce..ec20140d9f 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Dolphin Scheduler Official Website ### Design Features: -DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`. +DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`. Its main objectives are as follows: @@ -83,7 +83,7 @@ We would like to express our deep gratitude to all the open-source projects used ### Community You are so much welcomed to communicate with the developers and users of Dolphin Scheduler freely. There are two ways to find them: -1. Join the slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-l8k90ceu-wwUfobaDkJxjzMfZp4y1Ag). +1. Join the slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw). 2. Follow the [twitter account of Dolphin Scheduler](https://twitter.com/dolphinschedule) and get the latest news just on time. @@ -93,4 +93,3 @@ The community welcomes everyone to participate in contributing, please refer to ### License Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. - diff --git a/README_zh_CN.md b/README_zh_CN.md index 0e96da8371..a905827782 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -84,16 +84,8 @@ Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、g 2. 先订阅邮件开发列表:[订阅邮件列表](https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html), 订阅成功后发送邮件到dev@dolphinscheduler.apache.org. ### 社区 -1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-l8k90ceu-wwUfobaDkJxjzMfZp4y1Ag)加入slack channel +1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw)加入slack channel 2. 关注[Apache Dolphin Scheduler的Twitter账号](https://twitter.com/dolphinschedule)获取实时动态 ### 版权 请参考 [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) 文件. - - - - - - - - From 6911a39b491ef3ea65feffffe1fb4a6284e40475 Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Thu, 25 Feb 2021 18:44:21 +0800 Subject: [PATCH 56/68] [Improvement][Docker] Add FAQ in readme --- docker/build/README.md | 147 ++++++++++++++++++++++++- docker/build/README_zh_CN.md | 145 +++++++++++++++++++++++- docker/docker-swarm/docker-compose.yml | 12 ++ docker/docker-swarm/docker-stack.yml | 12 ++ 4 files changed, 309 insertions(+), 7 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index f1c952b4cb..a51038e8e5 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -1,12 +1,12 @@ -## What is Dolphin Scheduler? +## What is DolphinScheduler? -Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. +DolphinScheduler is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. GitHub URL: https://github.com/apache/incubator-dolphinscheduler Official Website: https://dolphinscheduler.apache.org -![Dolphin Scheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) +![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) @@ -118,7 +118,7 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script ## Environment Variables -The Dolphin Scheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. +The DolphinScheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. **`DATABASE_TYPE`** @@ -308,3 +308,142 @@ EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done ``` + +## FAQ + +### How to stop dolphinscheduler by docker-compose? + +Stop containers: + +``` +docker-compose stop +``` + +Stop containers and removes containers, networks and volumes: + +``` +docker-compose down -v +``` + +### How to deploy dolphinscheduler on Docker Swarm? + +Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)) + +Start a stack named dolphinscheduler + +``` +docker stack deploy -c docker-stack.yml dolphinscheduler +``` + +Stop and remove the stack named dolphinscheduler + +``` +docker stack rm dolphinscheduler +``` + +### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL? + +> Because of the commercial license, we cannot directly use the driver and client of MySQL. +> +> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) + +2. Create a new `Dockerfile` to add MySQL driver and client: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +RUN apk add --update --no-cache mysql-client +``` + +3. Build a new docker image including MySQL driver and client: + +``` +docker build -t apache/dolphinscheduler:mysql . +``` + +4. Modify all `image` fields to `apache/dolphinscheduler:mysql` in `docker-compose.yml` + +> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` + +5. Comment the `dolphinscheduler-postgresql` block in `docker-compose.yml` + +6. Add `dolphinscheduler-mysql` service in `docker-compose.yml` (**Optional**, you can directly use a external MySQL database) + +7. Modify all DATABASE environments in `docker-compose.yml` + +``` +DATABASE_TYPE: mysql +DATABASE_DRIVER: com.mysql.jdbc.Driver +DATABASE_HOST: dolphinscheduler-mysql +DATABASE_PORT: 3306 +DATABASE_USERNAME: root +DATABASE_PASSWORD: root +DATABASE_DATABASE: dolphinscheduler +DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8 +``` + +> If you have added `dolphinscheduler-mysql` service in `docker-compose.yml`, just set `DATABASE_HOST` to `dolphinscheduler-mysql` + +8. Run a dolphinscheduler (See **How to use this docker image**) + +### How to support MySQL datasource in `Datasource manage`? + +> Because of the commercial license, we cannot directly use the driver of MySQL. +> +> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) + +2. Create a new `Dockerfile` to add MySQL driver: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +``` + +3. Build a new docker image including MySQL driver: + +``` +docker build -t apache/dolphinscheduler:mysql-driver . +``` + +4. Modify all `image` fields to `apache/dolphinscheduler:mysql-driver` in `docker-compose.yml` + +> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` + +5. Run a dolphinscheduler (See **How to use this docker image**) + +6. Add a MySQL datasource in `Datasource manage` + +### How to support Oracle datasource in `Datasource manage`? + +> Because of the commercial license, we cannot directly use the driver of Oracle. +> +> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) + +2. Create a new `Dockerfile` to add Oracle driver: + +``` +FROM apache/dolphinscheduler:latest +COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib +``` + +3. Build a new docker image including Oracle driver: + +``` +docker build -t apache/dolphinscheduler:oracle-driver . +``` + +4. Modify all `image` fields to `apache/dolphinscheduler:oracle-driver` in `docker-compose.yml` + +> If you want to deploy dolphinscheduler on Docker Swarm, you need modify `docker-stack.yml` + +5. Run a dolphinscheduler (See **How to use this docker image**) + +6. Add a Oracle datasource in `Datasource manage` + +For more information please refer to the [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) documentation. diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 993a27435e..4e7cf58677 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -1,4 +1,4 @@ -## Dolphin Scheduler是什么? +## DolphinScheduler是什么? 一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。 @@ -6,7 +6,7 @@ GitHub URL: https://github.com/apache/incubator-dolphinscheduler Official Website: https://dolphinscheduler.apache.org -![Dolphin Scheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) +![DolphinScheduler](https://dolphinscheduler.apache.org/img/hlogo_colorful.svg) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) @@ -115,7 +115,7 @@ C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ## 环境变量 -Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 +DolphinScheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 **`DATABASE_TYPE`** @@ -305,3 +305,142 @@ EOF " > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*} done ``` + +## FAQ + +### 如何通过 docker-compose 停止 dolphinscheduler？ + +停止所有容器: + +``` +docker-compose stop +``` + +停止所有容器并移除所有容器，网络和存储卷: + +``` +docker-compose down -v +``` + +### 如何在 Docker Swarm 上部署 dolphinscheduler？ + +假设 Docker Swarm 集群已经部署（如果还没有创建 Docker Swarm 集群，请参考 [https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/） + +启动名为 dolphinscheduler 的 stack + +``` +docker stack deploy -c docker-stack.yml dolphinscheduler +``` + +启动并移除名为 dolphinscheduler 的 stack + +``` +docker stack rm dolphinscheduler +``` + +### 如何用 MySQL 替代 PostgreSQL 作为 DolphinScheduler 的数据库？ + +> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包和客户端. +> +> 如果你要使用 MySQL, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. + +1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`) + +2. 创建一个新的 `Dockerfile`，用于添加 MySQL 的驱动包和客户端: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +RUN apk add --update --no-cache mysql-client +``` + +3. 构建一个包含 MySQL 的驱动包和客户端的新镜像: + +``` +docker build -t apache/dolphinscheduler:mysql . +``` + +4. 修改 `docker-compose.yml` 文件中的所有 image 字段为 `apache/dolphinscheduler:mysql` + +> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` + +5. 注释 `docker-compose.yml` 文件中的 `dolphinscheduler-postgresql` 块 + +6. 在 `docker-compose.yml` 文件中添加 `dolphinscheduler-mysql` 服务（**可选**，你可以直接使用一个外部的 MySQL 数据库） + +7. 修改 `docker-compose.yml` 文件中的所有 DATABASE 环境变量 + +``` +DATABASE_TYPE: mysql +DATABASE_DRIVER: com.mysql.jdbc.Driver +DATABASE_HOST: dolphinscheduler-mysql +DATABASE_PORT: 3306 +DATABASE_USERNAME: root +DATABASE_PASSWORD: root +DATABASE_DATABASE: dolphinscheduler +DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8 +``` + +> 如果你已经添加了 `dolphinscheduler-mysql` 服务，设置 `DATABASE_HOST` 为 `dolphinscheduler-mysql` 即可 + +8. 运行 dolphinscheduler (详见**如何使用docker镜像**) + +### How to support MySQL datasource in `Datasource manage`？ + +> 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包. +> +> 如果你要添加 MySQL 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. + +1. 下载 MySQL 驱动包 [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (要求 `>=5.1.47`) + +2. 创建一个新的 `Dockerfile`，用于添加 MySQL 驱动包: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +``` + +3. 构建一个包含 MySQL 驱动包的新镜像: + +``` +docker build -t apache/dolphinscheduler:mysql-driver . +``` + +4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:mysql-driver` + +> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` + +5. 运行 dolphinscheduler (详见**如何使用docker镜像**) + +6. 在数据源中心添加一个 MySQL 数据源 + +### How to support Oracle datasource in `Datasource manage`？ + +> 由于商业许可证的原因，我们不能直接使用 Oracle 的驱动包. +> +> 如果你要添加 Oracle 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建. + +1. 下载 Oracle 驱动包 [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) + +2. 创建一个新的 `Dockerfile`，用于添加 Oracle 驱动包: + +``` +FROM apache/dolphinscheduler:latest +COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib +``` + +3. 构建一个包含 Oracle 驱动包的新镜像: + +``` +docker build -t apache/dolphinscheduler:oracle-driver . +``` + +4. 将 `docker-compose.yml` 文件中的所有 image 字段 修改为 `apache/dolphinscheduler:oracle-driver` + +> 如果你想在 Docker Swarm 上部署 dolphinscheduler，你需要修改 `docker-stack.yml` + +5. 运行 dolphinscheduler (详见**如何使用docker镜像**) + +6. 在数据源中心添加一个 Oracle 数据源 + +更多信息请查看 [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) 文档. diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index a4a221c56e..95818fbbf5 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -58,11 +58,14 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler @@ -92,11 +95,14 @@ services: environment: TZ: Asia/Shanghai ALERT_PLUGIN_DIR: lib/plugin/alert + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -126,11 +132,14 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -173,11 +182,14 @@ services: DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 7206e4e678..093059c30e 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -58,11 +58,14 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler @@ -89,11 +92,14 @@ services: environment: TZ: Asia/Shanghai ALERT_PLUGIN_DIR: lib/plugin/alert + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 healthcheck: test: ["CMD", "/root/checkpoint.sh", "AlertServer"] interval: 30s @@ -122,11 +128,14 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 healthcheck: test: ["CMD", "/root/checkpoint.sh", "MasterServer"] @@ -167,11 +176,14 @@ services: DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler ALERT_LISTEN_HOST: dolphinscheduler-alert + DATABASE_TYPE: postgresql + DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql DATABASE_PORT: 5432 DATABASE_USERNAME: root DATABASE_PASSWORD: root DATABASE_DATABASE: dolphinscheduler + DATABASE_PARAMS: characterEncoding=utf8 ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181 RESOURCE_STORAGE_TYPE: HDFS RESOURCE_UPLOAD_PATH: /dolphinscheduler From 27d7b436a1cc08f24f011efe3c19f84a1095e425 Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Fri, 26 Feb 2021 12:13:05 +0800 Subject: [PATCH 57/68] [Improvement][Docker] Add default login username and password in readme --- docker/build/README_zh_CN.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 4e7cf58677..61d963e673 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -24,6 +24,8 @@ $ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d 访问前端界面：http://192.168.xx.xx:12345/dolphinscheduler +默认的用户是`admin`，默认的密码是`dolphinscheduler123` + #### 或者通过环境变量 **`DATABASE_HOST`** **`DATABASE_PORT`** **`ZOOKEEPER_QUORUM`** 使用已存在的服务 你可以指定已经存在的 **`Postgres`** 和 **`Zookeeper`** 服务. 如下: From 5e58b3a6356ccc9347567c4f6d972f4128a033f2 Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Wed, 17 Feb 2021 02:53:36 +0800 Subject: [PATCH 58/68] [Improvement][K8s] Update outdated readme --- docker/kubernetes/dolphinscheduler/README.md | 122 +++++++++---------- 1 file changed, 58 insertions(+), 64 deletions(-) diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 318c3a9132..abdfabca42 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -42,13 +42,15 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | Parameter | Description | Default | | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | +| `nameOverride` | String to partially override common.names.fullname | `nil` | +| `fullnameOverride` | String to fully override common.names.fullname | `nil` | | `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` | | `image.registry` | Docker image registry for the Dolphins Scheduler | `docker.io` | | `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | | `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | -| `image.imagePullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | -| `image.pullSecres` | PullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | -| | | | +| `image.pullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | +| `image.pullSecrets` | Image pull secrets. An optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | +| | | | | `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | | `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | | `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | @@ -56,41 +58,46 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database type will use it. | `postgresql` | -| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database driver will use it. | `org.postgresql.Driver` | -| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database host will use it. | `localhost` | -| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database port will use it. | `5432` | -| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database username will use it. | `root` | -| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database password will use it. | `root` | -| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database database will use it. | `dolphinscheduler` | -| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enable` value to false. Dolphins Scheduler's database params will use it. | `characterEncoding=utf8` | -| | | | +| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database type will use it | `postgresql` | +| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database driver will use it | `org.postgresql.Driver` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database host will use it | `localhost` | +| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database port will use it | `5432` | +| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database username will use it | `root` | +| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database password will use it | `root` | +| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database database will use it | `dolphinscheduler` | +| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database params will use it | `characterEncoding=utf8` | +| | | | | `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | -| `zookeeper.taskQueue` | Specify task queue for `master` and `worker` | `zookeeper` | +| `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` | +| `zookeeper.service.port` | ZooKeeper port | `2181` | | `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | | `zookeeper.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `zookeeper.persistence.storageClass` | Zookeeper data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalZookeeper.taskQueue` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify task queue for `master` and `worker` | `zookeeper` | -| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | -| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enable` value to false. Specify Zookeeper root path for `master` and `worker` | `dolphinscheduler` | -| | | | -| `common.configmap.DOLPHINSCHEDULER_ENV_PATH` | Extra env file path. | `/tmp/dolphinscheduler/env` | -| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | File uploaded path of DS. | `/tmp/dolphinscheduler/files` | -| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource Storate type, support type are: S3、HDFS、NONE. | `NONE` | -| `common.configmap.RESOURCE_UPLOAD_PATH` | The base path of resource. | `/ds` | -| `common.configmap.FS_DEFAULT_FS` | The default fs of resource, for s3 is the `s3a` prefix and bucket name. | `s3a://xxxx` | -| `common.configmap.FS_S3A_ENDPOINT` | If the resource type is `S3`, you should fill this filed, it's the endpoint of s3. | `s3.xxx.amazonaws.com` | -| `common.configmap.FS_S3A_ACCESS_KEY` | The access key for your s3 bucket. | `xxxxxxx` | -| `common.configmap.FS_S3A_SECRET_KEY` | The secret key for your s3 bucket. | `xxxxxxx` | +| `zookeeper.zookeeperRoot` | Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` | +| `externalZookeeper.zookeeperQuorum` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify Zookeeper quorum | `127.0.0.1:2181` | +| `externalZookeeper.zookeeperRoot` | If exists external Zookeeper, and set `zookeeper.enabled` value to false. Specify dolphinscheduler root directory in Zookeeper | `/dolphinscheduler` | +| | | | +| `common.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` | +| `common.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` | +| `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` | +| `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` | +| `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` | +| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `s3.xxx.amazonaws.com` | +| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `xxxxxxx` | +| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `xxxxxxx` | +| `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` | +| `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` | +| `common.fsFileResourcePersistence.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` | +| `common.fsFileResourcePersistence.storage` | `PersistentVolumeClaim` Size | `20Gi` | +| | | | | `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | -| | | | | `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `master.annotations` | The `annotations` for master server | `{}` | +| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | | `master.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `master.tolerations` | If specified, the pod's tolerations | `{}` | -| `master.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `master.jvmOptions` | The JVM options for master server. | `""` | -| `master.resources` | The `resource` limit and request config for master server. | `{}` | -| `master.annotations` | The `annotations` for master server. | `{}` | +| `master.resources` | The `resource` limit and request config for master server | `{}` | +| `master.configmap.DOLPHINSCHEDULER_OPTS` | The java options for master server | `""` | | `master.configmap.MASTER_EXEC_THREADS` | Master execute thread num | `100` | | `master.configmap.MASTER_EXEC_TASK_NUM` | Master execute task number in parallel | `20` | | `master.configmap.MASTER_HEARTBEAT_INTERVAL` | Master heartbeat interval | `10` | @@ -98,6 +105,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `master.configmap.MASTER_TASK_COMMIT_INTERVAL` | Master commit task interval | `1000` | | `master.configmap.MASTER_MAX_CPULOAD_AVG` | Only less than cpu avg load, master server can work. default value : the number of cpu cores * 2 | `100` | | `master.configmap.MASTER_RESERVED_MEMORY` | Only larger than reserved memory, master server can work. default value : physical memory * 1/10, unit is G | `0.1` | +| `master.configmap.MASTER_LISTEN_PORT` | Master listen port | `5678` | | `master.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `master.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `master.livenessProbe.periodSeconds` | How often to perform the probe | `30` | @@ -114,22 +122,22 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `master.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `master.persistentVolumeClaim.storageClassName` | `Master` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `master.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | | `worker.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` | | `worker.replicas` | Replicas is the desired number of replicas of the given Template | `3` | +| `worker.annotations` | The `annotations` for worker server | `{}` | +| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | | `worker.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `worker.tolerations` | If specified, the pod's tolerations | `{}` | -| `worker.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `worker.jvmOptions` | The JVM options for worker server. | `""` | -| `worker.resources` | The `resource` limit and request config for worker server. | `{}` | -| `worker.annotations` | The `annotations` for worker server. | `{}` | +| `worker.resources` | The `resource` limit and request config for worker server | `{}` | +| `worker.configmap.DOLPHINSCHEDULER_OPTS` | The java options for worker server | `""` | | `worker.configmap.WORKER_EXEC_THREADS` | Worker execute thread num | `100` | | `worker.configmap.WORKER_HEARTBEAT_INTERVAL` | Worker heartbeat interval | `10` | -| `worker.configmap.WORKER_FETCH_TASK_NUM` | Submit the number of tasks at a time | `3` | | `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` | | `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | -| `worker.configmap.DOLPHINSCHEDULER_DATA_BASEDIR_PATH` | User data directory path, self configuration, please make sure the directory exists and have read write permissions | `/tmp/dolphinscheduler` | -| `worker.configmap.DOLPHINSCHEDULER_ENV` | System env path, self configuration, please read `values.yaml` | `[]` | +| `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` | +| `worker.configmap.WORKER_GROUP` | Worker group | `default` | +| `worker.configmap.WORKER_WEIGHT` | Worker weight | `100` | | `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `worker.livenessProbe.periodSeconds` | How often to perform the probe | `30` | @@ -151,32 +159,18 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `worker.persistentVolumeClaim.logsPersistentVolume.accessModes` | `PersistentVolumeClaim` Access Modes | `[ReadWriteOnce]` | | `worker.persistentVolumeClaim.logsPersistentVolume.storageClassName` | `Worker` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `worker.persistentVolumeClaim.logsPersistentVolume.storage` | `PersistentVolumeClaim` Size | `20Gi` | -| | | | +| | | | +| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` | | `alert.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | | `alert.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | | `alert.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `alert.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `alert.annotations` | The `annotations` for alert server | `{}` | +| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | | `alert.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `alert.tolerations` | If specified, the pod's tolerations | `{}` | -| `alert.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `alert.jvmOptions` | The JVM options for alert server. | `""` | -| `alert.resources` | The `resource` limit and request config for alert server. | `{}` | -| `alert.annotations` | The `annotations` for alert server. | `{}` | -| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin path. | `/opt/dolphinscheduler/alert/plugin` | -| `alert.configmap.XLS_FILE_PATH` | XLS file path | `/tmp/xls` | -| `alert.configmap.MAIL_SERVER_HOST` | Mail `SERVER HOST ` | `nil` | -| `alert.configmap.MAIL_SERVER_PORT` | Mail `SERVER PORT` | `nil` | -| `alert.configmap.MAIL_SENDER` | Mail `SENDER` | `nil` | -| `alert.configmap.MAIL_USER` | Mail `USER` | `nil` | -| `alert.configmap.MAIL_PASSWD` | Mail `PASSWORD` | `nil` | -| `alert.configmap.MAIL_SMTP_STARTTLS_ENABLE` | Mail `SMTP STARTTLS` enable | `false` | -| `alert.configmap.MAIL_SMTP_SSL_ENABLE` | Mail `SMTP SSL` enable | `false` | -| `alert.configmap.MAIL_SMTP_SSL_TRUST` | Mail `SMTP SSL TRUST` | `nil` | -| `alert.configmap.ENTERPRISE_WECHAT_ENABLE` | `Enterprise Wechat` enable | `false` | -| `alert.configmap.ENTERPRISE_WECHAT_CORP_ID` | `Enterprise Wechat` corp id | `nil` | -| `alert.configmap.ENTERPRISE_WECHAT_SECRET` | `Enterprise Wechat` secret | `nil` | -| `alert.configmap.ENTERPRISE_WECHAT_AGENT_ID` | `Enterprise Wechat` agent id | `nil` | -| `alert.configmap.ENTERPRISE_WECHAT_USERS` | `Enterprise Wechat` users | `nil` | +| `alert.resources` | The `resource` limit and request config for alert server | `{}` | +| `alert.configmap.DOLPHINSCHEDULER_OPTS` | The java options for alert server | `""` | +| `alert.configmap.ALERT_PLUGIN_DIR` | Alert plugin directory | `lib/plugin/alert` | | `alert.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `alert.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `alert.livenessProbe.periodSeconds` | How often to perform the probe | `30` | @@ -194,16 +188,16 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `alert.persistentVolumeClaim.storageClassName` | `Alert` logs data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | | `alert.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` | | | | | +| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` | | `api.strategy.type` | Type of deployment. Can be "Recreate" or "RollingUpdate" | `RollingUpdate` | | `api.strategy.rollingUpdate.maxSurge` | The maximum number of pods that can be scheduled above the desired number of pods | `25%` | | `api.strategy.rollingUpdate.maxUnavailable` | The maximum number of pods that can be unavailable during the update | `25%` | -| `api.replicas` | Replicas is the desired number of replicas of the given Template | `1` | +| `api.annotations` | The `annotations` for api server | `{}` | +| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | | `api.nodeSelector` | NodeSelector is a selector which must be true for the pod to fit on a node | `{}` | | `api.tolerations` | If specified, the pod's tolerations | `{}` | -| `api.affinity` | If specified, the pod's scheduling constraints | `{}` | -| `api.jvmOptions` | The JVM options for api server. | `""` | -| `api.resources` | The `resource` limit and request config for api server. | `{}` | -| `api.annotations` | The `annotations` for api server. | `{}` | +| `api.resources` | The `resource` limit and request config for api server | `{}` | +| `api.configmap.DOLPHINSCHEDULER_OPTS` | The java options for api server | `""` | | `api.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `api.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | | `api.livenessProbe.periodSeconds` | How often to perform the probe | `30` | From 83ea710a5eb4d457338e84362284e981bf5a469b Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Thu, 25 Feb 2021 16:55:17 +0800 Subject: [PATCH 59/68] [Improvement][K8s] Add FAQ in readme --- docker/kubernetes/dolphinscheduler/README.md | 116 ++++++++++++++++++- 1 file changed, 110 insertions(+), 6 deletions(-) diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index abdfabca42..1a8f13aeaa 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -1,9 +1,9 @@ -# Dolphin Scheduler +# DolphinScheduler -[Dolphin Scheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. +[DolphinScheduler](https://dolphinscheduler.apache.org) is a distributed and easy-to-expand visual DAG workflow scheduling system, dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing. ## Introduction -This chart bootstraps a [Dolphin Scheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. +This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org) distributed deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. ## Prerequisites @@ -22,7 +22,7 @@ $ helm repo add bitnami https://charts.bitnami.com/bitnami $ helm dependency update . $ helm install dolphinscheduler . ``` -These commands deploy Dolphin Scheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. +These commands deploy DolphinScheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. > **Tip**: List all releases using `helm list` @@ -67,7 +67,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database database will use it | `dolphinscheduler` | | `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database params will use it | `characterEncoding=utf8` | | | | | -| `zookeeper.enabled` | If not exists external Zookeeper, by default, the Dolphin Scheduler will use a internal Zookeeper | `true` | +| `zookeeper.enabled` | If not exists external Zookeeper, by default, the DolphinScheduler will use a internal Zookeeper | `true` | | `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` | | `zookeeper.service.port` | ZooKeeper port | `2181` | | `zookeeper.persistence.enabled` | Set `zookeeper.persistence.enabled` to `true` to mount a new volume for internal Zookeeper | `false` | @@ -222,4 +222,108 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `ingress.tls.hosts` | Ingress tls hosts | `dolphinscheduler.org` | | `ingress.tls.secretName` | Ingress tls secret name | `dolphinscheduler-tls` | -For more information please refer to the [chart](https://github.com/apache/incubator-dolphinscheduler.git) documentation. +## FAQ + +### How to use MySQL as the DolphinScheduler's database instead of PostgreSQL? + +> Because of the commercial license, we cannot directly use the driver and client of MySQL. +> +> If you want to use MySQL, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) + +2. Create a new `Dockerfile` to add MySQL driver and client: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +RUN apk add --update --no-cache mysql-client +``` + +3. Build a new docker image including MySQL driver and client: + +``` +docker build -t apache/dolphinscheduler:mysql . +``` + +4. Push the docker image `apache/dolphinscheduler:mysql` to a docker registry + +5. Modify image `registry` and `repository`, and update `tag` to `mysql` in `values.yaml` + +6. Modify postgresql `enabled` to `false` + +7. Modify externalDatabase (especially modify `host`, `username` and `password`): + +``` +externalDatabase: + type: "mysql" + driver: "com.mysql.jdbc.Driver" + host: "localhost" + port: "3306" + username: "root" + password: "root" + database: "dolphinscheduler" + params: "useUnicode=true&characterEncoding=UTF-8" +``` + +8. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) + +### How to support MySQL datasource in `Datasource manage`? + +> Because of the commercial license, we cannot directly use the driver of MySQL. +> +> If you want to add MySQL datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the MySQL driver [mysql-connector-java-5.1.49.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.49/mysql-connector-java-5.1.49.jar) (require `>=5.1.47`) + +2. Create a new `Dockerfile` to add MySQL driver: + +``` +FROM apache/dolphinscheduler:latest +COPY mysql-connector-java-5.1.49.jar /opt/dolphinscheduler/lib +``` + +3. Build a new docker image including MySQL driver: + +``` +docker build -t apache/dolphinscheduler:mysql-driver . +``` + +4. Push the docker image `apache/dolphinscheduler:mysql-driver` to a docker registry + +5. Modify image `registry` and `repository`, and update `tag` to `mysql-driver` in `values.yaml` + +6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) + +7. Add a MySQL datasource in `Datasource manage` + +### How to support Oracle datasource in `Datasource manage`? + +> Because of the commercial license, we cannot directly use the driver of Oracle. +> +> If you want to add Oracle datasource, you can build a new image based on the `apache/dolphinscheduler` image as follows. + +1. Download the Oracle driver [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`) + +2. Create a new `Dockerfile` to add Oracle driver: + +``` +FROM apache/dolphinscheduler:latest +COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib +``` + +3. Build a new docker image including Oracle driver: + +``` +docker build -t apache/dolphinscheduler:oracle-driver . +``` + +4. Push the docker image `apache/dolphinscheduler:oracle-driver` to a docker registry + +5. Modify image `registry` and `repository`, and update `tag` to `oracle-driver` in `values.yaml` + +6. Run a DolphinScheduler release in Kubernetes (See **Installing the Chart**) + +7. Add a Oracle datasource in `Datasource manage` + +For more information please refer to the [incubator-dolphinscheduler](https://github.com/apache/incubator-dolphinscheduler.git) documentation. From c20091173383bb24bba019a6b109868624ba94fe Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Fri, 26 Feb 2021 17:46:08 +0800 Subject: [PATCH 60/68] [Improvement][Docker] Rename worker group to worker groups --- docker/build/README.md | 4 ++-- docker/build/conf/dolphinscheduler/worker.properties.tpl | 2 +- docker/kubernetes/dolphinscheduler/README.md | 2 +- dolphinscheduler-server/src/main/resources/worker.properties | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index a51038e8e5..291db1f9a6 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -268,7 +268,7 @@ This environment variable sets port for `worker-server`. The default value is `1 **`WORKER_GROUPS`** -This environment variable sets group for `worker-server`. The default value is `default`. +This environment variable sets groups for `worker-server`. The default value is `default`. **`WORKER_WEIGHT`** @@ -319,7 +319,7 @@ Stop containers: docker-compose stop ``` -Stop containers and removes containers, networks and volumes: +Stop containers and remove containers, networks and volumes: ``` docker-compose down -v diff --git a/docker/build/conf/dolphinscheduler/worker.properties.tpl b/docker/build/conf/dolphinscheduler/worker.properties.tpl index d3ef35a813..cab729b6aa 100644 --- a/docker/build/conf/dolphinscheduler/worker.properties.tpl +++ b/docker/build/conf/dolphinscheduler/worker.properties.tpl @@ -30,7 +30,7 @@ worker.reserved.memory=${WORKER_RESERVED_MEMORY} # worker listener port worker.listen.port=${WORKER_LISTEN_PORT} -# default worker group +# default worker groups worker.groups=${WORKER_GROUPS} # default worker weight diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 1a8f13aeaa..07d4f5b0d8 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -136,7 +136,7 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `worker.configmap.WORKER_MAX_CPULOAD_AVG` | Only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2 | `100` | | `worker.configmap.WORKER_RESERVED_MEMORY` | Only larger than reserved memory, worker server can work. default value : physical memory * 1/10, unit is G | `0.1` | | `worker.configmap.WORKER_LISTEN_PORT` | Worker listen port | `1234` | -| `worker.configmap.WORKER_GROUP` | Worker group | `default` | +| `worker.configmap.WORKER_GROUPS` | Worker groups | `default` | | `worker.configmap.WORKER_WEIGHT` | Worker weight | `100` | | `worker.livenessProbe.enabled` | Turn on and off liveness probe | `true` | | `worker.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `30` | diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties index 5fdbf1d910..fd249e26bb 100644 --- a/dolphinscheduler-server/src/main/resources/worker.properties +++ b/dolphinscheduler-server/src/main/resources/worker.properties @@ -30,7 +30,7 @@ # worker listener port #worker.listen.port=1234 -# default worker group +# default worker groups #worker.groups=default # default worker weight From 3ff4e210691a9526e7da48ec9b5c92475fe949b0 Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Fri, 26 Feb 2021 19:04:45 +0800 Subject: [PATCH 61/68] [Improvement][K8s] Add readme to use namespace, access ui and delete pv --- docker/build/README.md | 2 +- docker/build/README_zh_CN.md | 1 + docker/kubernetes/dolphinscheduler/README.md | 48 +++++++++++++++++--- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index 291db1f9a6..dd567ba9b9 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -23,7 +23,7 @@ The default **postgres** user `root`, postgres password `root` and database `dol The default **zookeeper** is created in the `docker-compose.yml`. -Access the Web UI：http://192.168.xx.xx:12345/dolphinscheduler +Access the Web UI: http://192.168.xx.xx:12345/dolphinscheduler The default username is `admin` and the default password is `dolphinscheduler123` diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 61d963e673..6286559970 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -14,6 +14,7 @@ Official Website: https://dolphinscheduler.apache.org ## 如何使用docker镜像 #### 以 docker-compose 的方式启动dolphinscheduler(推荐) + ``` $ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d ``` diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 07d4f5b0d8..2fc4aee9c3 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -7,8 +7,8 @@ This chart bootstraps a [DolphinScheduler](https://dolphinscheduler.apache.org) ## Prerequisites -- Helm 3.1.0+ -- Kubernetes 1.12+ +- [Helm](https://helm.sh/) 3.1.0+ +- [Kubernetes](https://kubernetes.io/) 1.12+ - PV provisioner support in the underlying infrastructure ## Installing the Chart @@ -22,10 +22,38 @@ $ helm repo add bitnami https://charts.bitnami.com/bitnami $ helm dependency update . $ helm install dolphinscheduler . ``` + +To install the chart with a namespace named `test`: + +```bash +$ helm install dolphinscheduler . -n test +``` + +> **Tip**: If a namespace named `test` is used, the option `-n test` needs to be added to the `helm` and `kubectl` command + These commands deploy DolphinScheduler on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. > **Tip**: List all releases using `helm list` +## Access DolphinScheduler UI + +If `ingress.enabled` in `values.yaml` is set to `true`, you just access `http://${ingress.host}/dolphinscheduler` in browser. + +> **Tip**: If there is a problem with ingress access, please contact the Kubernetes administrator and refer to the [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) + +Otherwise, you need to execute port-forward command like: + +```bash +$ kubectl port-forward --address 0.0.0.0 svc/dolphinscheduler-api 12345:12345 +$ kubectl port-forward --address 0.0.0.0 -n test svc/dolphinscheduler-api 12345:12345 # with test namespace +``` + +> **Tip**: If the error of `unable to do port forwarding: socat not found` appears, you need to install `socat` at first + +And then access the web: http://192.168.xx.xx:12345/dolphinscheduler + +The default username is `admin` and the default password is `dolphinscheduler123` + ## Uninstalling the Chart To uninstall/delete the `dolphinscheduler` deployment: @@ -34,7 +62,15 @@ To uninstall/delete the `dolphinscheduler` deployment: $ helm uninstall dolphinscheduler ``` -The command removes all the Kubernetes components associated with the chart and deletes the release. +The command removes all the Kubernetes components but PVC's associated with the chart and deletes the release. + +To delete the PVC's associated with `dolphinscheduler`: + +```bash +$ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler +``` + +> **Note**: Deleting the PVC's will delete all data as well. Please be cautious before doing it. ## Configuration @@ -82,9 +118,9 @@ The following tables lists the configurable parameters of the Dolphins Scheduler | `common.configmap.RESOURCE_STORAGE_TYPE` | Resource storage type: HDFS, S3, NONE | `HDFS` | | `common.configmap.RESOURCE_UPLOAD_PATH` | Resource store on HDFS/S3 path, please make sure the directory exists on hdfs and have read write permissions | `/dolphinscheduler` | | `common.configmap.FS_DEFAULT_FS` | Resource storage file system like `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler` | `file:///` | -| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `s3.xxx.amazonaws.com` | -| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `xxxxxxx` | -| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is seted to `S3` | `xxxxxxx` | +| `common.configmap.FS_S3A_ENDPOINT` | S3 endpoint when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `s3.xxx.amazonaws.com` | +| `common.configmap.FS_S3A_ACCESS_KEY` | S3 access key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` | +| `common.configmap.FS_S3A_SECRET_KEY` | S3 secret key when `common.configmap.RESOURCE_STORAGE_TYPE` is set to `S3` | `xxxxxxx` | | `common.fsFileResourcePersistence.enabled` | Set `common.fsFileResourcePersistence.enabled` to `true` to mount a new file resource volume for `api` and `worker` | `false` | | `common.fsFileResourcePersistence.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` | | `common.fsFileResourcePersistence.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` | From 4cab4095e9bdfcb292f34a6ac1695ac9ef38de3c Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Fri, 26 Feb 2021 20:33:47 +0800 Subject: [PATCH 62/68] [Improvement][Docker] Update environment variables --- docker/build/README.md | 36 ++++++++++++++++++++++++-- docker/build/README_zh_CN.md | 36 ++++++++++++++++++++++++-- docker/docker-swarm/docker-compose.yml | 1 + docker/docker-swarm/docker-stack.yml | 1 + 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index dd567ba9b9..ebe94f2ea0 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -168,9 +168,41 @@ This environment variable sets the database for database. The default value is ` **Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`. -**`DOLPHINSCHEDULER_ENV_PATH`** +**`HADOOP_HOME`** -This environment variable sets the runtime environment for task. The default value is `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`. +This environment variable sets `HADOOP_HOME`. The default value is `/opt/soft/hadoop`. + +**`HADOOP_CONF_DIR`** + +This environment variable sets `HADOOP_CONF_DIR`. The default value is `/opt/soft/hadoop/etc/hadoop`. + +**`SPARK_HOME1`** + +This environment variable sets `SPARK_HOME1`. The default value is `/opt/soft/spark1`. + +**`SPARK_HOME2`** + +This environment variable sets `SPARK_HOME2`. The default value is `/opt/soft/spark2`. + +**`PYTHON_HOME`** + +This environment variable sets `PYTHON_HOME`. The default value is `/usr/bin/python`. + +**`JAVA_HOME`** + +This environment variable sets `JAVA_HOME`. The default value is `/usr/lib/jvm/java-1.8-openjdk`. + +**`HIVE_HOME`** + +This environment variable sets `HIVE_HOME`. The default value is `/opt/soft/hive`. + +**`FLINK_HOME`** + +This environment variable sets `FLINK_HOME`. The default value is `/opt/soft/flink`. + +**`DATAX_HOME`** + +This environment variable sets `DATAX_HOME`. The default value is `/opt/soft/datax/bin/datax.py`. **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 6286559970..9e42e955ce 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -168,9 +168,41 @@ DolphinScheduler映像使用了几个容易遗漏的环境变量。虽然这些 **注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时，必须指定这个环境变量，以便于你更好的搭建分布式服务。 -**`DOLPHINSCHEDULER_ENV_PATH`** +**`HADOOP_HOME`** -任务执行时的环境变量配置文件， 默认值 `/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。 +配置`dolphinscheduler`的`HADOOP_HOME`，默认值 `/opt/soft/hadoop`。 + +**`HADOOP_CONF_DIR`** + +配置`dolphinscheduler`的`HADOOP_CONF_DIR`，默认值 `/opt/soft/hadoop/etc/hadoop`。 + +**`SPARK_HOME1`** + +配置`dolphinscheduler`的`SPARK_HOME1`，默认值 `/opt/soft/spark1`。 + +**`SPARK_HOME2`** + +配置`dolphinscheduler`的`SPARK_HOME2`，默认值 `/opt/soft/spark2`。 + +**`PYTHON_HOME`** + +配置`dolphinscheduler`的`PYTHON_HOME`，默认值 `/usr/bin/python`。 + +**`JAVA_HOME`** + +配置`dolphinscheduler`的`JAVA_HOME`，默认值 `/usr/lib/jvm/java-1。8-openjdk`。 + +**`HIVE_HOME`** + +配置`dolphinscheduler`的`HIVE_HOME`，默认值 `/opt/soft/hive`。 + +**`FLINK_HOME`** + +配置`dolphinscheduler`的`FLINK_HOME`，默认值 `/opt/soft/flink`。 + +**`DATAX_HOME`** + +配置`dolphinscheduler`的`DATAX_HOME`，默认值 `/opt/soft/datax/bin/datax。py`。 **`DOLPHINSCHEDULER_DATA_BASEDIR_PATH`** diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 95818fbbf5..190b245ca0 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -132,6 +132,7 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index 093059c30e..e35ed64d6e 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -128,6 +128,7 @@ services: MASTER_TASK_COMMIT_INTERVAL: "1000" MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" + DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql From da6e1500cbd950105eed5e4219a700cc96045cff Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Sat, 27 Feb 2021 14:33:48 +0800 Subject: [PATCH 63/68] [Improvement][Docker] Add DOLPHINSCHEDULER_OPTS for docker compose/swarm --- docker/docker-swarm/docker-compose.yml | 6 +++++- docker/docker-swarm/docker-stack.yml | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml index 190b245ca0..01ac4bfb52 100644 --- a/docker/docker-swarm/docker-compose.yml +++ b/docker/docker-swarm/docker-compose.yml @@ -58,6 +58,7 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai + DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -95,6 +96,7 @@ services: environment: TZ: Asia/Shanghai ALERT_PLUGIN_DIR: lib/plugin/alert + DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -133,6 +135,7 @@ services: MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -172,6 +175,7 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + ALERT_LISTEN_HOST: dolphinscheduler-alert HADOOP_HOME: "/opt/soft/hadoop" HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" SPARK_HOME1: "/opt/soft/spark1" @@ -182,7 +186,7 @@ services: FLINK_HOME: "/opt/soft/flink" DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - ALERT_LISTEN_HOST: dolphinscheduler-alert + DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml index e35ed64d6e..4a34b37916 100644 --- a/docker/docker-swarm/docker-stack.yml +++ b/docker/docker-swarm/docker-stack.yml @@ -58,6 +58,7 @@ services: - 12345:12345 environment: TZ: Asia/Shanghai + DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -92,6 +93,7 @@ services: environment: TZ: Asia/Shanghai ALERT_PLUGIN_DIR: lib/plugin/alert + DOLPHINSCHEDULER_OPTS: "-Xms512m -Xmx512m -Xmn256m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -129,6 +131,7 @@ services: MASTER_MAX_CPULOAD_AVG: "100" MASTER_RESERVED_MEMORY: "0.1" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler + DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql @@ -166,6 +169,7 @@ services: WORKER_RESERVED_MEMORY: "0.1" WORKER_GROUPS: "default" WORKER_WEIGHT: "100" + ALERT_LISTEN_HOST: dolphinscheduler-alert HADOOP_HOME: "/opt/soft/hadoop" HADOOP_CONF_DIR: "/opt/soft/hadoop/etc/hadoop" SPARK_HOME1: "/opt/soft/spark1" @@ -176,7 +180,7 @@ services: FLINK_HOME: "/opt/soft/flink" DATAX_HOME: "/opt/soft/datax/bin/datax.py" DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler - ALERT_LISTEN_HOST: dolphinscheduler-alert + DOLPHINSCHEDULER_OPTS: "-Xms1g -Xmx1g -Xmn512m" DATABASE_TYPE: postgresql DATABASE_DRIVER: org.postgresql.Driver DATABASE_HOST: dolphinscheduler-postgresql From d170b92dc6ab5deda8ab88d5eede1e8d642ee158 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Sat, 27 Feb 2021 16:46:18 +1000 Subject: [PATCH 64/68] Update the url for `QuickStart in Docker` (#4895) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ec20140d9f..a99a20e354 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Overload processing: By using the task queue mechanism, the number of schedulabl ![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png) ### QuickStart in Docker -Please referer the official website document:[[QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/1.3.4/user_doc/docker-deployment.html)] +Please referer the official website document:[[QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/1.3.5/user_doc/docker-deployment.html)] ### How to Build From 5adf1db0274f9d9ed6b21166c7b3b6d35d9d6052 Mon Sep 17 00:00:00 2001 From: chengshiwen Date: Sat, 27 Feb 2021 15:35:44 +0800 Subject: [PATCH 65/68] [Improvement][Docker] Update readme --- docker/build/README.md | 6 ++--- docker/build/README_zh_CN.md | 14 +++++------ docker/kubernetes/dolphinscheduler/README.md | 26 ++++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docker/build/README.md b/docker/build/README.md index ebe94f2ea0..6612307445 100644 --- a/docker/build/README.md +++ b/docker/build/README.md @@ -62,7 +62,7 @@ $ docker run -d --name dolphinscheduler-master \ apache/dolphinscheduler:latest master-server ``` -* Start a **worker server**, For example: +* Start a **worker server** (including **logger server**), For example: ``` $ docker run -d --name dolphinscheduler-worker \ @@ -118,7 +118,7 @@ Please read `./docker/build/hooks/build` `./docker/build/hooks/build.bat` script ## Environment Variables -The DolphinScheduler image uses several environment variables which are easy to miss. While none of the variables are required, they may significantly aid you in using the image. +The DolphinScheduler Docker container is configured through environment variables, and the default value will be used if an environment variable is not set. **`DATABASE_TYPE`** @@ -359,7 +359,7 @@ docker-compose down -v ### How to deploy dolphinscheduler on Docker Swarm? -Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)) +Assuming that the Docker Swarm cluster has been created (If there is no Docker Swarm cluster, please refer to [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)) Start a stack named dolphinscheduler diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md index 9e42e955ce..311c53c02d 100644 --- a/docker/build/README_zh_CN.md +++ b/docker/build/README_zh_CN.md @@ -23,7 +23,7 @@ $ docker-compose -f ./docker/docker-swarm/docker-compose.yml up -d 同时，默认的`Zookeeper`也会在`docker-compose.yml`文件中被创建。 -访问前端界面：http://192.168.xx.xx:12345/dolphinscheduler +访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler 默认的用户是`admin`，默认的密码是`dolphinscheduler123` @@ -40,7 +40,7 @@ $ docker run -d --name dolphinscheduler \ apache/dolphinscheduler:latest all ``` -访问前端界面：http://192.168.xx.xx:12345/dolphinscheduler +访问前端页面：http://192.168.xx.xx:12345/dolphinscheduler #### 或者运行dolphinscheduler中的部分服务 @@ -62,7 +62,7 @@ $ docker run -d --name dolphinscheduler-master \ apache/dolphinscheduler:latest master-server ``` -* 启动一个 **worker server**, 如下: +* 启动一个 **worker server** (包括 **logger server**), 如下: ``` $ docker run -d --name dolphinscheduler-worker \ @@ -118,7 +118,7 @@ C:\incubator-dolphinscheduler>.\docker\build\hooks\build.bat ## 环境变量 -DolphinScheduler映像使用了几个容易遗漏的环境变量。虽然这些变量不是必须的，但是可以帮助你更容易配置镜像并根据你的需求定义相应的服务配置。 +DolphinScheduler Docker 容器通过环境变量进行配置，缺省时将会使用默认值 **`DATABASE_TYPE`** @@ -359,7 +359,7 @@ docker-compose down -v ### 如何在 Docker Swarm 上部署 dolphinscheduler？ -假设 Docker Swarm 集群已经部署（如果还没有创建 Docker Swarm 集群，请参考 [https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/） +假设 Docker Swarm 集群已经部署（如果还没有创建 Docker Swarm 集群，请参考 [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/)） 启动名为 dolphinscheduler 的 stack @@ -420,7 +420,7 @@ DATABASE_PARAMS: useUnicode=true&characterEncoding=UTF-8 8. 运行 dolphinscheduler (详见**如何使用docker镜像**) -### How to support MySQL datasource in `Datasource manage`？ +### 如何在数据源中心支持 MySQL 数据源？ > 由于商业许可证的原因，我们不能直接使用 MySQL 的驱动包. > @@ -449,7 +449,7 @@ docker build -t apache/dolphinscheduler:mysql-driver . 6. 在数据源中心添加一个 MySQL 数据源 -### How to support Oracle datasource in `Datasource manage`？ +### 如何在数据源中心支持 Oracle 数据源？ > 由于商业许可证的原因，我们不能直接使用 Oracle 的驱动包. > diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md index 2fc4aee9c3..0a5efe3163 100644 --- a/docker/kubernetes/dolphinscheduler/README.md +++ b/docker/kubernetes/dolphinscheduler/README.md @@ -74,34 +74,34 @@ $ kubectl delete pvc -l app.kubernetes.io/instance=dolphinscheduler ## Configuration -The following tables lists the configurable parameters of the Dolphins Scheduler chart and their default values. +The Configuration file is `values.yaml`, and the following tables lists the configurable parameters of the DolphinScheduler chart and their default values. | Parameter | Description | Default | | --------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------- | | `nameOverride` | String to partially override common.names.fullname | `nil` | | `fullnameOverride` | String to fully override common.names.fullname | `nil` | | `timezone` | World time and date for cities in all time zones | `Asia/Shanghai` | -| `image.registry` | Docker image registry for the Dolphins Scheduler | `docker.io` | -| `image.repository` | Docker image repository for the Dolphins Scheduler | `dolphinscheduler` | -| `image.tag` | Docker image version for the Dolphins Scheduler | `1.2.1` | +| `image.registry` | Docker image registry for the DolphinScheduler | `docker.io` | +| `image.repository` | Docker image repository for the DolphinScheduler | `dolphinscheduler` | +| `image.tag` | Docker image version for the DolphinScheduler | `latest` | | `image.pullPolicy` | Image pull policy. One of Always, Never, IfNotPresent | `IfNotPresent` | | `image.pullSecrets` | Image pull secrets. An optional list of references to secrets in the same namespace to use for pulling any of the images | `[]` | | | | | -| `postgresql.enabled` | If not exists external PostgreSQL, by default, the Dolphins Scheduler will use a internal PostgreSQL | `true` | +| `postgresql.enabled` | If not exists external PostgreSQL, by default, the DolphinScheduler will use a internal PostgreSQL | `true` | | `postgresql.postgresqlUsername` | The username for internal PostgreSQL | `root` | | `postgresql.postgresqlPassword` | The password for internal PostgreSQL | `root` | | `postgresql.postgresqlDatabase` | The database for internal PostgreSQL | `dolphinscheduler` | | `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` | | `postgresql.persistence.size` | `PersistentVolumeClaim` Size | `20Gi` | | `postgresql.persistence.storageClass` | PostgreSQL data Persistent Volume Storage Class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` | -| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database type will use it | `postgresql` | -| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database driver will use it | `org.postgresql.Driver` | -| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database host will use it | `localhost` | -| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database port will use it | `5432` | -| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database username will use it | `root` | -| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database password will use it | `root` | -| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database database will use it | `dolphinscheduler` | -| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. Dolphins Scheduler's database params will use it | `characterEncoding=utf8` | +| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` | +| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` | +| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` | +| `externalDatabase.port` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database port will use it | `5432` | +| `externalDatabase.username` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database username will use it | `root` | +| `externalDatabase.password` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database password will use it | `root` | +| `externalDatabase.database` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database database will use it | `dolphinscheduler` | +| `externalDatabase.params` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database params will use it | `characterEncoding=utf8` | | | | | | `zookeeper.enabled` | If not exists external Zookeeper, by default, the DolphinScheduler will use a internal Zookeeper | `true` | | `zookeeper.fourlwCommandsWhitelist` | A list of comma separated Four Letter Words commands to use | `srvr,ruok,wchs,cons` | From a388b6853df229bb54c8d46c433f63376809d211 Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Sat, 27 Feb 2021 21:36:03 +0800 Subject: [PATCH 66/68] Optimize HashMap Initial Capacity. (#4896) --- .../dolphinscheduler/api/controller/BaseController.java | 4 ++-- .../dolphinscheduler/common/utils/CollectionUtils.java | 9 ++++++++- .../dolphinscheduler/service/quartz/QuartzExecutors.java | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java index c9202d4ac6..f42b554791 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/BaseController.java @@ -48,7 +48,7 @@ public class BaseController { * @return check result code */ public Map checkPageParams(int pageNo, int pageSize) { - Map result = new HashMap<>(2); + Map result = new HashMap<>(4); Status resultEnum = Status.SUCCESS; String msg = Status.SUCCESS.getMsg(); if (pageNo <= 0) { @@ -202,7 +202,7 @@ public class BaseController { result.setCode(Status.SUCCESS.getCode()); result.setMsg(Status.SUCCESS.getMsg()); - Map map = new HashMap<>(4); + Map map = new HashMap<>(8); map.put(Constants.TOTAL_LIST, totalList); map.put(Constants.CURRENT_PAGE, currentPage); map.put(Constants.TOTAL_PAGE, totalPage); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java index ba55a37f81..e90c606b63 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java @@ -46,6 +46,11 @@ public class CollectionUtils { throw new UnsupportedOperationException("Construct CollectionUtils"); } + /** + * The load factor used when none specified in constructor. + */ + static final float DEFAULT_LOAD_FACTOR = 0.75f; + /** * Returns a new {@link Collection} containing a minus a subset of * b. Only the elements of b that satisfy the predicate @@ -95,6 +100,7 @@ public class CollectionUtils { * @return string to map */ public static Map stringToMap(String str, String separator, String keyPrefix) { + Map emptyMap = new HashMap<>(0); if (StringUtils.isEmpty(str)) { return emptyMap; @@ -103,7 +109,8 @@ public class CollectionUtils { return emptyMap; } String[] strings = str.split(separator); - Map map = new HashMap<>(strings.length); + int initialCapacity = (int)(strings.length / DEFAULT_LOAD_FACTOR) + 1; + Map map = new HashMap<>(initialCapacity); for (int i = 0; i < strings.length; i++) { String[] strArray = strings[i].split("="); if (strArray.length != 2) { diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java index fd91e4076d..96209af93c 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/QuartzExecutors.java @@ -370,7 +370,7 @@ public class QuartzExecutors { * @return data map */ public static Map buildDataMap(int projectId, int scheduleId, Schedule schedule) { - Map dataMap = new HashMap<>(3); + Map dataMap = new HashMap<>(8); dataMap.put(PROJECT_ID, projectId); dataMap.put(SCHEDULE_ID, scheduleId); dataMap.put(SCHEDULE, JSONUtils.toJsonString(schedule)); From 139211f3ddf9e30fb058c659fd467b18a95d0dbe Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Sat, 27 Feb 2021 21:39:21 +0800 Subject: [PATCH 67/68] Reduce queries after successful project creation. (#4901) --- .../dolphinscheduler/api/service/impl/ProjectServiceImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java index 81e0eebe2f..50bee6f553 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectServiceImpl.java @@ -97,8 +97,7 @@ public class ProjectServiceImpl extends BaseServiceImpl implements ProjectServic .build(); if (projectMapper.insert(project) > 0) { - Project insertedProject = projectMapper.queryByName(name); - result.put(Constants.DATA_LIST, insertedProject); + result.put(Constants.DATA_LIST, project); putMsg(result, Status.SUCCESS); } else { putMsg(result, Status.CREATE_PROJECT_ERROR); From a25c3974fcba99d96e884c9f928d5dae4d455785 Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Mon, 1 Mar 2021 15:13:25 +0800 Subject: [PATCH 68/68] [Improvement][README] Update official website link in readme (#4903) --- .github/ISSUE_TEMPLATE/feature_request.md | 2 +- .../ISSUE_TEMPLATE/improvement_suggestion.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- README.md | 32 +++++++------ README_zh_CN.md | 45 +++++++++++-------- 5 files changed, 48 insertions(+), 35 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 82d811e880..37aebf4a13 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -8,7 +8,7 @@ assignees: '' *For better global communication, please give priority to using English description, thx! * -*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.* +*Please review https://dolphinscheduler.apache.org/en-us/community/development/issue.html when describe an issue.* **Describe the feature** A clear and concise description of what the feature is. diff --git a/.github/ISSUE_TEMPLATE/improvement_suggestion.md b/.github/ISSUE_TEMPLATE/improvement_suggestion.md index 544d98eae5..d01936b19d 100644 --- a/.github/ISSUE_TEMPLATE/improvement_suggestion.md +++ b/.github/ISSUE_TEMPLATE/improvement_suggestion.md @@ -8,7 +8,7 @@ assignees: '' *For better global communication, please give priority to using English description, thx! * -*Please review https://dolphinscheduler.apache.org/en-us/docs/development/issue.html when describe an issue.* +*Please review https://dolphinscheduler.apache.org/en-us/community/development/issue.html when describe an issue.* **Describe the question** A clear and concise description of what the improvement is. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6235a6ce84..c150c845c3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ ## *Tips* - *Thanks very much for contributing to Apache DolphinScheduler.* -- *Please review https://dolphinscheduler.apache.org/en-us/community/index.html before opening a pull request.* +- *Please review https://dolphinscheduler.apache.org/en-us/community/development/pull-request.html before opening a pull request.* ## What is the purpose of the pull request diff --git a/README.md b/README.md index a99a20e354..0c933f642f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ Dolphin Scheduler Official Website [dolphinscheduler.apache.org](https://dolphinscheduler.apache.org) ============ + [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) @@ -12,8 +13,7 @@ Dolphin Scheduler Official Website [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) - -### Design Features: +## Design Features DolphinScheduler is a distributed and extensible workflow scheduler platform with powerful DAG visual interfaces, dedicated to solving complex job dependencies in the data pipeline and providing various types of jobs available `out of the box`. @@ -34,8 +34,7 @@ Its main objectives are as follows: - Support internationalization. - More features waiting for partners to explore... - -### What's in DolphinScheduler +## What's in DolphinScheduler Stability | Accessibility | Features | Scalability | -- | -- | -- | -- @@ -43,8 +42,7 @@ Decentralized multi-master and multi-worker | Visualization of workflow key info support HA | Visualization of all workflow operations, dragging tasks to draw DAGs, configuring data sources and resources. At the same time, for third-party systems, provide API mode operations. | Users on DolphinScheduler can achieve many-to-one or one-to-one mapping relationship through tenants and Hadoop users, which is very important for scheduling large data jobs. | The scheduler supports distributed scheduling, and the overall scheduling capability will increase linearly with the scale of the cluster. Master and Worker support dynamic adjustment. Overload processing: By using the task queue mechanism, the number of schedulable tasks on a single machine can be flexibly configured. Machine jam can be avoided with high tolerance to numbers of tasks cached in task queue. | One-click deployment | Support traditional shell tasks, and big data platform task scheduling: MR, Spark, SQL (MySQL, PostgreSQL, hive, spark SQL), Python, Procedure, Sub_Process | | - -### User Interface Screenshots +## User Interface Screenshots ![home page](https://user-images.githubusercontent.com/15833811/75218288-bf286400-57d4-11ea-8263-d639c6511d5f.jpg) ![dag](https://user-images.githubusercontent.com/15833811/75236750-3374fe80-57f9-11ea-857d-62a66a5a559d.png) @@ -55,11 +53,15 @@ Overload processing: By using the task queue mechanism, the number of schedulabl ![security](https://user-images.githubusercontent.com/15833811/75236441-bfd2f180-57f8-11ea-88bd-f24311e01b7e.png) ![treeview](https://user-images.githubusercontent.com/15833811/75217191-3fe56100-57d1-11ea-8856-f19180d9a879.png) -### QuickStart in Docker -Please referer the official website document:[[QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/1.3.5/user_doc/docker-deployment.html)] +## QuickStart in Docker +Please referer the official website document: [QuickStart in Docker](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/docker-deployment.html) -### How to Build +## QuickStart in Kubernetes + +Please referer the official website document: [QuickStart in Kubernetes](https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/kubernetes-deployment.html) + +## How to Build ```bash ./mvnw clean install -Prelease @@ -72,24 +74,26 @@ dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: Source code package of DolphinScheduler ``` -### Thanks +## Thanks + DolphinScheduler is based on a lot of excellent open-source projects, such as google guava, guice, grpc, netty, ali bonecp, quartz, and many open-source projects of Apache and so on. We would like to express our deep gratitude to all the open-source projects used in Dolphin Scheduler. We hope that we are not only the beneficiaries of open-source, but also give back to the community. Besides, we hope everyone who have the same enthusiasm and passion for open source could join in and contribute to the open-source community! -### Get Help +## Get Help + 1. Submit an [[issue](https://github.com/apache/incubator-dolphinscheduler/issues/new/choose)] 1. Subscribe to this mail list: https://dolphinscheduler.apache.org/en-us/community/development/subscribe.html, then email dev@dolphinscheduler.apache.org +## Community -### Community You are so much welcomed to communicate with the developers and users of Dolphin Scheduler freely. There are two ways to find them: 1. Join the slack channel by [this invitation link](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw). 2. Follow the [twitter account of Dolphin Scheduler](https://twitter.com/dolphinschedule) and get the latest news just on time. +## How to Contribute -### How to Contribute The community welcomes everyone to participate in contributing, please refer to this website to find out more: [[How to contribute](https://dolphinscheduler.apache.org/en-us/community/development/contribute.html)] +## License -### License Please refer to the [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) file. diff --git a/README_zh_CN.md b/README_zh_CN.md index a905827782..abd3d378f4 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -1,23 +1,24 @@ Dolphin Scheduler Official Website [dolphinscheduler.apache.org](https://dolphinscheduler.apache.org) ============ + [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [![Total Lines](https://tokei.rs/b1/github/apache/Incubator-DolphinScheduler?category=lines)](https://github.com/apache/Incubator-DolphinScheduler) [![codecov](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev/graph/badge.svg)](https://codecov.io/gh/apache/incubator-dolphinscheduler/branch/dev) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache-dolphinscheduler&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler) -> Dolphin Scheduler for Big Data - - [![Stargazers over time](https://starchart.cc/apache/incubator-dolphinscheduler.svg)](https://starchart.cc/apache/incubator-dolphinscheduler) [![CN doc](https://img.shields.io/badge/文档-中文版-blue.svg)](README_zh_CN.md) [![EN doc](https://img.shields.io/badge/document-English-blue.svg)](README.md) +## 设计特点 + +一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。 -**设计特点：** 一个分布式易扩展的可视化DAG工作流任务调度系统。致力于解决数据处理流程中错综复杂的依赖关系，使调度系统在数据处理流程中`开箱即用`。 其主要目标如下： + - 以DAG图的方式将Task按照任务的依赖关系关联起来，可实时可视化监控任务的运行状态 - 支持丰富的任务类型：Shell、MR、Spark、SQL(mysql、postgresql、hive、sparksql),Python,Sub_Process、Procedure等 - 支持工作流定时调度、依赖调度、手动调度、手动暂停/停止/恢复，同时支持失败重试/告警、从指定节点恢复失败、Kill任务等操作 @@ -33,8 +34,7 @@ Dolphin Scheduler Official Website - 支持国际化 - 还有更多等待伙伴们探索 - -### 系统部分截图 +## 系统部分截图 ![home page](https://user-images.githubusercontent.com/15833811/75208819-abbad000-57b7-11ea-8d3c-67e7c270671f.jpg) @@ -50,42 +50,51 @@ Dolphin Scheduler Official Website ![security](https://user-images.githubusercontent.com/15833811/75209633-baa28200-57b9-11ea-9def-94bef2e212a7.jpg) - -### 近期研发计划 +## 近期研发计划 DolphinScheduler的工作计划：研发计划 ，其中 In Develop卡片下是正在研发的功能，TODO卡片是待做事项(包括 feature ideas) -### 参与贡献 +## 参与贡献 非常欢迎大家来参与贡献，贡献流程请参考： -[[参与贡献](https://dolphinscheduler.apache.org/zh-cn/docs/development/contribute.html)] +[[参与贡献](https://dolphinscheduler.apache.org/zh-cn/community/development/contribute.html)] -### How to Build +## 快速试用 Docker + +请参考官方文档: [快速试用 Docker 部署](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/docker-deployment.html) + +## 快速试用 Kubernetes + +请参考官方文档: [快速试用 Kubernetes 部署](https://dolphinscheduler.apache.org/zh-cn/docs/latest/user_doc/kubernetes-deployment.html) + +## 如何构建 ```bash ./mvnw clean install -Prelease ``` -Artifact: +制品: ``` -dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-dolphinscheduler-bin.tar.gz: Binary package of DolphinScheduler -dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: Source code package of DolphinScheduler +dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-dolphinscheduler-bin.tar.gz: DolphinScheduler 二进制包 +dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${latest.release.version}-src.zip: DolphinScheduler 源代码包 ``` -### 感谢 +## 感谢 Dolphin Scheduler使用了很多优秀的开源项目，比如google的guava、guice、grpc，netty，ali的bonecp，quartz，以及apache的众多开源项目等等， 正是由于站在这些开源项目的肩膀上，才有Dolphin Scheduler的诞生的可能。对此我们对使用的所有开源软件表示非常的感谢！我们也希望自己不仅是开源的受益者，也能成为开源的贡献者,也希望对开源有同样热情和信念的伙伴加入进来，一起为开源献出一份力！ +## 获得帮助 -### 获得帮助 1. 提交issue 2. 先订阅邮件开发列表:[订阅邮件列表](https://dolphinscheduler.apache.org/zh-cn/community/development/subscribe.html), 订阅成功后发送邮件到dev@dolphinscheduler.apache.org. -### 社区 +## 社区 + 1. 通过[该申请链接](https://join.slack.com/t/asf-dolphinscheduler/shared_invite/zt-mzqu52gi-rCggPkSHQ0DZYkwbTxO1Gw)加入slack channel 2. 关注[Apache Dolphin Scheduler的Twitter账号](https://twitter.com/dolphinschedule)获取实时动态 -### 版权 +## 版权 + 请参考 [LICENSE](https://github.com/apache/incubator-dolphinscheduler/blob/dev/LICENSE) 文件.