From 1bdf359b1a4c50e260960fd35c2896b184e1954b Mon Sep 17 00:00:00 2001 From: Gong Dewei Date: Tue, 19 Oct 2021 10:11:53 +0800 Subject: [PATCH] [3.0] Manage global resources and executor services, fix zk client connections (#9033) * shutdown scheduledExecutors and executorServiceRing * fix NPE of configuration * Fix notify loop after executor service is shutdown * improve registry center in tests * polish unregister shutdown hook log * Fix systemConfiguration NPE * enable surefire reuseForks for fast testing * Fix checking DubboShutdownHook is alive * fix SPI/Bean instantiation constructor matching * shutdown share executor * release zk client * Improve application destroy processing * Add GlobalResourcesRepository to manager static resources, including ExecutorService and HashedWheelTimer * revert zk client address mapping * Fix zk client early close problem * improve executor service * improve both start by module and by application * release sharedScheduledExecutor and fix AccessLogFilter * Fix tests * checking zk registry if destroyed * fix testSystemPropertyOverrideReferenceConfig config error * improve KeepRunningOnSpringClosedTest * SPI extension/scope bean prefers parameterized constructor instead of default constructor * Support compatible usage of ZookeeperRegistryFactory * Improve the application/module destroy process, the deployer destruction is divided into pre-destroy and post-destroy * Destroy NacosDynamicConfiguration * Destroy MetadataReportFactory * Recreate adaptive classes * Destroy global resources on dubbo shutdown * Fix zk thread leaks when create client failed * Improve protocol destroying * Improve thread name of protocol port * Manage global EventLoopGroup, HashedWheelTimer, etc. * Protected metadata info in memory * Support checking unclosed threads when finish test class * Improve application check started * add test log4j.properties * add pom.xml of dubbo-test-check * fix CodecSupport.checkSerialization error * Fix ReferenceCountExchangeClientTest log msg checking * Fix ServiceInstanceMetadataUtilsTest and MigrationInvokerTest * Fix protocol destroy problem * rename test check jvm args * cancel asyncMetadataFuture and unregisterServiceInstance in pre-destroy application * Ignore refresh metadata if application is stopping * Remove unused destroyDynamicConfigurations method * Change ZookeeperTransporter to application scope * Add testMultiProviderApplicationsStopOneByOne * Destroy ZookeeperTransporter in RemotingScopeModelInitializer * support specify test check report file --- dubbo-cluster/pom.xml | 8 +- .../ShortestResponseLoadBalance.java | 14 +- .../support/ForkingClusterInvoker.java | 5 +- dubbo-common/pom.xml | 1 - .../java/org/apache/dubbo/common/URL.java | 5 + .../beans/support/InstantiationStrategy.java | 32 +- .../concurrent/CallableSafeInitializer.java | 101 +++++ .../dubbo/common/config/Environment.java | 9 +- .../common/config/ModuleEnvironment.java | 1 + .../nop/NopDynamicConfiguration.java | 9 +- .../CompositeDynamicConfiguration.java | 16 + .../common/deploy/ApplicationDeployer.java | 20 +- .../dubbo/common/deploy/DeployState.java | 35 +- .../apache/dubbo/common/deploy/Deployer.java | 5 +- .../dubbo/common/deploy/ModuleDeployer.java | 8 +- .../extension/AdaptiveClassCodeGenerator.java | 12 + .../dubbo/common/resource/Disposable.java | 26 ++ .../resource/GlobalResourceInitializer.java | 65 ++++ .../resource/GlobalResourcesRepository.java | 130 +++++++ .../manager/DefaultExecutorRepository.java | 125 ++++--- .../manager/ExecutorRepository.java | 8 +- .../support/cached/CachedThreadPool.java | 2 +- .../support/eager/EagerThreadPool.java | 2 +- .../support/fixed/FixedThreadPool.java | 2 +- .../support/limited/LimitedThreadPool.java | 2 +- .../utils/ClassLoaderResourceLoader.java | 23 +- .../dubbo/config/ServiceConfigBase.java | 9 +- .../dubbo/rpc/model/ApplicationModel.java | 38 +- .../dubbo/rpc/model/FrameworkModel.java | 33 +- .../apache/dubbo/rpc/model/ModuleModel.java | 15 +- .../apache/dubbo/rpc/model/ScopeModel.java | 12 +- .../rpc/model/ScopeModelDestroyListener.java | 4 +- .../FileSystemDynamicConfigurationTest.java | 12 +- .../manager/ExecutorRepositoryTest.java | 57 ++- dubbo-compatible/pom.xml | 6 + .../config/ConfigScopeModelInitializer.java | 3 +- .../dubbo/config/DubboShutdownHook.java | 4 +- .../apache/dubbo/config/ServiceConfig.java | 24 +- .../config/bootstrap/DubboBootstrap.java | 46 +-- .../deploy/DefaultApplicationDeployer.java | 345 ++++++++++++------ .../config/deploy/DefaultModuleDeployer.java | 29 +- .../config/deploy/FrameworkModelCleaner.java | 64 ++++ .../config/AbstractMethodConfigTest.java | 8 + .../config/AbstractReferenceConfigTest.java | 7 + .../dubbo/config/ConfigCenterConfigTest.java | 7 + .../dubbo/config/ProtocolConfigTest.java | 15 +- .../dubbo/config/ProviderConfigTest.java | 1 + .../DubboBootstrapMultiInstanceTest.java | 315 +++++++++++++++- .../DubboDeployApplicationListener.java | 4 +- ...ava => KeepRunningOnSpringClosedTest.java} | 40 +- .../JavaConfigReferenceBeanTest.java | 189 ++++++---- dubbo-config/pom.xml | 9 + .../apollo/ApolloDynamicConfiguration.java | 23 +- .../nacos/NacosConfigServiceWrapper.java | 4 + .../nacos/NacosDynamicConfiguration.java | 26 +- .../ZookeeperDynamicConfiguration.java | 9 +- .../ZookeeperDynamicConfigurationFactory.java | 8 +- dubbo-configcenter/pom.xml | 9 + dubbo-filter/pom.xml | 9 + .../apache/dubbo/metadata/MetadataInfo.java | 11 +- .../report/MetadataReportFactory.java | 3 + .../support/AbstractMetadataReport.java | 14 +- .../AbstractMetadataReportFactory.java | 30 +- .../zookeeper/ZookeeperMetadataReport.java | 12 +- .../ZookeeperMetadataReportFactory.java | 8 +- .../ZookeeperMetadataReportTest.java | 15 +- dubbo-metadata/pom.xml | 8 + dubbo-metrics/pom.xml | 9 + .../dubbo/monitor/dubbo/DubboMonitor.java | 5 +- .../src/test/resources/log4j.properties | 7 + dubbo-monitor/pom.xml | 10 + dubbo-native/pom.xml | 7 +- .../serialize/Serialization$Adaptive.java | 20 +- .../MetadataReportFactory$Adaptive.java | 3 + .../ZookeeperTransporter$Adaptive.java | 28 -- .../apache/dubbo/rpc/Protocol$Adaptive.java | 10 +- .../dubbo/rpc/ProxyFactory$Adaptive.java | 18 +- .../dubbo/rpc/cluster/Cluster$Adaptive.java | 12 +- .../org/apache/dubbo/utils/CodeGenerator.java | 3 +- .../qos/command/impl/ShutdownTelnet.java | 4 +- .../qos/command/impl/ChangeTelnetTest.java | 17 +- .../qos/command/impl/CountTelnetTest.java | 6 +- .../qos/command/impl/InvokeTelnetTest.java | 2 - .../qos/command/impl/PortTelnetTest.java | 12 +- .../dubbo/qos/command/impl/PwdTelnetTest.java | 11 +- .../qos/command/impl/SelectTelnetTest.java | 9 +- .../qos/command/impl/ShutdownTelnetTest.java | 6 +- .../qos/command/util/CommandHelperTest.java | 1 + .../qos/legacy/ChangeTelnetHandlerTest.java | 4 +- .../dubbo/qos/legacy/ProtocolUtils.java | 17 +- .../qos/legacy/TraceTelnetHandlerTest.java | 4 +- dubbo-plugin/pom.xml | 8 + .../InMemoryWritableMetadataService.java | 19 +- .../migration/MigrationRuleListener.java | 2 +- .../integration/RegistryProtocol.java | 22 +- .../registry/support/AbstractRegistry.java | 6 +- .../ServiceInstanceMetadataUtilsTest.java | 8 +- .../migration/MigrationInvokerTest.java | 9 +- .../src/test/resources/log4j.xml | 4 +- .../registry/zookeeper/ZookeeperRegistry.java | 21 +- .../zookeeper/ZookeeperRegistryFactory.java | 11 +- .../zookeeper/ZookeeperServiceDiscovery.java | 1 + .../zookeeper/ZookeeperRegistryTest.java | 7 +- dubbo-registry/pom.xml | 9 + .../RemotingScopeModelInitializer.java | 60 +++ .../apache/dubbo/remoting/api/Connection.java | 2 +- .../remoting/api/NettyEventLoopFactory.java | 9 +- .../exchange/support/DefaultFuture.java | 16 +- .../exchange/support/DefaultFuture2.java | 18 +- .../support/header/HeaderExchangeClient.java | 32 +- .../support/header/HeaderExchangeServer.java | 17 +- .../dispatcher/WrappedChannelHandler.java | 12 +- .../zookeeper/AbstractZookeeperClient.java | 5 +- .../AbstractZookeeperTransporter.java | 10 + .../zookeeper/ZookeeperTransporter.java | 11 +- ...ache.dubbo.rpc.model.ScopeModelInitializer | 1 + .../transport/netty4/NettyClient.java | 18 +- .../curator5/Curator5ZookeeperClient.java | 20 +- .../curator/CuratorZookeeperClient.java | 21 +- dubbo-remoting/pom.xml | 9 + .../dubbo/rpc/filter/AccessLogFilter.java | 25 +- .../dubbo/rpc/protocol/AbstractProtocol.java | 9 +- .../dubbo/rpc/filter/AccessLogFilterTest.java | 7 +- .../rpc/protocol/dubbo/DubboProtocol.java | 21 ++ .../dubbo/decode/DubboTelnetDecodeTest.java | 11 + .../protocol/dubbo/support/ProtocolUtils.java | 21 +- .../src/test/resources/log4j.xml | 8 + .../dubbo/rpc/protocol/grpc/GrpcProtocol.java | 16 +- .../dubbo/rpc/protocol/rest/RestProtocol.java | 4 +- .../rpc/protocol/tri/TripleProtocol.java | 23 +- dubbo-rpc/pom.xml | 9 + dubbo-serialization/pom.xml | 9 + .../AwaitingNonWebApplicationListener.java | 12 +- dubbo-spring-boot/pom.xml | 6 + dubbo-test/dubbo-test-check/pom.xml | 51 +++ .../dubbo/test/check/DubboTestChecker.java | 309 ++++++++++++++++ ...it.platform.launcher.TestExecutionListener | 1 + dubbo-test/dubbo-test-common/pom.xml | 6 + dubbo-test/pom.xml | 1 + 139 files changed, 2549 insertions(+), 764 deletions(-) create mode 100644 dubbo-common/src/main/java/org/apache/dubbo/common/concurrent/CallableSafeInitializer.java create mode 100644 dubbo-common/src/main/java/org/apache/dubbo/common/resource/Disposable.java create mode 100644 dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourceInitializer.java create mode 100644 dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourcesRepository.java create mode 100644 dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/FrameworkModelCleaner.java rename dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/{ShutdownHookTest.java => KeepRunningOnSpringClosedTest.java} (57%) create mode 100644 dubbo-monitor/dubbo-monitor-default/src/test/resources/log4j.properties delete mode 100644 dubbo-native/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter$Adaptive.java create mode 100644 dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/RemotingScopeModelInitializer.java create mode 100644 dubbo-remoting/dubbo-remoting-api/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.model.ScopeModelInitializer create mode 100644 dubbo-test/dubbo-test-check/pom.xml create mode 100644 dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/DubboTestChecker.java create mode 100644 dubbo-test/dubbo-test-check/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener diff --git a/dubbo-cluster/pom.xml b/dubbo-cluster/pom.xml index 26b1295ffb..fbb0d177b4 100644 --- a/dubbo-cluster/pom.xml +++ b/dubbo-cluster/pom.xml @@ -60,5 +60,11 @@ ${project.parent.version} test + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + - \ No newline at end of file + diff --git a/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/loadbalance/ShortestResponseLoadBalance.java b/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/loadbalance/ShortestResponseLoadBalance.java index a898c8467b..1f06079d8f 100644 --- a/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/loadbalance/ShortestResponseLoadBalance.java +++ b/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/loadbalance/ShortestResponseLoadBalance.java @@ -17,7 +17,6 @@ package org.apache.dubbo.rpc.cluster.loadbalance; import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.rpc.Invocation; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.RpcStatus; @@ -29,7 +28,6 @@ import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,7 +44,7 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements public static final String NAME = "shortestresponse"; - private int SLIDE_PERIOD = 30_000; + private int slidePeriod = 30_000; private ConcurrentMap methodMap = new ConcurrentHashMap<>(); @@ -54,13 +52,15 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements private volatile long lastUpdateTime = System.currentTimeMillis(); + private ExecutorService executorService; + @Override public void setApplicationModel(ApplicationModel applicationModel) { - SLIDE_PERIOD = applicationModel.getModelEnvironment().getConfiguration().getInt(Constants.SHORTEST_RESPONSE_SLIDE_PERIOD, 30_000); + slidePeriod = applicationModel.getModelEnvironment().getConfiguration().getInt(Constants.SHORTEST_RESPONSE_SLIDE_PERIOD, 30_000); + executorService = applicationModel.getApplicationExecutorRepository().getSharedExecutor(); } protected static class SlideWindowData { - private final static ExecutorService EXECUTOR_SERVICE = Executors.newSingleThreadExecutor((new NamedThreadFactory("Dubbo-slidePeriod-reset"))); private long succeededOffset; private long succeededElapsedOffset; @@ -138,10 +138,10 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements } } - if (System.currentTimeMillis() - lastUpdateTime > SLIDE_PERIOD + if (System.currentTimeMillis() - lastUpdateTime > slidePeriod && onResetSlideWindow.compareAndSet(false, true)) { //reset slideWindowData in async way - SlideWindowData.EXECUTOR_SERVICE.execute(() -> { + executorService.execute(() -> { methodMap.values().forEach(SlideWindowData::reset); lastUpdateTime = System.currentTimeMillis(); onResetSlideWindow.set(false); diff --git a/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/support/ForkingClusterInvoker.java b/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/support/ForkingClusterInvoker.java index 8980b34d47..6439c465cd 100644 --- a/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/support/ForkingClusterInvoker.java +++ b/dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/support/ForkingClusterInvoker.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -53,11 +52,11 @@ public class ForkingClusterInvoker extends AbstractClusterInvoker { * Use {@link NamedInternalThreadFactory} to produce {@link org.apache.dubbo.common.threadlocal.InternalThread} * which with the use of {@link org.apache.dubbo.common.threadlocal.InternalThreadLocal} in {@link RpcContext}. */ - private final ExecutorService executor = Executors.newCachedThreadPool( - new NamedInternalThreadFactory("forking-cluster-timer", true)); + private final ExecutorService executor; public ForkingClusterInvoker(Directory directory) { super(directory); + executor = directory.getUrl().getOrDefaultApplicationModel().getApplicationExecutorRepository().getSharedExecutor(); } @Override diff --git a/dubbo-common/pom.xml b/dubbo-common/pom.xml index 1e2d18a83b..0c8d76835b 100644 --- a/dubbo-common/pom.xml +++ b/dubbo-common/pom.xml @@ -94,7 +94,6 @@ test - diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/URL.java b/dubbo-common/src/main/java/org/apache/dubbo/common/URL.java index 2a4cdd08df..848d599bd6 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/URL.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/URL.java @@ -1577,6 +1577,11 @@ class URL implements Serializable { return attributes.get(key); } + public Object getAttribute(String key, Object defaultValue) { + Object val = attributes.get(key); + return val != null ? val : defaultValue; + } + public URL putAttribute(String key, Object obj) { attributes.put(key, obj); return this; diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/beans/support/InstantiationStrategy.java b/dubbo-common/src/main/java/org/apache/dubbo/common/beans/support/InstantiationStrategy.java index b1b67cef0e..02e4063a1b 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/beans/support/InstantiationStrategy.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/beans/support/InstantiationStrategy.java @@ -33,7 +33,6 @@ import java.util.List; */ public class InstantiationStrategy { - private boolean supportConstructorWithArguments; private ScopeModelAccessor scopeModelAccessor; public InstantiationStrategy() { @@ -42,19 +41,17 @@ public class InstantiationStrategy { public InstantiationStrategy(ScopeModelAccessor scopeModelAccessor) { this.scopeModelAccessor = scopeModelAccessor; - this.supportConstructorWithArguments = (this.scopeModelAccessor != null); } public T instantiate(Class type) throws ReflectiveOperationException { - // 1. try default constructor + // should not use default constructor directly, maybe also has another constructor matched scope model arguments + // 1. try get default constructor + Constructor defaultConstructor = null; try { - return type.getConstructor().newInstance(); + defaultConstructor = type.getConstructor(); } catch (NoSuchMethodException e) { // ignore no default constructor - if (!supportConstructorWithArguments) { - throw new IllegalArgumentException("Default constructor was not found for type: " + type.getName()); - } } // 2. use matched constructor if found @@ -65,22 +62,35 @@ public class InstantiationStrategy { matchedConstructors.add(constructor); } } + // remove default constructor from matchedConstructors + if (defaultConstructor != null) { + matchedConstructors.remove(defaultConstructor); + } + + // match order: + // 1. the only matched constructor with parameters + // 2. default constructor if absent + + Constructor targetConstructor; if (matchedConstructors.size() > 1) { throw new IllegalArgumentException("Expect only one but found " + matchedConstructors.size() + " matched constructors for type: " + type.getName() + ", matched constructors: " + matchedConstructors); - } else if (matchedConstructors.size() == 0) { + } else if (matchedConstructors.size() == 1) { + targetConstructor = matchedConstructors.get(0); + } else if (defaultConstructor != null) { + targetConstructor = defaultConstructor; + } else { throw new IllegalArgumentException("None matched constructor was found for type: " + type.getName()); } // create instance with arguments - Constructor constructor = matchedConstructors.get(0); - Class[] parameterTypes = constructor.getParameterTypes(); + Class[] parameterTypes = targetConstructor.getParameterTypes(); Object[] args = new Object[parameterTypes.length]; for (int i = 0; i < parameterTypes.length; i++) { args[i] = getArgumentValueForType(parameterTypes[i]); } - return (T) constructor.newInstance(args); + return (T) targetConstructor.newInstance(args); } private boolean isMatched(Constructor constructor) { diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/concurrent/CallableSafeInitializer.java b/dubbo-common/src/main/java/org/apache/dubbo/common/concurrent/CallableSafeInitializer.java new file mode 100644 index 0000000000..092ec41a87 --- /dev/null +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/concurrent/CallableSafeInitializer.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.common.concurrent; + +import org.apache.dubbo.common.resource.Disposable; + +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +/** + *

+ * A safe and lazy and removable initializer implementation that wraps a + * {@code Callable} object. + *

+ * @see org.apache.commons.lang3.concurrent.AtomicSafeInitializer + */ +public class CallableSafeInitializer { + /** A guard which ensures that initialize() is called only once. */ + private final AtomicReference> factory = + new AtomicReference<>(); + + /** Holds the reference to the managed object. */ + private final AtomicReference reference = new AtomicReference<>(); + + /** The Callable to be executed. */ + private final Callable callable; + + public CallableSafeInitializer(Callable callable) { + this.callable = callable; + } + + /** + * Get (and initialize, if not initialized yet) the required object + * + * @return lazily initialized object + * exception + */ + //@Override + public final T get() { + T result; + + while ((result = reference.get()) == null) { + if (factory.compareAndSet(null, this)) { + reference.set(initialize()); + } + } + + return result; + } + + /** + * Creates and initializes the object managed by this + * {@code AtomicInitializer}. This method is called by {@link #get()} when + * the managed object is not available yet. An implementation can focus on + * the creation of the object. No synchronization is needed, as this is + * already handled by {@code get()}. This method is guaranteed to be called + * only once. + * + * @return the managed data object + */ + protected T initialize() { + try { + return callable.call(); + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + public T remove() { + return this.remove(null); + } + + public T remove(Consumer action) { + // release + T value = reference.getAndSet(null); + if (value != null && action != null) { + if (action != null) { + action.accept(value); + } else if (value instanceof Disposable) { + ((Disposable) value).destroy(); + } + } + factory.set(null); + return value; + } +} diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/config/Environment.java b/dubbo-common/src/main/java/org/apache/dubbo/common/config/Environment.java index 6df25f1909..67ecf61f9e 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/config/Environment.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/config/Environment.java @@ -246,7 +246,14 @@ public class Environment extends LifecycleAdapter implements ApplicationExt { globalConfiguration = null; globalConfigurationMaps = null; defaultDynamicGlobalConfiguration = null; - defaultDynamicConfiguration = null; + if (defaultDynamicConfiguration != null) { + try { + defaultDynamicConfiguration.close(); + } catch (Exception e) { + logger.warn("close dynamic configuration failed: " + e.getMessage(), e); + } + defaultDynamicConfiguration = null; + } } /** diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/config/ModuleEnvironment.java b/dubbo-common/src/main/java/org/apache/dubbo/common/config/ModuleEnvironment.java index 5aad696d23..c177f5133a 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/config/ModuleEnvironment.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/config/ModuleEnvironment.java @@ -119,6 +119,7 @@ public class ModuleEnvironment extends Environment implements ModuleExt { @Override public void destroy() throws IllegalStateException { + super.destroy(); this.orderedPropertiesConfiguration = null; this.globalConfiguration = null; this.dynamicGlobalConfiguration = null; diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/nop/NopDynamicConfiguration.java b/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/nop/NopDynamicConfiguration.java index 213e8c21b7..026399976e 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/nop/NopDynamicConfiguration.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/nop/NopDynamicConfiguration.java @@ -20,10 +20,10 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.config.configcenter.ConfigurationListener; import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; -import static java.util.Collections.emptySortedSet; - import java.util.SortedSet; +import static java.util.Collections.emptySortedSet; + /** * The default extension of {@link DynamicConfiguration}. If user does not specify a config center, or specifies one * that is not a valid extension, it will default to this one. @@ -71,4 +71,9 @@ public class NopDynamicConfiguration implements DynamicConfiguration { public SortedSet getConfigKeys(String group) { return emptySortedSet(); } + + @Override + public void close() throws Exception { + // no-op + } } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/wrapper/CompositeDynamicConfiguration.java b/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/wrapper/CompositeDynamicConfiguration.java index 061385412f..a2e8deace4 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/wrapper/CompositeDynamicConfiguration.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/wrapper/CompositeDynamicConfiguration.java @@ -18,6 +18,8 @@ package org.apache.dubbo.common.config.configcenter.wrapper; import org.apache.dubbo.common.config.configcenter.ConfigurationListener; import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; import java.util.HashSet; import java.util.Set; @@ -32,6 +34,8 @@ public class CompositeDynamicConfiguration implements DynamicConfiguration { public static final String NAME = "COMPOSITE"; + private static final Logger logger = LoggerFactory.getLogger(CompositeDynamicConfiguration.class); + private Set configurations = new HashSet<>(); public void addConfiguration(DynamicConfiguration configuration) { @@ -87,6 +91,18 @@ public class CompositeDynamicConfiguration implements DynamicConfiguration { return (SortedSet) iterateConfigOperation(configuration -> configuration.getConfigKeys(group)); } + @Override + public void close() throws Exception { + for (DynamicConfiguration configuration : configurations) { + try { + configuration.close(); + } catch (Exception e) { + logger.warn("close dynamic configuration failed: " + e.getMessage(), e); + } + } + configurations.clear(); + } + private void iterateListenerOperation(Consumer consumer) { for (DynamicConfiguration configuration : configurations) { consumer.accept(configuration); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ApplicationDeployer.java b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ApplicationDeployer.java index 0a61208b4d..567322458e 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ApplicationDeployer.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ApplicationDeployer.java @@ -19,7 +19,7 @@ package org.apache.dubbo.common.deploy; import org.apache.dubbo.common.config.ReferenceCache; import org.apache.dubbo.rpc.model.ApplicationModel; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; /** * initialize and start application instance @@ -33,17 +33,29 @@ public interface ApplicationDeployer extends Deployer { /** * Starts the component. + * @return */ - CompletableFuture start() throws IllegalStateException; + Future start() throws IllegalStateException; /** * Stops the component. */ void stop() throws IllegalStateException; + /** + * Register application instance and start internal services + */ void prepareApplicationInstance(); - void destroy(); + /** + * Pre-processing before destroy model + */ + void preDestroy(); + + /** + * Post-processing after destroy model + */ + void postDestroy(); /** * Indicates that the Application is initialized or not. @@ -61,6 +73,6 @@ public interface ApplicationDeployer extends Deployer { void checkStarting(); - void checkStarted(CompletableFuture checkerStartFuture); + void checkStarted(); } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/DeployState.java b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/DeployState.java index eda92c49cd..2790639a70 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/DeployState.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/DeployState.java @@ -20,5 +20,38 @@ package org.apache.dubbo.common.deploy; * Deploy state enum */ public enum DeployState { - PENDING, STARTING, STARTED, STOPPING, STOPPED, FAILED + /** + * Unknown state + */ + UNKNOWN, + + /** + * Pending, wait for start + */ + PENDING, + + /** + * Starting + */ + STARTING, + + /** + * Started + */ + STARTED, + + /** + * Stopping + */ + STOPPING, + + /** + * Stopped + */ + STOPPED, + + /** + * Failed + */ + FAILED } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/Deployer.java b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/Deployer.java index b4d0704739..568dda29ee 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/Deployer.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/Deployer.java @@ -18,7 +18,7 @@ package org.apache.dubbo.common.deploy; import org.apache.dubbo.rpc.model.ScopeModel; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; /** */ @@ -31,8 +31,9 @@ public interface Deployer { /** * Starts the component. + * @return */ - CompletableFuture start() throws IllegalStateException; + Future start() throws IllegalStateException; /** * Stops the component. diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ModuleDeployer.java b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ModuleDeployer.java index 45894c6c80..6c32be839e 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ModuleDeployer.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ModuleDeployer.java @@ -20,7 +20,7 @@ import org.apache.dubbo.common.config.ReferenceCache; import org.apache.dubbo.config.ServiceConfigBase; import org.apache.dubbo.rpc.model.ModuleModel; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; /** * Export/refer services of module @@ -29,11 +29,13 @@ public interface ModuleDeployer extends Deployer { void initialize() throws IllegalStateException; - CompletableFuture start() throws IllegalStateException; + Future start() throws IllegalStateException; void stop() throws IllegalStateException; - void destroy() throws IllegalStateException; + void preDestroy() throws IllegalStateException; + + void postDestroy() throws IllegalStateException; boolean isInitialized(); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/extension/AdaptiveClassCodeGenerator.java b/dubbo-common/src/main/java/org/apache/dubbo/common/extension/AdaptiveClassCodeGenerator.java index b19f8c27d4..61a9a8d3cd 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/extension/AdaptiveClassCodeGenerator.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/extension/AdaptiveClassCodeGenerator.java @@ -26,6 +26,7 @@ import org.apache.dubbo.rpc.model.ScopeModelUtil; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -90,6 +91,14 @@ public class AdaptiveClassCodeGenerator { * generate and return class code */ public String generate() { + return this.generate(false); + } + + /** + * generate and return class code + * @param sort - whether sort methods + */ + public String generate(boolean sort) { // no need to generate adaptive class since there's no adaptive method found. if (!hasAdaptiveMethod()) { throw new IllegalStateException("No adaptive method exist on extension " + type.getName() + ", refuse to create the adaptive class!"); @@ -101,6 +110,9 @@ public class AdaptiveClassCodeGenerator { code.append(generateClassDeclaration()); Method[] methods = type.getMethods(); + if (sort) { + Arrays.sort(methods, Comparator.comparing(Method::toString)); + } for (Method method : methods) { code.append(generateMethod(method)); } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/resource/Disposable.java b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/Disposable.java new file mode 100644 index 0000000000..78c075cdf3 --- /dev/null +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/Disposable.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.common.resource; + +/** + * An interface for destroying resources + */ +public interface Disposable { + + void destroy(); + +} diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourceInitializer.java b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourceInitializer.java new file mode 100644 index 0000000000..80c467eee6 --- /dev/null +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourceInitializer.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.common.resource; + +import org.apache.dubbo.common.concurrent.CallableSafeInitializer; + +import java.util.concurrent.Callable; +import java.util.function.Consumer; + +/** + * A initializer to release resource automatically on dubbo shutdown + */ +public class GlobalResourceInitializer extends CallableSafeInitializer { + + /** + * The Dispose action to be executed on shutdown. + */ + private Consumer disposeAction; + + private Disposable disposable; + + public GlobalResourceInitializer(Callable initializer) { + super(initializer); + } + + public GlobalResourceInitializer(Callable initializer, Consumer disposeAction) { + super(initializer); + this.disposeAction = disposeAction; + } + + public GlobalResourceInitializer(Callable callable, Disposable disposable) { + super(callable); + this.disposable = disposable; + } + + @Override + protected T initialize() { + T value = super.initialize(); + // register disposable to release automatically + if (this.disposable != null) { + GlobalResourcesRepository.getInstance().registerDisposable(this.disposable); + } else { + GlobalResourcesRepository.getInstance().registerDisposable(() -> this.remove(disposeAction)); + } + return value; + } + + public interface DestroyHandler { + void dispose(GlobalResourceInitializer initializer); + } +} diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourcesRepository.java b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourcesRepository.java new file mode 100644 index 0000000000..8e60e5d976 --- /dev/null +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourcesRepository.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.common.resource; + +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.utils.NamedThreadFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Global resources repository between all framework models. + * It will be destroyed only after all framework model is destroyed. + */ +public class GlobalResourcesRepository { + + private static final Logger logger = LoggerFactory.getLogger(GlobalResourcesRepository.class); + + private volatile static GlobalResourcesRepository instance; + private volatile ExecutorService executorService; + private final List oneoffDisposables = Collections.synchronizedList(new ArrayList<>()); + private final List reusedDisposables = Collections.synchronizedList(new ArrayList<>()); + + private GlobalResourcesRepository() { + } + + public static GlobalResourcesRepository getInstance() { + if (instance == null) { + synchronized (GlobalResourcesRepository.class) { + if (instance == null) { + instance = new GlobalResourcesRepository(); + } + } + } + return instance; + } + + public static ExecutorService getGlobalExecutorService() { + return getInstance().getExecutorService(); + } + + public ExecutorService getExecutorService() { + if (executorService == null || executorService.isShutdown()) { + synchronized (this) { + if (executorService == null || executorService.isShutdown()) { + executorService = Executors.newCachedThreadPool(new NamedThreadFactory("Dubbo-global-shared-handler", true)); + } + } + } + return executorService; + } + + public void destroy() { + if (logger.isInfoEnabled()) { + logger.info("Destroying global resources ..."); + } + if (executorService != null) { + executorService.shutdownNow(); + executorService = null; + } + + // notify disposables + // NOTE: don't clear reused disposables for reuse purpose + for (Disposable disposable : new ArrayList<>(reusedDisposables)) { + try { + disposable.destroy(); + } catch (Exception e) { + logger.warn("destroy resources failed: " + e.getMessage(), e); + } + } + + for (Disposable disposable : new ArrayList<>(oneoffDisposables)) { + try { + disposable.destroy(); + } catch (Exception e) { + logger.warn("destroy resources failed: " + e.getMessage(), e); + } + } + // clear one-off disposable + oneoffDisposables.clear(); + + if (logger.isInfoEnabled()) { + logger.info("Dubbo is completely destroyed"); + } + } + + /** + * Register a one-off disposable, the disposable is removed automatically on first shutdown. + * @param disposable + */ + public void registerDisposable(Disposable disposable) { + this.registerDisposable(disposable, false); + } + + /** + * Register a disposable + * @param disposable + * @param reused true - the disposable is keep and reused. false - the disposable is removed automatically on first shutdown + */ + public void registerDisposable(Disposable disposable, boolean reused) { + List disposables = reused ? reusedDisposables : oneoffDisposables; + if (!disposables.contains(disposable)) { + disposables.add(disposable); + } + } + + public void removeDisposable(Disposable disposable) { + this.reusedDisposables.remove(disposable); + this.oneoffDisposables.remove(disposable); + } + +} diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/DefaultExecutorRepository.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/DefaultExecutorRepository.java index 731fbd4c55..0ed4ac5e36 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/DefaultExecutorRepository.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/DefaultExecutorRepository.java @@ -32,6 +32,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel; import org.apache.dubbo.rpc.model.ModuleModel; import org.apache.dubbo.rpc.model.ScopeModelAware; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -48,6 +49,7 @@ import static org.apache.dubbo.common.constants.CommonConstants.DEFAULT_REFER_TH import static org.apache.dubbo.common.constants.CommonConstants.EXECUTOR_SERVICE_COMPONENT_KEY; import static org.apache.dubbo.common.constants.CommonConstants.SIDE_KEY; import static org.apache.dubbo.common.constants.CommonConstants.THREADS_KEY; +import static org.apache.dubbo.common.constants.CommonConstants.THREAD_NAME_KEY; /** * Consider implementing {@code Licycle} to enable executors shutdown when the process stops. @@ -57,17 +59,18 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA private int DEFAULT_SCHEDULER_SIZE = Runtime.getRuntime().availableProcessors(); - private final ExecutorService SHARED_EXECUTOR = Executors.newCachedThreadPool(new NamedThreadFactory("DubboSharedHandler", true)); + private final ExecutorService sharedExecutor; + private final ScheduledExecutorService sharedScheduledExecutor; private Ring scheduledExecutors = new Ring<>(); - private volatile ExecutorService serviceExportExecutor; + private volatile ScheduledExecutorService serviceExportExecutor; private volatile ExecutorService serviceReferExecutor; private ScheduledExecutorService reconnectScheduledExecutor; - public Ring registryNotificationExecutorRing = new Ring<>(); + public Ring registryNotificationExecutorRing = new Ring<>(); private Ring serviceDiscoveryAddressNotificationExecutorRing = new Ring<>(); @@ -77,25 +80,28 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA private ExecutorService poolRouterExecutor; - private static Ring executorServiceRing = new Ring(); + private Ring executorServiceRing = new Ring(); - private static final Object LOCK = new Object(); + private final Object LOCK = new Object(); private ExtensionAccessor extensionAccessor; private ApplicationModel applicationModel; public DefaultExecutorRepository() { + sharedExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("Dubbo-shared-handler", true)); + sharedScheduledExecutor = Executors.newScheduledThreadPool(8, new NamedThreadFactory("Dubbo-shared-scheduler", true)); + for (int i = 0; i < DEFAULT_SCHEDULER_SIZE; i++) { ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor( - new NamedThreadFactory("Dubbo-framework-scheduler", true)); + new NamedThreadFactory("Dubbo-framework-scheduler-" + i, true)); scheduledExecutors.addItem(scheduler); executorServiceRing.addItem(new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, - new LinkedBlockingQueue(1024), new NamedInternalThreadFactory("Dubbo-state-router-loop", true) + new LinkedBlockingQueue(1024), new NamedInternalThreadFactory("Dubbo-state-router-loop-" + i, true) , new ThreadPoolExecutor.AbortPolicy())); } -// + // reconnectScheduledExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Dubbo-reconnect-scheduler")); poolRouterExecutor = new ThreadPoolExecutor(1, 10, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue(1024), new NamedInternalThreadFactory("Dubbo-state-router-pool-router", true), new ThreadPoolExecutor.AbortPolicy()); @@ -123,7 +129,11 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA Map executors = data.computeIfAbsent(EXECUTOR_SERVICE_COMPONENT_KEY, k -> new ConcurrentHashMap<>()); // Consumer's executor is sharing globally, key=Integer.MAX_VALUE. Provider's executor is sharing by protocol. Integer portKey = CONSUMER_SIDE.equalsIgnoreCase(url.getParameter(SIDE_KEY)) ? Integer.MAX_VALUE : url.getPort(); - ExecutorService executor = executors.computeIfAbsent(portKey, k -> createExecutor(url)); + if (url.getParameter(THREAD_NAME_KEY) == null) { + url = url.putAttribute(THREAD_NAME_KEY, "Dubbo-protocol-"+portKey); + } + URL finalUrl = url; + ExecutorService executor = executors.computeIfAbsent(portKey, k -> createExecutor(finalUrl)); // If executor has been shut down, create a new one if (executor.isShutdown() || executor.isTerminated()) { executors.remove(portKey); @@ -156,7 +166,7 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA logger.info("Executor for " + url + " is shutdown."); } if (executor == null) { - return SHARED_EXECUTOR; + return sharedExecutor; } else { return executor; } @@ -201,17 +211,16 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA } @Override - public ExecutorService getServiceExportExecutor() { + public ScheduledExecutorService getServiceExportExecutor() { if (serviceExportExecutor == null) { synchronized (LOCK) { if (serviceExportExecutor == null) { int coreSize = getExportThreadNum(); - serviceExportExecutor = Executors.newFixedThreadPool(coreSize, + serviceExportExecutor = Executors.newScheduledThreadPool(coreSize, new NamedThreadFactory("Dubbo-service-export", true)); } } } - return serviceExportExecutor; } @@ -219,14 +228,13 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA public void shutdownServiceExportExecutor() { synchronized (LOCK) { if (serviceExportExecutor != null && !serviceExportExecutor.isShutdown()) { - try{ + try { serviceExportExecutor.shutdown(); - }catch (Throwable ignored){ + } catch (Throwable ignored) { // ignored - logger.warn(ignored.getMessage(),ignored); + logger.warn(ignored.getMessage(), ignored); } } - serviceExportExecutor = null; } } @@ -242,7 +250,6 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA } } } - return serviceReferExecutor; } @@ -250,13 +257,12 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA public void shutdownServiceReferExecutor() { synchronized (LOCK) { if (serviceReferExecutor != null && !serviceReferExecutor.isShutdown()) { - try{ + try { serviceReferExecutor.shutdown(); - }catch (Throwable ignored){ - logger.warn(ignored.getMessage(),ignored); + } catch (Throwable ignored) { + logger.warn(ignored.getMessage(), ignored); } } - serviceReferExecutor = null; } } @@ -341,7 +347,12 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA @Override public ExecutorService getSharedExecutor() { - return SHARED_EXECUTOR; + return sharedExecutor; + } + + @Override + public ScheduledExecutorService getSharedScheduledExecutor() { + return sharedScheduledExecutor; } private ExecutorService createExecutor(URL url) { @@ -355,20 +366,10 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA @Override public void destroyAll() { - try{ - poolRouterExecutor.shutdown(); - }catch (Throwable ignored){ - // ignored - logger.warn(ignored.getMessage(),ignored); - } -// serviceDiscoveryAddressNotificationExecutor.shutdown(); -// registryNotificationExecutor.shutdown(); - try{ - metadataRetryExecutor.shutdown(); - }catch (Throwable ignored){ - // ignored - logger.warn(ignored.getMessage(),ignored); - } + logger.info("destroying executor repository .."); + shutdownExecutorService(poolRouterExecutor, "poolRouterExecutor"); + shutdownExecutorService(metadataRetryExecutor, "metadataRetryExecutor"); + shutdownServiceExportExecutor(); shutdownServiceReferExecutor(); @@ -376,32 +377,50 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA if (executors != null) { executors.values().forEach(executor -> { if (executor != null && !executor.isShutdown()) { - try{ + try { ExecutorUtil.shutdownNow(executor, 100); - }catch (Throwable ignored){ + } catch (Throwable ignored) { // ignored - logger.warn(ignored.getMessage(),ignored); + logger.warn(ignored.getMessage(), ignored); } } }); } }); + data.clear(); - // shutdown all executor services - for (ScheduledExecutorService executorService : scheduledExecutors.listItems()) { - try { - executorService.shutdown(); - } catch (Exception e) { - logger.warn("shutdown scheduledExecutors failed: " + e.getMessage(), e); - } + // scheduledExecutors + shutdownExecutorServices(scheduledExecutors.listItems(), "scheduledExecutors"); + + // executorServiceRing + shutdownExecutorServices(executorServiceRing.listItems(), "executorServiceRing"); + + // shutdown share executor + shutdownExecutorService(sharedExecutor, "sharedExecutor"); + shutdownExecutorService(sharedScheduledExecutor, "sharedScheduledExecutor"); + + // serviceDiscoveryAddressNotificationExecutorRing + shutdownExecutorServices(serviceDiscoveryAddressNotificationExecutorRing.listItems(), + "serviceDiscoveryAddressNotificationExecutorRing"); + + // registryNotificationExecutorRing + shutdownExecutorServices(registryNotificationExecutorRing.listItems(), + "registryNotificationExecutorRing"); + + } + + private void shutdownExecutorServices(List executorServices, String msg) { + for (ExecutorService executorService : executorServices) { + shutdownExecutorService(executorService, msg); } + } - for (ExecutorService executorService : executorServiceRing.listItems()) { - try { - executorService.shutdown(); - } catch (Exception e) { - logger.warn("shutdown executorServiceRing failed: " + e.getMessage(), e); - } + private void shutdownExecutorService(ExecutorService executorService, String name) { + try { + executorService.shutdownNow(); + } catch (Exception e) { + String msg = "shutdown executor service [" + name + "] failed: "; + logger.warn(msg + e.getMessage(), e); } } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepository.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepository.java index dabd581443..4e78ea8a77 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepository.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepository.java @@ -58,7 +58,7 @@ public interface ExecutorRepository { ExecutorService nextExecutorExecutor(); - ExecutorService getServiceExportExecutor(); + ScheduledExecutorService getServiceExportExecutor(); /** * The executor only used in bootstrap currently, we should call this method to release the resource @@ -92,6 +92,12 @@ public interface ExecutorRepository { */ ExecutorService getSharedExecutor(); + /** + * Get the shared schedule executor + * @return + */ + ScheduledExecutorService getSharedScheduledExecutor(); + ExecutorService getPoolRouterExecutor(); /** diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/cached/CachedThreadPool.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/cached/CachedThreadPool.java index 02d3bb6f2d..eb14fba089 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/cached/CachedThreadPool.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/cached/CachedThreadPool.java @@ -47,7 +47,7 @@ public class CachedThreadPool implements ThreadPool { @Override public Executor getExecutor(URL url) { - String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME); + String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME)); int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS); int threads = url.getParameter(THREADS_KEY, Integer.MAX_VALUE); int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/eager/EagerThreadPool.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/eager/EagerThreadPool.java index 6d126894aa..8aa8fac819 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/eager/EagerThreadPool.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/eager/EagerThreadPool.java @@ -44,7 +44,7 @@ public class EagerThreadPool implements ThreadPool { @Override public Executor getExecutor(URL url) { - String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME); + String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME)); int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS); int threads = url.getParameter(THREADS_KEY, Integer.MAX_VALUE); int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/fixed/FixedThreadPool.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/fixed/FixedThreadPool.java index bcf12f4917..71ee07449a 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/fixed/FixedThreadPool.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/fixed/FixedThreadPool.java @@ -43,7 +43,7 @@ public class FixedThreadPool implements ThreadPool { @Override public Executor getExecutor(URL url) { - String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME); + String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME)); int threads = url.getParameter(THREADS_KEY, DEFAULT_THREADS); int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES); return new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/limited/LimitedThreadPool.java b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/limited/LimitedThreadPool.java index 87c46d3742..25e3003900 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/limited/LimitedThreadPool.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/limited/LimitedThreadPool.java @@ -45,7 +45,7 @@ public class LimitedThreadPool implements ThreadPool { @Override public Executor getExecutor(URL url) { - String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME); + String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME)); int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS); int threads = url.getParameter(THREADS_KEY, DEFAULT_THREADS); int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/ClassLoaderResourceLoader.java b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/ClassLoaderResourceLoader.java index ee149c4b63..be2a5619ea 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/common/utils/ClassLoaderResourceLoader.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/common/utils/ClassLoaderResourceLoader.java @@ -16,6 +16,8 @@ */ package org.apache.dubbo.common.utils; +import org.apache.dubbo.common.resource.GlobalResourcesRepository; + import java.io.IOException; import java.lang.ref.SoftReference; import java.lang.reflect.Field; @@ -30,25 +32,21 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; public class ClassLoaderResourceLoader { - private final static ExecutorService executorService = - new ThreadPoolExecutor(0, Integer.MAX_VALUE, - 60L, TimeUnit.SECONDS, - new SynchronousQueue<>(), - new NamedThreadFactory("DubboClassLoaderResourceLoader", true)); private static SoftReference>>> classLoaderResourcesCache = null; + static { + // register resources destroy listener + GlobalResourcesRepository.getInstance().registerDisposable(()-> destroy(), true); + } + public static Map> loadResources(String fileName, List classLoaders) { Map> resources = new ConcurrentHashMap<>(); CountDownLatch countDownLatch = new CountDownLatch(classLoaders.size()); for (ClassLoader classLoader : classLoaders) { - executorService.submit(()->{ + GlobalResourcesRepository.getGlobalExecutorService().submit(() -> { resources.put(classLoader, loadResources(fileName, classLoader)); countDownLatch.countDown(); }); @@ -99,6 +97,11 @@ public class ClassLoaderResourceLoader { return urlCache.get(fileName); } + public static void destroy() { + if (classLoaderResourcesCache != null) { + classLoaderResourcesCache.clear(); + } + } private static void setRef(URL url) { try { diff --git a/dubbo-common/src/main/java/org/apache/dubbo/config/ServiceConfigBase.java b/dubbo-common/src/main/java/org/apache/dubbo/config/ServiceConfigBase.java index 41d3dc7410..0d4ce7f4a3 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/config/ServiceConfigBase.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/config/ServiceConfigBase.java @@ -185,13 +185,14 @@ public abstract class ServiceConfigBase extends AbstractServiceConfig { } if (!interfaceClass.isInstance(ref)) { throw new IllegalStateException("The class " - + ref.getClass().getName() + getClassloaderDesc(ref.getClass()) + " unimplemented interface " - + interfaceClass + getClassloaderDesc(interfaceClass) + "!"); + + getClassDesc(ref.getClass()) + " unimplemented interface " + + getClassDesc(interfaceClass) + "!"); } } - private String getClassloaderDesc(Class clazz) { - return "[classloader=" + clazz.getClassLoader().getClass().getName() + "@" + clazz.getClassLoader().hashCode() + "]"; + private String getClassDesc(Class clazz) { + ClassLoader classLoader = clazz.getClassLoader(); + return clazz.getName() + "[classloader=" + classLoader.getClass().getName() + "@" + classLoader.hashCode() + "]"; } public Optional getContextPath(ProtocolConfig protocolConfig) { diff --git a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ApplicationModel.java b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ApplicationModel.java index 1d9f18f349..8fd6f0c15a 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ApplicationModel.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ApplicationModel.java @@ -201,13 +201,20 @@ public class ApplicationModel extends ScopeModel { } @Override - public void onDestroy() { - // TODO destroy application resources - for (ModuleModel moduleModel : new ArrayList<>(moduleModels)) { - moduleModel.destroy(); + protected void onDestroy() { + + if (deployer != null) { + deployer.preDestroy(); } - notifyDestroy(); + // destroy application resources + for (ModuleModel moduleModel : new ArrayList<>(moduleModels)) { + if (moduleModel != internalModule) { + moduleModel.destroy(); + } + } + // destroy internal module later + internalModule.destroy(); if (defaultInstance == this) { synchronized (ApplicationModel.class) { @@ -219,10 +226,12 @@ public class ApplicationModel extends ScopeModel { } if (deployer != null) { - deployer.destroy(); - deployer = null; + deployer.postDestroy(); } + // destroy other resources (e.g. ZookeeperTransporter ) + notifyDestroy(); + if (environment != null) { environment.destroy(); environment = null; @@ -235,6 +244,8 @@ public class ApplicationModel extends ScopeModel { serviceRepository.destroy(); serviceRepository = null; } + // try destroy framework if no any application + frameworkModel.tryDestroy(); } public FrameworkModel getFrameworkModel() { @@ -301,12 +312,13 @@ public class ApplicationModel extends ScopeModel { if (moduleModel == defaultModule) { defaultModule = findDefaultModule(); } - if (this.moduleModels.size() == 1 && this.moduleModels.get(0) == internalModule) { - this.internalModule.destroy(); - } - if (this.moduleModels.isEmpty()) { - destroy(); - } + } + } + + void tryDestroy() { + if (this.moduleModels.isEmpty() + || (this.moduleModels.size() == 1 && this.moduleModels.get(0) == internalModule)) { + destroy(); } } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/FrameworkModel.java b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/FrameworkModel.java index 6ed1917473..d979e28186 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/FrameworkModel.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/FrameworkModel.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.common.extension.ExtensionScope; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourcesRepository; import java.util.ArrayList; import java.util.Collections; @@ -49,7 +50,6 @@ public class FrameworkModel extends ScopeModel { private FrameworkServiceRepository serviceRepository; - public FrameworkModel() { super(null, ExtensionScope.FRAMEWORK); initialize(); @@ -70,20 +70,38 @@ public class FrameworkModel extends ScopeModel { } @Override - public void onDestroy() { - //TODO destroy framework model + protected void onDestroy() { + // destroy all application model for (ApplicationModel applicationModel : new ArrayList<>(applicationModels)) { applicationModel.destroy(); } - allInstances.remove(this); - if (defaultInstance == this) { - synchronized (FrameworkModel.class) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Dubbo framework[" + getInternalId() + "] is destroying ..."); + } + synchronized (FrameworkModel.class) { + allInstances.remove(this); + if (defaultInstance == this) { defaultInstance = null; } } + // notify destroy and clean framework resources + // see org.apache.dubbo.config.deploy.FrameworkModelCleaner notifyDestroy(); + + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Dubbo framework[" + getInternalId() + "] is destroyed"); + } + + // if all FrameworkModels are destroyed, clean global static resources, shutdown dubbo completely + if (allInstances.isEmpty()) { + destroyGlobalResources(); + } + } + + private void destroyGlobalResources() { + GlobalResourcesRepository.getInstance().destroy(); } public static FrameworkModel defaultModel() { @@ -120,6 +138,9 @@ public class FrameworkModel extends ScopeModel { synchronized void removeApplication(ApplicationModel model) { this.applicationModels.remove(model); + } + + synchronized void tryDestroy() { if (applicationModels.size() == 0) { destroy(); } diff --git a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ModuleModel.java b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ModuleModel.java index e11c1b5e4c..936261eda3 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ModuleModel.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ModuleModel.java @@ -81,14 +81,20 @@ public class ModuleModel extends ScopeModel { } @Override - public void onDestroy() { - notifyDestroy(); + protected void onDestroy() { + if (deployer != null) { + deployer.preDestroy(); + } + applicationModel.removeModule(this); if (deployer != null) { - deployer.destroy(); - deployer = null; + deployer.postDestroy(); } + + // destroy other resources + notifyDestroy(); + if (serviceRepository != null) { serviceRepository.destroy(); serviceRepository = null; @@ -98,6 +104,7 @@ public class ModuleModel extends ScopeModel { moduleEnvironment.destroy(); moduleEnvironment = null; } + applicationModel.tryDestroy(); } public ApplicationModel getApplicationModel() { diff --git a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModel.java b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModel.java index 1cc4712866..3e4cd9e108 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModel.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModel.java @@ -69,7 +69,6 @@ public abstract class ScopeModel implements ExtensionAccessor { private Map attributes; private AtomicBoolean destroyed = new AtomicBoolean(false); - private volatile boolean stopping; public ScopeModel(ScopeModel parent, ExtensionScope scope) { this.parent = parent; @@ -92,7 +91,6 @@ public abstract class ScopeModel implements ExtensionAccessor { this.destroyListeners = new LinkedList<>(); this.attributes = new ConcurrentHashMap<>(); this.classLoaders = new ConcurrentHashSet<>(); - this.stopping = false; // Add Framework's ClassLoader by default ClassLoader dubboClassLoader = ScopeModel.class.getClassLoader(); @@ -119,21 +117,13 @@ public abstract class ScopeModel implements ExtensionAccessor { return destroyed.get(); } - public void setStopping() { - stopping = true; - } - - public boolean isStopping() { - return stopping; - } - protected void notifyDestroy() { for (ScopeModelDestroyListener destroyListener : destroyListeners) { destroyListener.onDestroy(this); } } - public abstract void onDestroy(); + protected abstract void onDestroy(); public final void addDestroyListener(ScopeModelDestroyListener listener) { destroyListeners.add(listener); diff --git a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModelDestroyListener.java b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModelDestroyListener.java index 93f6e840f4..5b73ededf9 100644 --- a/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModelDestroyListener.java +++ b/dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModelDestroyListener.java @@ -16,6 +16,6 @@ */ package org.apache.dubbo.rpc.model; -public interface ScopeModelDestroyListener { - void onDestroy(ScopeModel scopeModel); +public interface ScopeModelDestroyListener { + void onDestroy(T scopeModel); } diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/config/configcenter/file/FileSystemDynamicConfigurationTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/config/configcenter/file/FileSystemDynamicConfigurationTest.java index 20e127a3f9..97f55d23d0 100644 --- a/dubbo-common/src/test/java/org/apache/dubbo/common/config/configcenter/file/FileSystemDynamicConfigurationTest.java +++ b/dubbo-common/src/test/java/org/apache/dubbo/common/config/configcenter/file/FileSystemDynamicConfigurationTest.java @@ -16,23 +16,22 @@ */ package org.apache.dubbo.common.config.configcenter.file; +import org.apache.commons.io.FileUtils; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; - -import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable; import java.io.File; +import java.io.IOException; import java.util.TreeSet; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Arrays.asList; -import static org.apache.commons.io.FileUtils.deleteQuietly; import static org.apache.dubbo.common.URL.valueOf; import static org.apache.dubbo.common.config.configcenter.DynamicConfiguration.DEFAULT_GROUP; import static org.apache.dubbo.common.config.configcenter.file.FileSystemDynamicConfiguration.CONFIG_CENTER_DIR_PARAM_NAME; @@ -60,13 +59,18 @@ public class FileSystemDynamicConfigurationTest { public void init() { File rootDirectory = new File(getClassPath(), "config-center"); rootDirectory.mkdirs(); + try { + FileUtils.cleanDirectory(rootDirectory); + } catch (IOException e) { + e.printStackTrace(); + } URL url = valueOf("dubbo://127.0.0.1:20880").addParameter(CONFIG_CENTER_DIR_PARAM_NAME, rootDirectory.getAbsolutePath()); configuration = new FileSystemDynamicConfiguration(url); } @AfterEach public void destroy() throws Exception { - deleteQuietly(configuration.getRootDirectory()); + FileUtils.deleteQuietly(configuration.getRootDirectory()); configuration.close(); } diff --git a/dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepositoryTest.java b/dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepositoryTest.java index 82202da18c..33ab9fb7a4 100644 --- a/dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepositoryTest.java +++ b/dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepositoryTest.java @@ -23,16 +23,12 @@ import org.junit.jupiter.api.Test; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicBoolean; public class ExecutorRepositoryTest { private ExecutorRepository executorRepository = ApplicationModel.defaultModel().getExtensionLoader(ExecutorRepository.class).getDefaultExtension(); @Test - public void test() { - testGetExecutor(); - testUpdateExecutor(); - } - public void testGetExecutor() { testGet(URL.valueOf("dubbo://127.0.0.1:23456")); testGet(URL.valueOf("dubbo://127.0.0.1:23456?side=consumer")); @@ -54,6 +50,7 @@ public class ExecutorRepositoryTest { Assertions.assertNotEquals(executorService, executorRepository.getExecutor(url)); } + @Test public void testUpdateExecutor() { URL url = URL.valueOf("dubbo://127.0.0.1:23456?threads=5"); ThreadPoolExecutor executorService = (ThreadPoolExecutor) executorRepository.createExecutorIfAbsent(url); @@ -75,7 +72,57 @@ public class ExecutorRepositoryTest { executorService.setCorePoolSize(5); executorRepository.updateThreadpool(url, executorService); + } + @Test + public void testSharedExecutor() throws Exception { + ExecutorService sharedExecutor = executorRepository.getSharedExecutor(); + MockTask task1 = new MockTask(2000); + MockTask task2 = new MockTask(100); + MockTask task3 = new MockTask(200); + sharedExecutor.execute(task1); + sharedExecutor.execute(task2); + sharedExecutor.submit(task3); + Thread.sleep(150); + Assertions.assertTrue(task1.isRunning()); + Assertions.assertFalse(task1.isDone()); + Assertions.assertTrue(task2.isRunning()); + Assertions.assertTrue(task2.isDone()); + Assertions.assertTrue(task3.isRunning()); + Assertions.assertFalse(task3.isDone()); + + Thread.sleep(200); + Assertions.assertTrue(task3.isDone()); + Assertions.assertFalse(task1.isDone()); + } + + private static class MockTask implements Runnable { + private long waitTimeMS; + private AtomicBoolean running = new AtomicBoolean(); + private AtomicBoolean done = new AtomicBoolean(); + + public MockTask(long waitTimeMS) { + this.waitTimeMS = waitTimeMS; + } + + @Override + public void run() { + running.set(true); + try { + Thread.sleep(waitTimeMS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + done.set(true); + } + + public boolean isDone() { + return done.get(); + } + + public boolean isRunning() { + return running.get(); + } } } diff --git a/dubbo-compatible/pom.xml b/dubbo-compatible/pom.xml index d12ddd94c4..8da57cca53 100644 --- a/dubbo-compatible/pom.xml +++ b/dubbo-compatible/pom.xml @@ -98,5 +98,11 @@ fastjson test + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigScopeModelInitializer.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigScopeModelInitializer.java index 8639cc2bc7..955da26d23 100644 --- a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigScopeModelInitializer.java +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigScopeModelInitializer.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.deploy.ApplicationDeployer; import org.apache.dubbo.common.deploy.ModuleDeployer; import org.apache.dubbo.config.deploy.DefaultApplicationDeployer; import org.apache.dubbo.config.deploy.DefaultModuleDeployer; +import org.apache.dubbo.config.deploy.FrameworkModelCleaner; import org.apache.dubbo.config.utils.DefaultConfigValidator; import org.apache.dubbo.rpc.model.ApplicationModel; import org.apache.dubbo.rpc.model.FrameworkModel; @@ -31,7 +32,7 @@ public class ConfigScopeModelInitializer implements ScopeModelInitializer { @Override public void initializeFrameworkModel(FrameworkModel frameworkModel) { - + frameworkModel.addDestroyListener(new FrameworkModelCleaner()); } @Override diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/DubboShutdownHook.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/DubboShutdownHook.java index 2ecba748b3..9fa94b24ca 100644 --- a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/DubboShutdownHook.java +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/DubboShutdownHook.java @@ -87,8 +87,8 @@ public class DubboShutdownHook extends Thread { */ public void unregister() { if (registered.compareAndSet(true, false)) { - if (this.isAlive() || destroyed.get()) { - // DubboShutdownHook is running + if (this.isAlive()) { + // DubboShutdownHook thread is running return; } try { diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java index be1d486833..663836a7e9 100644 --- a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java @@ -23,11 +23,11 @@ import org.apache.dubbo.common.deploy.ModuleDeployer; import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.threadpool.manager.ExecutorRepository; import org.apache.dubbo.common.url.component.ServiceConfigURL; import org.apache.dubbo.common.utils.ClassUtils; import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.common.utils.ConfigUtils; -import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.common.utils.StringUtils; import org.apache.dubbo.config.annotation.Service; import org.apache.dubbo.config.invoker.DelegateProviderMetaDataInvoker; @@ -57,8 +57,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -108,11 +106,6 @@ public class ServiceConfig extends ServiceConfigBase { */ private static final Map RANDOM_PORT_MAP = new HashMap(); - /** - * A delayed exposure service timer - */ - private static final ScheduledExecutorService DELAY_EXPORT_EXECUTOR = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboServiceDelayExporter", true)); - private Protocol protocolSPI; /** @@ -259,13 +252,14 @@ public class ServiceConfig extends ServiceConfigBase { } protected void doDelayExport() { - DELAY_EXPORT_EXECUTOR.schedule(() -> { - try { - doExport(); - } catch (Exception e) { - logger.error("Failed to export service config: " + interfaceName, e); - } - }, getDelay(), TimeUnit.MILLISECONDS); + getScopeModel().getDefaultExtension(ExecutorRepository.class).getServiceExportExecutor() + .schedule(() -> { + try { + doExport(); + } catch (Exception e) { + logger.error("Failed to export service config: " + interfaceName, e); + } + }, getDelay(), TimeUnit.MILLISECONDS); } protected void exported() { diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/bootstrap/DubboBootstrap.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/bootstrap/DubboBootstrap.java index 4477b41f68..201c7d37d4 100644 --- a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/bootstrap/DubboBootstrap.java +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/bootstrap/DubboBootstrap.java @@ -46,16 +46,14 @@ import org.apache.dubbo.config.bootstrap.builders.ReferenceBuilder; import org.apache.dubbo.config.bootstrap.builders.RegistryBuilder; import org.apache.dubbo.config.bootstrap.builders.ServiceBuilder; import org.apache.dubbo.config.context.ConfigManager; -import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory; -import org.apache.dubbo.rpc.Protocol; import org.apache.dubbo.rpc.model.ApplicationModel; import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.model.ModuleModel; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.Lock; @@ -127,10 +125,6 @@ public final class DubboBootstrap { return getInstance(frameworkModel.newApplication()); } - public static DubboBootstrap newInstance(ApplicationModel applicationModel) { - return getInstance(applicationModel); - } - /** * Try reset dubbo status for new instance. * @@ -153,8 +147,6 @@ public final class DubboBootstrap { instance.destroy(); instance = null; } - AbstractMetadataReportFactory.destroy(); - destroyAllProtocols(); FrameworkModel.destroyAll(); } else { instance = null; @@ -227,7 +219,7 @@ public final class DubboBootstrap { * @return */ public DubboBootstrap start(boolean wait) { - CompletableFuture future = applicationDeployer.start(); + Future future = applicationDeployer.start(); if (wait) { try { future.get(); @@ -238,6 +230,14 @@ public final class DubboBootstrap { return this; } + /** + * Start dubbo application but no wait for finish. + * @return the future object + */ + public Future asyncStart() { + return applicationDeployer.start(); + } + /** * Stop dubbo application * @return @@ -249,7 +249,7 @@ public final class DubboBootstrap { } public void destroy() { - applicationDeployer.destroy(); + applicationModel.destroy(); } public boolean isInitialized() { @@ -330,30 +330,6 @@ public final class DubboBootstrap { return applicationDeployer.getReferenceCache(); } - /** - * Destroy all the protocols. - */ - private static void destroyProtocols(FrameworkModel frameworkModel) { - //TODO destroy protocol in framework scope - ExtensionLoader loader = frameworkModel.getExtensionLoader(Protocol.class); - for (String protocolName : loader.getLoadedExtensions()) { - try { - Protocol protocol = loader.getLoadedExtension(protocolName); - if (protocol != null) { - protocol.destroy(); - } - } catch (Throwable t) { - logger.warn(t.getMessage(), t); - } - } - } - - private static void destroyAllProtocols() { - for (FrameworkModel frameworkModel : FrameworkModel.getAllInstances()) { - destroyProtocols(frameworkModel); - } - } - private void executeMutually(Runnable runnable) { try { lock.lock(); diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java index 61ac94a12c..81c718fc33 100644 --- a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java @@ -26,6 +26,7 @@ import org.apache.dubbo.common.config.configcenter.wrapper.CompositeDynamicConfi import org.apache.dubbo.common.deploy.AbstractDeployer; import org.apache.dubbo.common.deploy.ApplicationDeployListener; import org.apache.dubbo.common.deploy.ApplicationDeployer; +import org.apache.dubbo.common.deploy.DeployState; import org.apache.dubbo.common.deploy.ModuleDeployer; import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.common.lang.ShutdownHookCallbacks; @@ -48,16 +49,13 @@ import org.apache.dubbo.metadata.MetadataServiceExporter; import org.apache.dubbo.metadata.WritableMetadataService; import org.apache.dubbo.metadata.report.MetadataReportFactory; import org.apache.dubbo.metadata.report.MetadataReportInstance; -import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory; import org.apache.dubbo.registry.client.DefaultServiceInstance; import org.apache.dubbo.registry.client.ServiceInstance; import org.apache.dubbo.registry.client.metadata.ServiceInstanceMetadataUtils; import org.apache.dubbo.registry.client.metadata.store.InMemoryWritableMetadataService; import org.apache.dubbo.registry.client.metadata.store.RemoteMetadataServiceImpl; import org.apache.dubbo.registry.support.RegistryManager; -import org.apache.dubbo.rpc.Protocol; import org.apache.dubbo.rpc.model.ApplicationModel; -import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.model.ModuleModel; import org.apache.dubbo.rpc.model.ScopeModel; import org.apache.dubbo.rpc.model.ScopeModelUtil; @@ -70,6 +68,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -115,8 +114,9 @@ public class DefaultApplicationDeployer extends AbstractDeployer asyncMetadataFuture; private String identifier; - private CompletableFuture startFuture; + private volatile CompletableFuture startFuture; private DubboShutdownHook dubboShutdownHook; + private Object startedLock = new Object(); public DefaultApplicationDeployer(ApplicationModel applicationModel) { super(applicationModel); @@ -512,25 +512,26 @@ public class DefaultApplicationDeployer extends AbstractDeployer moduleModels = new ArrayList<>(applicationModel.getModuleModels()); - List futures = new ArrayList<>(moduleModels.size()); - - for (ModuleModel moduleModel : moduleModels) { - // export services in module + private boolean hasPendingModule() { + boolean found = false; + for (ModuleModel moduleModel : applicationModel.getModuleModels()) { if (moduleModel.getDeployer().isPending()) { - CompletableFuture moduleFuture = moduleModel.getDeployer().start(); - futures.add(moduleFuture); + found = true; + break; } } + return found; + } + + private CompletableFuture getStartFuture() { + if (startFuture == null) { + synchronized (this) { + if (startFuture == null) { + startFuture = new CompletableFuture(); + } + } + } + return startFuture; + } + + + private void doStart() { + startModules(); // prepare application instance prepareApplicationInstance(); - // notify on each module started -// executorRepository.getSharedExecutor().submit(()-> { -// awaitDeployFinished(futures); -// onStarted(); -// }); + executorRepository.getSharedExecutor().submit(() -> { + while (true) { + // notify on each module started + synchronized (startedLock) { + try { + startedLock.wait(500); + } catch (InterruptedException e) { + // ignore + } + } + + // if has new module, do start again + if (hasPendingModule()) { + startModules(); + continue; + } + + DeployState newState = checkState(); + if (!(newState == DeployState.STARTING || newState == DeployState.PENDING)) { + // start finished or error + break; + } + } + }); } - private void awaitDeployFinished(List futures) { - try { - CompletableFuture mergedFuture = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); - mergedFuture.get(); - } catch (Exception e) { - logger.error(getIdentifier() + " await deploy finished failed", e); + private void startModules() { + // copy current modules, ignore new module during starting + List moduleModels = new ArrayList<>(applicationModel.getModuleModels()); + for (ModuleModel moduleModel : moduleModels) { + // export services in module + if (moduleModel.getDeployer().isPending()) { + moduleModel.getDeployer().start(); + } } } @@ -595,8 +630,14 @@ public class DefaultApplicationDeployer extends AbstractDeployer { - InMemoryWritableMetadataService localMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(applicationModel); - if (!applicationModel.getDeployer().isStopping() || !applicationModel.getDeployer().isStopped()) { - localMetadataService.blockUntilUpdated(); + asyncMetadataFuture = executorRepository.getSharedScheduledExecutor().scheduleAtFixedRate(() -> { + + // ignore refresh metadata on stopping + if (applicationModel.isDestroyed()) { + return; } + + InMemoryWritableMetadataService localMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(applicationModel); + localMetadataService.blockUntilUpdated(); try { - ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance); + if (!applicationModel.isDestroyed()) { + ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance); + } } catch (Exception e) { - logger.error("Refresh instance and metadata error", e); + if (!applicationModel.isDestroyed()) { + logger.error("Refresh instance and metadata error", e); + } } finally { localMetadataService.releaseBlock(); } @@ -780,36 +829,45 @@ public class DefaultApplicationDeployer extends AbstractDeployer 0) { + if (pending + starting + stopping + stopped == 0) { + // all modules have been started + newState = DeployState.STARTED; + } else if (pending + starting > 0) { + // some module is pending and some is started + newState = DeployState.STARTING; + } else if (stopping + stopped > 0) { + newState = DeployState.STOPPING; + } + } else if (starting > 0) { + // any module is starting + newState = DeployState.STARTING; + } else if (pending > 0) { + if (starting + starting + stopping + stopped == 0) { + // all modules have not starting or started + newState = DeployState.PENDING; + } else if (stopping + stopped > 0) { + // some is pending and some is stopping or stopped + newState = DeployState.STOPPING; + } + } else if (stopping > 0) { + // some is stopping and some stopped + newState = DeployState.STOPPING; + } else if (stopped > 0) { + // all modules are stopped + newState = DeployState.STOPPED; + } + return newState; } private void onStarting() { + if (isStarting()) { + return; + } setStarting(); if (logger.isInfoEnabled()) { logger.info(getIdentifier() + " is starting."); } } - private void onStarted(CompletableFuture checkerStartFuture) { + private void onStarted() { + if (isStarted()) { + return; + } setStarted(); if (logger.isInfoEnabled()) { logger.info(getIdentifier() + " is ready."); } - if (startFuture != null) { - startFuture.complete(true); + // refresh metadata + try { + if (serviceInstance != null) { + ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance); + } + } catch (Exception e) { + logger.error("refresh metadata failed: " + e.getMessage(), e); } - if (checkerStartFuture != null) { - checkerStartFuture.complete(true); + // complete future + completeStartFuture(true); + // shutdown export/refer executor after started + executorRepository.shutdownServiceExportExecutor(); + executorRepository.shutdownServiceReferExecutor(); + } + + private void completeStartFuture(boolean success) { + if (startFuture != null) { + startFuture.complete(success); + startFuture = null; } } private void onStopping() { - applicationModel.setStopping(); + if (isStopping()) { + return; + } setStopping(); if (logger.isInfoEnabled()) { logger.info(getIdentifier() + " is stopping."); @@ -873,6 +1016,9 @@ public class DefaultApplicationDeployer extends AbstractDeployer loader = frameworkModel.getExtensionLoader(Protocol.class); - for (String protocolName : loader.getLoadedExtensions()) { - try { - Protocol protocol = loader.getLoadedExtension(protocolName); - if (protocol != null) { - protocol.destroy(); - } - } catch (Throwable t) { - logger.warn(t.getMessage(), t); - } - } - } - } - private void destroyServiceDiscoveries() { RegistryManager.getInstance(applicationModel).getServiceDiscoveries().forEach(serviceDiscovery -> { try { @@ -923,15 +1048,11 @@ public class DefaultApplicationDeployer extends AbstractDeployer metadataReportFactories = getExtensionLoader(MetadataReportFactory.class).getLoadedExtensionInstances(); + for (MetadataReportFactory metadataReportFactory : metadataReportFactories) { + metadataReportFactory.destroy(); + } } private ApplicationConfig getApplication() { @@ -940,10 +1061,10 @@ public class DefaultApplicationDeployer extends AbstractDeployer impleme } @Override - public synchronized CompletableFuture start() throws IllegalStateException { + public synchronized Future start() throws IllegalStateException { if (isStarting() || isStarted()) { return startFuture; } @@ -164,15 +165,22 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme @Override public void stop() throws IllegalStateException { - destroy(); + moduleModel.destroy(); } @Override - public synchronized void destroy() throws IllegalStateException { + public void preDestroy() throws IllegalStateException { if (isStopping() || isStopped()) { return; } onModuleStopping(); + } + + @Override + public synchronized void postDestroy() throws IllegalStateException { + if (isStopped()) { + return; + } unexportServices(); unreferServices(); @@ -206,7 +214,6 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme } serviceRepository.destroy(); } - moduleModel.destroy(); onModuleStopped(); } @@ -219,7 +226,9 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme private void onModuleStarted(CompletableFuture startFuture) { setStarted(); logger.info(getIdentifier() + " has started."); - applicationDeployer.checkStarted(startFuture); + applicationDeployer.checkStarted(); + // complete module start future after application state changed, fix #9012 ? + startFuture.complete(true); } private void onModuleStopping() { @@ -345,7 +354,6 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme } catch (Exception e) { logger.warn(getIdentifier() + " export services occurred an exception."); } finally { - executorRepository.shutdownServiceExportExecutor(); logger.info(getIdentifier() + " export services finished."); asyncExportingFutures.clear(); } @@ -359,7 +367,6 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme } catch (Exception e) { logger.warn(getIdentifier() + " refer services occurred an exception."); } finally { - executorRepository.shutdownServiceReferExecutor(); logger.info(getIdentifier() + " refer services finished."); asyncReferringFutures.clear(); } @@ -390,10 +397,10 @@ public class DefaultModuleDeployer extends AbstractDeployer impleme private String getIdentifier() { if (identifier == null) { - if (moduleModel.getModelName() != null && !StringUtils.isEquals(moduleModel.getModelName(), moduleModel.getInternalName())) { - identifier = moduleModel.getModelName() + "[" + moduleModel.getInternalId() + "]"; - } else { - identifier = "Dubbo Module" + "[" + moduleModel.getInternalId() + "]"; + identifier = "Dubbo module[" + moduleModel.getInternalId() + "]"; + if (moduleModel.getModelName() != null + && !StringUtils.isEquals(moduleModel.getModelName(), moduleModel.getInternalName())) { + identifier += "(" + moduleModel.getModelName() + ")"; } } return identifier; diff --git a/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/FrameworkModelCleaner.java b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/FrameworkModelCleaner.java new file mode 100644 index 0000000000..cf926d0466 --- /dev/null +++ b/dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/FrameworkModelCleaner.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.config.deploy; + +import org.apache.dubbo.common.extension.ExtensionLoader; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.rpc.Protocol; +import org.apache.dubbo.rpc.model.FrameworkModel; +import org.apache.dubbo.rpc.model.ScopeModelDestroyListener; + +/** + * A cleaner to release resources of framework model + */ +public class FrameworkModelCleaner implements ScopeModelDestroyListener { + + private static final Logger logger = LoggerFactory.getLogger(FrameworkModelCleaner.class); + + @Override + public void onDestroy(FrameworkModel frameworkModel) { + destroyFrameworkResources(frameworkModel); + } + + /** + * Destroy all framework resources. + */ + private void destroyFrameworkResources(FrameworkModel frameworkModel) { + // destroy protocol in framework scope + destroyProtocols(frameworkModel); + } + + /** + * Destroy all the protocols. + */ + private void destroyProtocols(FrameworkModel frameworkModel) { + ExtensionLoader loader = frameworkModel.getExtensionLoader(Protocol.class); + for (String protocolName : loader.getLoadedExtensions()) { + try { + Protocol protocol = loader.getLoadedExtension(protocolName); + if (protocol != null) { + protocol.destroy(); + } + } catch (Throwable t) { + logger.warn(t.getMessage(), t); + } + } + } + + +} diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractMethodConfigTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractMethodConfigTest.java index 5b82bc8831..ceb63d5827 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractMethodConfigTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractMethodConfigTest.java @@ -16,6 +16,8 @@ */ package org.apache.dubbo.config; +import org.apache.dubbo.config.bootstrap.DubboBootstrap; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; import java.util.HashMap; @@ -28,6 +30,12 @@ import static org.hamcrest.Matchers.isEmptyOrNullString; import static org.hamcrest.Matchers.sameInstance; public class AbstractMethodConfigTest { + + @AfterAll + public static void afterAll() { + DubboBootstrap.reset(); + } + @Test public void testTimeout() throws Exception { MethodConfig methodConfig = new MethodConfig(); diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractReferenceConfigTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractReferenceConfigTest.java index 2966322a56..f4dd078452 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractReferenceConfigTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractReferenceConfigTest.java @@ -24,6 +24,8 @@ import org.apache.dubbo.rpc.cluster.RouterFactory; import org.apache.dubbo.rpc.cluster.router.condition.ConditionRouterFactory; import org.apache.dubbo.rpc.cluster.router.condition.config.AppRouterFactory; import org.apache.dubbo.rpc.cluster.router.tag.TagRouterFactory; +import org.apache.dubbo.rpc.model.FrameworkModel; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -47,6 +49,11 @@ import static org.mockito.Mockito.when; public class AbstractReferenceConfigTest { + @AfterAll + public static void afterAll() throws Exception { + FrameworkModel.destroyAll(); + } + @Test public void testCheck() throws Exception { ReferenceConfig referenceConfig = new ReferenceConfig(); diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ConfigCenterConfigTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ConfigCenterConfigTest.java index 5ee50cc4d3..d006c0f90b 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ConfigCenterConfigTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ConfigCenterConfigTest.java @@ -91,6 +91,7 @@ public class ConfigCenterConfigTest { .initialize(); } catch (Exception e) { // ignore + e.printStackTrace(); } Collection configCenters = ApplicationModel.defaultModel().getApplicationConfigManager().getConfigCenters(); @@ -100,6 +101,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { SysProps.clear(); + DubboBootstrap.getInstance().stop(); } } @@ -128,6 +130,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { SysProps.clear(); + DubboBootstrap.getInstance().stop(); } } @@ -155,6 +158,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { SysProps.clear(); + DubboBootstrap.getInstance().stop(); } } @@ -183,6 +187,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { ApplicationModel.defaultModel().getModelEnvironment().getPropertiesConfiguration().refresh(); + DubboBootstrap.getInstance().stop(); } } @@ -211,6 +216,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { SysProps.clear(); + DubboBootstrap.getInstance().stop(); } } @@ -240,6 +246,7 @@ public class ConfigCenterConfigTest { Assertions.assertEquals(false, configCenter.isCheck()); } finally { ApplicationModel.defaultModel().getModelEnvironment().getPropertiesConfiguration().refresh(); + DubboBootstrap.getInstance().stop(); } } diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProtocolConfigTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProtocolConfigTest.java index 11aea9ddcf..b9fe5502f9 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProtocolConfigTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProtocolConfigTest.java @@ -20,6 +20,7 @@ package org.apache.dubbo.config; import org.apache.dubbo.common.utils.NetUtils; import org.apache.dubbo.config.bootstrap.DubboBootstrap; import org.apache.dubbo.config.context.ConfigManager; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -31,7 +32,9 @@ import java.util.HashMap; import java.util.Map; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.is; public class ProtocolConfigTest { @@ -45,6 +48,11 @@ public class ProtocolConfigTest { SysProps.clear(); } + @AfterAll + public static void afterAll() { + DubboBootstrap.reset(); + } + @Test public void testName() throws Exception { ProtocolConfig protocol = new ProtocolConfig(); @@ -245,6 +253,7 @@ public class ProtocolConfigTest { Assertions.assertEquals("rest", protocolConfig.getName()); Assertions.assertEquals(port, protocolConfig.getPort()); } finally { + DubboBootstrap.getInstance().stop(); } } @@ -265,6 +274,7 @@ public class ProtocolConfigTest { Assertions.assertEquals("rest", protocolConfig.getName()); Assertions.assertEquals(port, protocolConfig.getPort()); } finally { + DubboBootstrap.getInstance().stop(); } } @@ -287,6 +297,7 @@ public class ProtocolConfigTest { Assertions.assertEquals("rest", protocolConfig.getName()); Assertions.assertEquals(port, protocolConfig.getPort()); } finally { + DubboBootstrap.getInstance().stop(); } } @@ -314,6 +325,7 @@ public class ProtocolConfigTest { Assertions.assertEquals("rest", protocol.getName()); Assertions.assertEquals(port1, protocol.getPort()); } finally { + DubboBootstrap.getInstance().stop(); } } @@ -340,6 +352,7 @@ public class ProtocolConfigTest { Assertions.assertEquals("rest", protocol.getName()); Assertions.assertEquals(port1, protocol.getPort()); } finally { + DubboBootstrap.getInstance().stop(); } } diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProviderConfigTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProviderConfigTest.java index 30143ed274..78585cc22b 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProviderConfigTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProviderConfigTest.java @@ -31,6 +31,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; public class ProviderConfigTest { + @Test public void testProtocol() throws Exception { ProviderConfig provider = new ProviderConfig(); diff --git a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/bootstrap/DubboBootstrapMultiInstanceTest.java b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/bootstrap/DubboBootstrapMultiInstanceTest.java index 12b27964c7..125737f52d 100644 --- a/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/bootstrap/DubboBootstrapMultiInstanceTest.java +++ b/dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/bootstrap/DubboBootstrapMultiInstanceTest.java @@ -16,6 +16,8 @@ */ package org.apache.dubbo.config.bootstrap; +import org.apache.dubbo.common.deploy.ApplicationDeployer; +import org.apache.dubbo.common.deploy.DeployState; import org.apache.dubbo.common.deploy.ModuleDeployer; import org.apache.dubbo.common.utils.NetUtils; import org.apache.dubbo.common.utils.StringUtils; @@ -34,6 +36,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel; import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.model.FrameworkServiceRepository; import org.apache.dubbo.rpc.model.ModuleModel; +import org.apache.dubbo.test.check.DubboTestChecker; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -41,9 +44,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.util.HashMap; import java.util.List; +import java.util.Map; -import static org.apache.dubbo.metadata.MetadataConstants.METADATA_PUBLISH_DELAY_KEY; +import static org.apache.dubbo.remoting.Constants.EVENT_LOOP_BOSS_POOL_NAME; public class DubboBootstrapMultiInstanceTest { @@ -51,8 +57,12 @@ public class DubboBootstrapMultiInstanceTest { private static RegistryConfig registryConfig; + private static DubboTestChecker testChecker; + private static String testClassName; + @BeforeAll public static void beforeAll() { + FrameworkModel.destroyAll(); registryCenter = new ZookeeperSingleRegistryCenter(NetUtils.getAvailablePort()); registryCenter.startup(); RegistryCenter.Instance instance = registryCenter.getRegistryCenterInstance().get(0); @@ -60,11 +70,45 @@ public class DubboBootstrapMultiInstanceTest { instance.getType(), instance.getHostname(), instance.getPort())); + + // pre-check threads + //precheckUnclosedThreads(); } @AfterAll - public static void afterAll() { + public static void afterAll() throws Exception { registryCenter.shutdown(); + FrameworkModel.destroyAll(); + + // check threads + //checkUnclosedThreads(); + } + + private static Map precheckUnclosedThreads() throws IOException { + // create a special DubboTestChecker + if (testChecker == null) { + testChecker = new DubboTestChecker(); + testChecker.init(null); + testClassName = DubboBootstrapMultiInstanceTest.class.getName(); + } + return testChecker.checkUnclosedThreads(testClassName, 0); + } + + private static void checkUnclosedThreads() { + Map unclosedThreadMap = testChecker.checkUnclosedThreads(testClassName, 3000); + if (unclosedThreadMap.size() > 0) { + String str = getStackTraceString(unclosedThreadMap); + Assertions.fail("Found unclosed threads: " + unclosedThreadMap.size()+"\n" + str); + } + } + + private static String getStackTraceString(Map unclosedThreadMap) { + StringBuilder sb = new StringBuilder(); + for (Thread thread : unclosedThreadMap.keySet()) { + sb.append(DubboTestChecker.getFullStacktrace(thread, unclosedThreadMap.get(thread))); + sb.append("\n"); + } + return sb.toString(); } @BeforeEach @@ -173,7 +217,7 @@ public class DubboBootstrapMultiInstanceTest { @Test public void testMultiModuleApplication() throws InterruptedException { - SysProps.setProperty(METADATA_PUBLISH_DELAY_KEY, "1"); + //SysProps.setProperty(METADATA_PUBLISH_DELAY_KEY, "100"); String version1 = "1.0"; String version2 = "2.0"; String version3 = "3.0"; @@ -224,7 +268,7 @@ public class DubboBootstrapMultiInstanceTest { providerBootstrap.start(); - Thread.sleep(100); + //Thread.sleep(200); // consumer app consumerBootstrap = DubboBootstrap.newInstance(); @@ -257,7 +301,134 @@ public class DubboBootstrapMultiInstanceTest { consumerBootstrap.destroy(); } } + } + @Test + public void testMultiProviderApplicationsStopOneByOne() { + + String version1 = "1.0"; + String version2 = "2.0"; + + DubboBootstrap providerBootstrap1 = null; + DubboBootstrap providerBootstrap2 = null; + ZookeeperSingleRegistryCenter registryCenter2 = null; + + try { + + // save threads before provider app 1 + Map stackTraces0 = Thread.getAllStackTraces(); + + // start provider app 1 + ServiceConfig serviceConfig1 = new ServiceConfig(); + serviceConfig1.setInterface(DemoService.class); + serviceConfig1.setRef(new DemoServiceImpl()); + serviceConfig1.setVersion(version1); + + ProtocolConfig protocolConfig1 = new ProtocolConfig("dubbo", NetUtils.getAvailablePort()); + + providerBootstrap1 = DubboBootstrap.getInstance(); + providerBootstrap1.application("provider1") + .registry(registryConfig) + .service(serviceConfig1) + .protocol(protocolConfig1) + .start(); + + // save threads of provider app 1 + Map lastAllThreadStackTraces = Thread.getAllStackTraces(); + Map stackTraces1 = findNewThreads(lastAllThreadStackTraces, stackTraces0); + Assertions.assertTrue(stackTraces1.size() > 0, "Get threads of provider app 1 failed"); + + // start zk server 2 + registryCenter2 = new ZookeeperSingleRegistryCenter(NetUtils.getAvailablePort()); + registryCenter2.startup(); + RegistryCenter.Instance instance = registryCenter2.getRegistryCenterInstance().get(0); + RegistryConfig registryConfig2 = new RegistryConfig(String.format("%s://%s:%s", + instance.getType(), + instance.getHostname(), + instance.getPort())); + + // start provider app 2 use a difference zk server 2 + ServiceConfig serviceConfig2 = new ServiceConfig(); + serviceConfig2.setInterface(DemoService.class); + serviceConfig2.setRef(new DemoServiceImpl()); + serviceConfig2.setVersion(version2); + + ProtocolConfig protocolConfig2 = new ProtocolConfig("dubbo", NetUtils.getAvailablePort()); + + providerBootstrap2 = DubboBootstrap.newInstance(); + providerBootstrap2.application("provider2") + .registry(registryConfig2) + .service(serviceConfig2) + .protocol(protocolConfig2) + .start(); + + // save threads of provider app 2 + Map stackTraces2 = findNewThreads(Thread.getAllStackTraces(), stackTraces0); + Assertions.assertTrue(stackTraces2.size() > 0, "Get threads of provider app 2 failed"); + + // stop provider app 1 and check threads + providerBootstrap1.stop(); + + // TODO Remove ignore thread prefix of NettyServerBoss if supporting close protocol server only used by one application + // see org.apache.dubbo.config.deploy.DefaultApplicationDeployer.postDestroy + // NettyServer will close when all applications are shutdown, but not close if any application of the framework is alive, just ignore it currently + checkUnclosedThreadsOfApp(stackTraces1, "Found unclosed threads of app 1: ", new String[]{EVENT_LOOP_BOSS_POOL_NAME, "Dubbo-global-shared-handler"}); + + + // stop provider app 2 and check threads + providerBootstrap2.stop(); + // shutdown register center after dubbo application to avoid unregister services blocking + registryCenter2.shutdown(); + checkUnclosedThreadsOfApp(stackTraces2, "Found unclosed threads of app 2: ", null); + + } finally { + if (providerBootstrap1 != null) { + providerBootstrap1.stop(); + } + if (providerBootstrap2 != null) { + providerBootstrap2.stop(); + } + if (registryCenter2 != null) { + registryCenter2.shutdown(); + } + } + } + + private Map findNewThreads(Map newAllThreadMap, Map prevThreadMap) { + Map deltaThreadMap = new HashMap<>(newAllThreadMap); + deltaThreadMap.keySet().removeAll(prevThreadMap.keySet()); + // expect deltaThreadMap not contains any elements of prevThreadMap + Assertions.assertFalse(deltaThreadMap.keySet().stream().filter(thread -> prevThreadMap.containsKey(thread)).findAny().isPresent()); + return deltaThreadMap; + } + + private void checkUnclosedThreadsOfApp(Map stackTraces1, String msg, String[] ignoredThreadPrefixes) { + int waitTimeMs = 5000; + System.out.println("Wait "+waitTimeMs+"ms to check threads of app ..."); + try { + Thread.sleep(waitTimeMs); + } catch (InterruptedException e) { + } + HashMap unclosedThreadMap1 = new HashMap<>(stackTraces1); + unclosedThreadMap1.keySet().removeIf(thread -> !thread.isAlive()); + if (ignoredThreadPrefixes!= null && ignoredThreadPrefixes.length > 0) { + unclosedThreadMap1.keySet().removeIf(thread -> isIgnoredThread(thread.getName(), ignoredThreadPrefixes)); + } + if (unclosedThreadMap1.size() > 0) { + String str = getStackTraceString(unclosedThreadMap1); + Assertions.fail(msg + unclosedThreadMap1.size()+"\n" + str); + } + } + + private boolean isIgnoredThread(String name, String[] ignoredThreadPrefixes) { + if (ignoredThreadPrefixes!= null && ignoredThreadPrefixes.length > 0) { + for (String prefix : ignoredThreadPrefixes) { + if (name.startsWith(prefix)) { + return true; + } + } + } + return false; } @Test @@ -402,6 +573,142 @@ public class DubboBootstrapMultiInstanceTest { } } + @Test + public void testBothStartByModuleAndByApplication() throws Exception { + String version1 = "1.0"; + String version2 = "2.0"; + String version3 = "3.0"; + + String serviceKey1 = DemoService.class.getName() + ":" + version1; + String serviceKey2 = DemoService.class.getName() + ":" + version2; + String serviceKey3 = DemoService.class.getName() + ":" + version3; + + // provider app + DubboBootstrap providerBootstrap = null; + try { + providerBootstrap = DubboBootstrap.newInstance(); + + ServiceConfig serviceConfig1 = new ServiceConfig(); + serviceConfig1.setInterface(DemoService.class); + serviceConfig1.setRef(new DemoServiceImpl()); + serviceConfig1.setVersion(version1); + + //provider module 1 + providerBootstrap + .application("provider-app") + .registry(registryConfig) + .protocol(new ProtocolConfig("dubbo", -1)) + .service(builder -> builder + .interfaceClass(Greeting.class) + .ref(new GreetingLocal2())) + .newModule() + .service(serviceConfig1) + .endModule(); + + // 1. start module1 + ModuleDeployer moduleDeployer1 = serviceConfig1.getScopeModel().getDeployer(); + moduleDeployer1.start().get(); + Assertions.assertEquals(DeployState.STARTED, moduleDeployer1.getState()); + + ApplicationModel applicationModel = providerBootstrap.getApplicationModel(); + ApplicationDeployer applicationDeployer = applicationModel.getDeployer(); + Assertions.assertEquals(DeployState.STARTING, applicationDeployer.getState()); + ModuleModel defaultModule = applicationModel.getDefaultModule(); + Assertions.assertEquals(DeployState.PENDING, defaultModule.getDeployer().getState()); + + // 2. start application after module1 is started + providerBootstrap.start(); + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(DeployState.STARTED, defaultModule.getDeployer().getState()); + + // 3. add module2 and re-start application + ServiceConfig serviceConfig2 = new ServiceConfig(); + serviceConfig2.setInterface(DemoService.class); + serviceConfig2.setRef(new DemoServiceImpl()); + serviceConfig2.setVersion(version2); + ModuleModel moduleModel2 = providerBootstrap.newModule() + .service(serviceConfig2) + .getModuleModel(); + providerBootstrap.start(); + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(DeployState.STARTED, moduleModel2.getDeployer().getState()); + + // 4. add module3 and start module3 + ServiceConfig serviceConfig3 = new ServiceConfig(); + serviceConfig3.setInterface(DemoService.class); + serviceConfig3.setRef(new DemoServiceImpl()); + serviceConfig3.setVersion(version3); + ModuleModel moduleModel3 = providerBootstrap.newModule() + .service(serviceConfig3) + .getModuleModel(); + moduleModel3.getDeployer().start().get(); + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(DeployState.STARTED, moduleModel3.getDeployer().getState()); + + } finally { + if (providerBootstrap != null) { + providerBootstrap.stop(); + } + } + } + + + @Test + public void testBothStartByModuleAndByApplication2() throws Exception { + String version1 = "1.0"; + String version2 = "2.0"; + String version3 = "3.0"; + + String serviceKey1 = DemoService.class.getName() + ":" + version1; + String serviceKey2 = DemoService.class.getName() + ":" + version2; + String serviceKey3 = DemoService.class.getName() + ":" + version3; + + // provider app + DubboBootstrap providerBootstrap = null; + try { + providerBootstrap = DubboBootstrap.newInstance(); + + ServiceConfig serviceConfig1 = new ServiceConfig(); + serviceConfig1.setInterface(DemoService.class); + serviceConfig1.setRef(new DemoServiceImpl()); + serviceConfig1.setVersion(version1); + + //provider module 1 + providerBootstrap + .application("provider-app") + .registry(registryConfig) + .protocol(new ProtocolConfig("dubbo", -1)) + .service(builder -> builder + .interfaceClass(Greeting.class) + .ref(new GreetingLocal2())) + .newModule() + .service(serviceConfig1) + .endModule(); + + // 1. start module1 but no wait + ModuleDeployer moduleDeployer1 = serviceConfig1.getScopeModel().getDeployer(); + moduleDeployer1.start(); + Assertions.assertEquals(DeployState.STARTING, moduleDeployer1.getState()); + + ApplicationModel applicationModel = providerBootstrap.getApplicationModel(); + ApplicationDeployer applicationDeployer = applicationModel.getDeployer(); + Assertions.assertEquals(DeployState.STARTING, applicationDeployer.getState()); + ModuleModel defaultModule = applicationModel.getDefaultModule(); + Assertions.assertEquals(DeployState.PENDING, defaultModule.getDeployer().getState()); + + // 2. start application after module1 is starting + providerBootstrap.start(); + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(DeployState.STARTED, moduleDeployer1.getState()); + Assertions.assertEquals(DeployState.STARTED, defaultModule.getDeployer().getState()); + + } finally { + if (providerBootstrap != null) { + providerBootstrap.stop(); + } + } + } + private DubboBootstrap configConsumerApp(DubboBootstrap dubboBootstrap) { ReferenceConfig referenceConfig = new ReferenceConfig<>(); diff --git a/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/context/DubboDeployApplicationListener.java b/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/context/DubboDeployApplicationListener.java index 249c2e5fad..918e0b6f70 100644 --- a/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/context/DubboDeployApplicationListener.java +++ b/dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/context/DubboDeployApplicationListener.java @@ -36,7 +36,7 @@ import org.springframework.context.event.ContextClosedEvent; import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.core.Ordered; -import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; /** * An ApplicationListener to control Dubbo application. @@ -105,7 +105,7 @@ public class DubboDeployApplicationListener implements ApplicationListener { context.setKeepRunningOnSpringClosed(true); }); + ClassPathXmlApplicationContext providerContext = null; try { - ClassPathXmlApplicationContext providerContext; String resourcePath = "org/apache/dubbo/config/spring"; providerContext = new ClassPathXmlApplicationContext( resourcePath + "/demo-provider.xml", resourcePath + "/demo-provider-properties.xml"); providerContext.start(); - DubboStateListener listener = providerContext.getBean(DubboStateListener.class); - for (int i = 0; i < 10; i++) { - if (DeployState.STARTED.equals(listener.getState())) { - break; - } - Thread.sleep(100); - } + // Expect 1: dubbo application state is STARTED after spring context start finish. + // No need check and wait + + DubboStateListener dubboStateListener = providerContext.getBean(DubboStateListener.class); + Assertions.assertEquals(DeployState.STARTED, dubboStateListener.getState()); ModuleModel moduleModel = providerContext.getBean(ModuleModel.class); - Assertions.assertTrue(moduleModel.getDeployer().isStarted()); - Assertions.assertEquals(true, DubboBootstrap.getInstance().isStarted()); - Assertions.assertEquals(false, DubboBootstrap.getInstance().isStopped()); + ModuleDeployer moduleDeployer = moduleModel.getDeployer(); + Assertions.assertTrue(moduleDeployer.isStarted()); + + ApplicationDeployer applicationDeployer = moduleModel.getApplicationModel().getDeployer(); + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(true, applicationDeployer.isStarted()); + Assertions.assertEquals(false, applicationDeployer.isStopped()); // close spring context providerContext.close(); - // expect dubbo bootstrap will not be destroyed after closing spring context - Assertions.assertEquals(true, DubboBootstrap.getInstance().isStarted()); - Assertions.assertEquals(false, DubboBootstrap.getInstance().isStopped()); + // Expect 2: dubbo application will not be destroyed after closing spring context cause setKeepRunningOnSpringClosed(true) + Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState()); + Assertions.assertEquals(true, applicationDeployer.isStarted()); + Assertions.assertEquals(false, applicationDeployer.isStopped()); } finally { DubboBootstrap.getInstance().stop(); SysProps.clear(); + if (providerContext != null) { + providerContext.close(); + } } } } diff --git a/dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/reference/javaconfig/JavaConfigReferenceBeanTest.java b/dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/reference/javaconfig/JavaConfigReferenceBeanTest.java index ad1797528a..a4ec239667 100644 --- a/dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/reference/javaconfig/JavaConfigReferenceBeanTest.java +++ b/dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/reference/javaconfig/JavaConfigReferenceBeanTest.java @@ -16,6 +16,7 @@ */ package org.apache.dubbo.config.spring.reference.javaconfig; +import com.sun.management.HotSpotDiagnosticMXBean; import org.apache.dubbo.config.annotation.DubboReference; import org.apache.dubbo.config.annotation.DubboService; import org.apache.dubbo.config.annotation.Reference; @@ -42,8 +43,10 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; +import javax.management.MBeanServer; import java.io.PrintWriter; import java.io.StringWriter; +import java.lang.management.ManagementFactory; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -59,8 +62,33 @@ public class JavaConfigReferenceBeanTest { } @AfterAll - public static void afterAll() { + public static void afterAll() throws Exception { multipleRegistryCenter.shutdown(); + multipleRegistryCenter = null; + + //Thread.sleep(10000); + //dumpHeap("/Users/gongdewei/work/dump/dubbo/", true); + } + + public static void dumpHeap(String dirpath, boolean live) throws Exception { + + java.lang.management.RuntimeMXBean runtime = + java.lang.management.ManagementFactory.getRuntimeMXBean(); + java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm"); + jvm.setAccessible(true); + sun.management.VMManagement mgmt = + (sun.management.VMManagement) jvm.get(runtime); + java.lang.reflect.Method pid_method = + mgmt.getClass().getDeclaredMethod("getProcessId"); + pid_method.setAccessible(true); + int pid = (Integer) pid_method.invoke(mgmt); + + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + String filepath = dirpath + pid + ".hprof"; + mxBean.dumpHeap(filepath, live); + System.out.println("Dump heap to file: " + filepath); } @BeforeEach @@ -78,19 +106,21 @@ public class JavaConfigReferenceBeanTest { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class, AnnotationAtFieldConfiguration.class); - Map helloServiceMap = context.getBeansOfType(HelloService.class); - Assertions.assertEquals(2, helloServiceMap.size()); - Assertions.assertNotNull(helloServiceMap.get("helloService")); - Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); + try { + Map helloServiceMap = context.getBeansOfType(HelloService.class); + Assertions.assertEquals(2, helloServiceMap.size()); + Assertions.assertNotNull(helloServiceMap.get("helloService")); + Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); - Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); - Assertions.assertEquals(1, referenceBeanMap.size()); - ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); - Assertions.assertEquals("demo", referenceBean.getGroup()); - Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); - Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); - - context.close(); + Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); + Assertions.assertEquals(1, referenceBeanMap.size()); + ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); + Assertions.assertEquals("demo", referenceBean.getGroup()); + Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); + Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); + } finally { + context.close(); + } } @Test @@ -122,19 +152,21 @@ public class JavaConfigReferenceBeanTest { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class, AnnotationBeanConfiguration.class); - Map helloServiceMap = context.getBeansOfType(HelloService.class); - Assertions.assertEquals(2, helloServiceMap.size()); - Assertions.assertNotNull(helloServiceMap.get("helloService")); - Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); + try { + Map helloServiceMap = context.getBeansOfType(HelloService.class); + Assertions.assertEquals(2, helloServiceMap.size()); + Assertions.assertNotNull(helloServiceMap.get("helloService")); + Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); - Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); - Assertions.assertEquals(1, referenceBeanMap.size()); - ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); - Assertions.assertEquals("demo", referenceBean.getGroup()); - Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); - Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); - - context.close(); + Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); + Assertions.assertEquals(1, referenceBeanMap.size()); + ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); + Assertions.assertEquals("demo", referenceBean.getGroup()); + Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); + Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); + } finally { + context.close(); + } } @Test @@ -142,33 +174,36 @@ public class JavaConfigReferenceBeanTest { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class, GenericServiceAnnotationBeanConfiguration.class); - Map helloServiceMap = context.getBeansOfType(HelloService.class); - Assertions.assertEquals(1, helloServiceMap.size()); - Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); + try { + Map helloServiceMap = context.getBeansOfType(HelloService.class); + Assertions.assertEquals(1, helloServiceMap.size()); + Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); - Map genericServiceMap = context.getBeansOfType(GenericService.class); - Assertions.assertEquals(3, genericServiceMap.size()); - Assertions.assertNotNull(genericServiceMap.get("genericHelloService")); + Map genericServiceMap = context.getBeansOfType(GenericService.class); + Assertions.assertEquals(3, genericServiceMap.size()); + Assertions.assertNotNull(genericServiceMap.get("genericHelloService")); - Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); - Assertions.assertEquals(2, referenceBeanMap.size()); + Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); + Assertions.assertEquals(2, referenceBeanMap.size()); - ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService"); - Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup()); - Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass()); - Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface()); + ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService"); + Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup()); + Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass()); + Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface()); - ReferenceBean genericServiceWithoutInterfaceBean = referenceBeanMap.get("&genericServiceWithoutInterface"); - Assertions.assertEquals("demo", genericServiceWithoutInterfaceBean.getGroup()); - Assertions.assertEquals(GenericService.class, genericServiceWithoutInterfaceBean.getInterfaceClass()); - Assertions.assertEquals("org.apache.dubbo.config.spring.api.LocalMissClass", genericServiceWithoutInterfaceBean.getServiceInterface()); + ReferenceBean genericServiceWithoutInterfaceBean = referenceBeanMap.get("&genericServiceWithoutInterface"); + Assertions.assertEquals("demo", genericServiceWithoutInterfaceBean.getGroup()); + Assertions.assertEquals(GenericService.class, genericServiceWithoutInterfaceBean.getInterfaceClass()); + Assertions.assertEquals("org.apache.dubbo.config.spring.api.LocalMissClass", genericServiceWithoutInterfaceBean.getServiceInterface()); - GenericService genericServiceWithoutInterface = context.getBean("genericServiceWithoutInterface", GenericService.class); - Assertions.assertNotNull(genericServiceWithoutInterface); - Object sayHelloResult = genericServiceWithoutInterface.$invoke("sayHello", new String[]{"java.lang.String"}, new Object[]{"Dubbo"}); - Assertions.assertEquals("Hello Dubbo", sayHelloResult); + GenericService genericServiceWithoutInterface = context.getBean("genericServiceWithoutInterface", GenericService.class); + Assertions.assertNotNull(genericServiceWithoutInterface); + Object sayHelloResult = genericServiceWithoutInterface.$invoke("sayHello", new String[]{"java.lang.String"}, new Object[]{"Dubbo"}); + Assertions.assertEquals("Hello Dubbo", sayHelloResult); + } finally { + context.close(); + } - context.close(); } @Test @@ -176,22 +211,24 @@ public class JavaConfigReferenceBeanTest { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class, ReferenceBeanConfiguration.class); - Map helloServiceMap = context.getBeansOfType(HelloService.class); - Assertions.assertEquals(2, helloServiceMap.size()); - Assertions.assertNotNull(helloServiceMap.get("helloService")); - Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); + try { + Map helloServiceMap = context.getBeansOfType(HelloService.class); + Assertions.assertEquals(2, helloServiceMap.size()); + Assertions.assertNotNull(helloServiceMap.get("helloService")); + Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); - Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); - Assertions.assertEquals(2, referenceBeanMap.size()); - ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); - Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); - Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); + Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); + Assertions.assertEquals(2, referenceBeanMap.size()); + ReferenceBean referenceBean = referenceBeanMap.get("&helloService"); + Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass()); + Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface()); - ReferenceBean demoServiceReferenceBean = referenceBeanMap.get("&demoService"); - Assertions.assertEquals(DemoService.class, demoServiceReferenceBean.getInterfaceClass()); - Assertions.assertEquals(DemoService.class.getName(), demoServiceReferenceBean.getServiceInterface()); - - context.close(); + ReferenceBean demoServiceReferenceBean = referenceBeanMap.get("&demoService"); + Assertions.assertEquals(DemoService.class, demoServiceReferenceBean.getInterfaceClass()); + Assertions.assertEquals(DemoService.class.getName(), demoServiceReferenceBean.getServiceInterface()); + } finally { + context.close(); + } } @Test @@ -199,24 +236,26 @@ public class JavaConfigReferenceBeanTest { AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class, GenericServiceReferenceBeanConfiguration.class); - Map helloServiceMap = context.getBeansOfType(HelloService.class); - Assertions.assertEquals(1, helloServiceMap.size()); - Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); + try { + Map helloServiceMap = context.getBeansOfType(HelloService.class); + Assertions.assertEquals(1, helloServiceMap.size()); + Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl")); - Map genericServiceMap = context.getBeansOfType(GenericService.class); - Assertions.assertEquals(2, genericServiceMap.size()); - Assertions.assertNotNull(genericServiceMap.get("localMissClassGenericServiceImpl")); - Assertions.assertNotNull(genericServiceMap.get("genericHelloService")); + Map genericServiceMap = context.getBeansOfType(GenericService.class); + Assertions.assertEquals(2, genericServiceMap.size()); + Assertions.assertNotNull(genericServiceMap.get("localMissClassGenericServiceImpl")); + Assertions.assertNotNull(genericServiceMap.get("genericHelloService")); - Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); - Assertions.assertEquals(1, referenceBeanMap.size()); + Map referenceBeanMap = context.getBeansOfType(ReferenceBean.class); + Assertions.assertEquals(1, referenceBeanMap.size()); - ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService"); - Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup()); - Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass()); - Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface()); - - context.close(); + ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService"); + Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup()); + Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass()); + Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface()); + } finally { + context.close(); + } } @Test diff --git a/dubbo-config/pom.xml b/dubbo-config/pom.xml index ea7e398b17..168b5ba810 100644 --- a/dubbo-config/pom.xml +++ b/dubbo-config/pom.xml @@ -33,4 +33,13 @@ dubbo-config-api dubbo-config-spring + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-configcenter/dubbo-configcenter-apollo/src/main/java/org/apache/dubbo/configcenter/support/apollo/ApolloDynamicConfiguration.java b/dubbo-configcenter/dubbo-configcenter-apollo/src/main/java/org/apache/dubbo/configcenter/support/apollo/ApolloDynamicConfiguration.java index dbf68221ce..b4a49aabc2 100644 --- a/dubbo-configcenter/dubbo-configcenter-apollo/src/main/java/org/apache/dubbo/configcenter/support/apollo/ApolloDynamicConfiguration.java +++ b/dubbo-configcenter/dubbo-configcenter-apollo/src/main/java/org/apache/dubbo/configcenter/support/apollo/ApolloDynamicConfiguration.java @@ -16,15 +16,6 @@ */ package org.apache.dubbo.configcenter.support.apollo; -import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.config.configcenter.ConfigChangeType; -import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent; -import org.apache.dubbo.common.config.configcenter.ConfigurationListener; -import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; -import org.apache.dubbo.common.logger.Logger; -import org.apache.dubbo.common.logger.LoggerFactory; -import org.apache.dubbo.common.utils.StringUtils; - import com.ctrip.framework.apollo.Config; import com.ctrip.framework.apollo.ConfigChangeListener; import com.ctrip.framework.apollo.ConfigFile; @@ -33,6 +24,14 @@ import com.ctrip.framework.apollo.core.enums.ConfigFileFormat; import com.ctrip.framework.apollo.enums.ConfigSourceType; import com.ctrip.framework.apollo.enums.PropertyChangeType; import com.ctrip.framework.apollo.model.ConfigChange; +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.config.configcenter.ConfigChangeType; +import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent; +import org.apache.dubbo.common.config.configcenter.ConfigurationListener; +import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.utils.StringUtils; import java.util.Arrays; import java.util.Collections; @@ -113,6 +112,12 @@ public class ApolloDynamicConfiguration implements DynamicConfiguration { } } + // TODO release apollo +// @Override +// public void close() throws Exception { +// DynamicConfiguration.super.close(); +// } + private String getAddressWithProtocolPrefix(URL url) { String address = url.getBackupAddress(); if (StringUtils.isNotEmpty(address)) { diff --git a/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosConfigServiceWrapper.java b/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosConfigServiceWrapper.java index 4ac0a789ca..c6a0a4457e 100644 --- a/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosConfigServiceWrapper.java +++ b/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosConfigServiceWrapper.java @@ -65,6 +65,10 @@ public class NacosConfigServiceWrapper { return configService.removeConfig(handleInnerSymbol(dataId), handleInnerSymbol(group)); } + public void shutdown() throws NacosException { + configService.shutDown(); + } + /** * see {@link com.alibaba.nacos.client.config.utils.ParamUtils#isValid(java.lang.String)} */ diff --git a/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java b/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java index 4c2e4eaec3..83a0facf40 100644 --- a/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java +++ b/dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java @@ -17,17 +17,6 @@ package org.apache.dubbo.configcenter.support.nacos; -import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.config.configcenter.ConfigChangeType; -import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent; -import org.apache.dubbo.common.config.configcenter.ConfigItem; -import org.apache.dubbo.common.config.configcenter.ConfigurationListener; -import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; -import org.apache.dubbo.common.logger.Logger; -import org.apache.dubbo.common.logger.LoggerFactory; -import org.apache.dubbo.common.utils.MD5Utils; -import org.apache.dubbo.common.utils.StringUtils; - import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; @@ -38,6 +27,16 @@ import com.alibaba.nacos.api.config.listener.AbstractSharedListener; import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.client.config.http.HttpAgent; import com.alibaba.nacos.common.http.HttpRestResult; +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.config.configcenter.ConfigChangeType; +import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent; +import org.apache.dubbo.common.config.configcenter.ConfigItem; +import org.apache.dubbo.common.config.configcenter.ConfigurationListener; +import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.utils.MD5Utils; +import org.apache.dubbo.common.utils.StringUtils; import java.lang.reflect.Field; import java.util.HashMap; @@ -176,6 +175,11 @@ public class NacosDynamicConfiguration implements DynamicConfiguration { return configListener; } + @Override + public void close() throws Exception { + configService.shutdown(); + } + @Override public void addListener(String key, String group, ConfigurationListener listener) { String listenerKey = buildListenerKey(key, group); diff --git a/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java b/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java index 0ab06d2fb9..d755fd8b5c 100644 --- a/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java +++ b/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java @@ -25,7 +25,6 @@ import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.remoting.zookeeper.ZookeeperClient; import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; - import org.apache.zookeeper.data.Stat; import java.util.Collection; @@ -43,7 +42,7 @@ public class ZookeeperDynamicConfiguration extends TreePathDynamicConfiguration private Executor executor; // The final root path would be: /configRootPath/"config" private String rootPath; - private final ZookeeperClient zkClient; + private ZookeeperClient zkClient; private CacheListener cacheListener; private URL url; @@ -83,7 +82,11 @@ public class ZookeeperDynamicConfiguration extends TreePathDynamicConfiguration @Override protected void doClose() throws Exception { - zkClient.close(); + // zkClient is shared in framework, should not close it here + // zkClient.close(); + // See: org.apache.dubbo.remoting.zookeeper.AbstractZookeeperTransporter#destroy() + // All zk clients is created and destroyed in ZookeeperTransporter. + zkClient = null; } @Override diff --git a/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfigurationFactory.java b/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfigurationFactory.java index 6dcedd98eb..309cf43faf 100644 --- a/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfigurationFactory.java +++ b/dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfigurationFactory.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.config.configcenter.AbstractDynamicConfigurationF import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; import org.apache.dubbo.common.extension.DisableInject; import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; +import org.apache.dubbo.rpc.model.ApplicationModel; /** * @@ -29,8 +30,11 @@ public class ZookeeperDynamicConfigurationFactory extends AbstractDynamicConfigu private ZookeeperTransporter zookeeperTransporter; - public ZookeeperDynamicConfigurationFactory() { - this.zookeeperTransporter = ZookeeperTransporter.getExtension(); + private ApplicationModel applicationModel; + + public ZookeeperDynamicConfigurationFactory(ApplicationModel applicationModel) { + this.applicationModel = applicationModel; + this.zookeeperTransporter = ZookeeperTransporter.getExtension(applicationModel); } @DisableInject diff --git a/dubbo-configcenter/pom.xml b/dubbo-configcenter/pom.xml index f69f389546..b27bae411f 100644 --- a/dubbo-configcenter/pom.xml +++ b/dubbo-configcenter/pom.xml @@ -35,4 +35,13 @@ dubbo-configcenter-apollo dubbo-configcenter-nacos + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-filter/pom.xml b/dubbo-filter/pom.xml index 9b588c52d0..4da7c40f59 100644 --- a/dubbo-filter/pom.xml +++ b/dubbo-filter/pom.xml @@ -33,4 +33,13 @@ dubbo-filter-cache dubbo-filter-validation + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/MetadataInfo.java b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/MetadataInfo.java index e9f53edee0..b445d157d8 100644 --- a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/MetadataInfo.java +++ b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/MetadataInfo.java @@ -18,6 +18,8 @@ package org.apache.dubbo.metadata; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.extension.ExtensionLoader; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; import org.apache.dubbo.common.url.component.URLParam; import org.apache.dubbo.common.utils.ArrayUtils; import org.apache.dubbo.common.utils.CollectionUtils; @@ -51,6 +53,7 @@ import static org.apache.dubbo.rpc.Constants.INTERFACES; public class MetadataInfo implements Serializable { public static final MetadataInfo EMPTY = new MetadataInfo(); + private static final Logger logger = LoggerFactory.getLogger(MetadataInfo.class); private String app; private String revision; @@ -115,7 +118,13 @@ public class MetadataInfo implements Serializable { for (Map.Entry entry : new TreeMap<>(services).entrySet()) { sb.append(entry.getValue().toDescString()); } - this.revision = RevisionResolver.calRevision(sb.toString()); + String tempRevision = RevisionResolver.calRevision(sb.toString()); + if (!StringUtils.isEquals(this.revision, tempRevision)) { + if (logger.isInfoEnabled()) { + logger.info(String.format("metadata revision changed: %s -> %s, app: %s, services: %d", this.revision, tempRevision, this.app, this.services.size())); + } + this.revision = tempRevision; + } } return revision; } diff --git a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory.java b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory.java index 1414c8b213..78f98f67e6 100644 --- a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory.java +++ b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory.java @@ -27,4 +27,7 @@ public interface MetadataReportFactory { @Adaptive({"protocol"}) MetadataReport getMetadataReport(URL url); + + default void destroy() { + } } diff --git a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java index 986b8a63de..4f184913f6 100644 --- a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java +++ b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java @@ -16,6 +16,8 @@ */ package org.apache.dubbo.metadata.report.support; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; @@ -30,9 +32,6 @@ import org.apache.dubbo.metadata.report.identifier.MetadataIdentifier; import org.apache.dubbo.metadata.report.identifier.ServiceMetadataIdentifier; import org.apache.dubbo.metadata.report.identifier.SubscriberMetadataIdentifier; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -101,6 +100,7 @@ public abstract class AbstractMetadataReport implements MetadataReport { File file; private AtomicBoolean initialized = new AtomicBoolean(false); public MetadataReportRetry metadataReportRetry; + private ScheduledExecutorService reportTimerScheduler; public AbstractMetadataReport(URL reportServerURL) { setUrl(reportServerURL); @@ -129,8 +129,8 @@ public abstract class AbstractMetadataReport implements MetadataReport { reportServerURL.getParameter(RETRY_PERIOD_KEY, DEFAULT_METADATA_REPORT_RETRY_PERIOD)); // cycle report the data switch if (reportServerURL.getParameter(CYCLE_REPORT_KEY, DEFAULT_METADATA_REPORT_CYCLE_REPORT)) { - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMetadataReportTimer", true)); - scheduler.scheduleAtFixedRate(this::publishAll, calculateStartTime(), ONE_DAY_IN_MILLISECONDS, TimeUnit.MILLISECONDS); + reportTimerScheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMetadataReportTimer", true)); + reportTimerScheduler.scheduleAtFixedRate(this::publishAll, calculateStartTime(), ONE_DAY_IN_MILLISECONDS, TimeUnit.MILLISECONDS); } } @@ -302,8 +302,12 @@ public abstract class AbstractMetadataReport implements MetadataReport { if (reportCacheExecutor != null) { reportCacheExecutor.shutdown(); } + if (reportTimerScheduler != null) { + reportTimerScheduler.shutdown(); + } if (metadataReportRetry != null) { metadataReportRetry.destroy(); + metadataReportRetry = null; } } diff --git a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReportFactory.java b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReportFactory.java index 29f21d8feb..b2a51422f1 100644 --- a/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReportFactory.java +++ b/dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReportFactory.java @@ -37,14 +37,12 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac /** * The lock for the acquisition process of the registry */ - private static final ReentrantLock LOCK = new ReentrantLock(); + private final ReentrantLock lock = new ReentrantLock(); /** * Registry Collection Map */ - private static final Map SERVICE_STORE_MAP = new ConcurrentHashMap<>(); - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMetadataReportFactory.class); + private final Map serviceStoreMap = new ConcurrentHashMap<>(); @Override public MetadataReport getMetadataReport(URL url) { @@ -52,15 +50,15 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac .removeParameters(EXPORT_KEY, REFER_KEY); String key = url.toServiceString(); - MetadataReport metadataReport = SERVICE_STORE_MAP.get(key); + MetadataReport metadataReport = serviceStoreMap.get(key); if (metadataReport != null) { return metadataReport; } // Lock the metadata access process to ensure a single instance of the metadata instance - LOCK.lock(); + lock.lock(); try { - metadataReport = SERVICE_STORE_MAP.get(key); + metadataReport = serviceStoreMap.get(key); if (metadataReport != null) { return metadataReport; } @@ -69,7 +67,7 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac metadataReport = createMetadataReport(url); } catch (Exception e) { if (!check) { - LOGGER.warn("The metadata reporter failed to initialize", e); + logger.warn("The metadata reporter failed to initialize", e); } else { throw e; } @@ -79,19 +77,20 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac throw new IllegalStateException("Can not create metadata Report " + url); } if (metadataReport != null) { - SERVICE_STORE_MAP.put(key, metadataReport); + serviceStoreMap.put(key, metadataReport); } return metadataReport; } finally { // Release the lock - LOCK.unlock(); + lock.unlock(); } } - public static void destroy() { - LOCK.lock(); + @Override + public void destroy() { + lock.lock(); try { - for (MetadataReport metadataReport : SERVICE_STORE_MAP.values()) { + for (MetadataReport metadataReport : serviceStoreMap.values()) { try{ metadataReport.destroy(); }catch (Throwable ignored){ @@ -99,11 +98,10 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac logger.warn(ignored.getMessage(),ignored); } } - SERVICE_STORE_MAP.clear(); + serviceStoreMap.clear(); } finally { - LOCK.unlock(); + lock.unlock(); } - } protected abstract MetadataReport createMetadataReport(URL url); diff --git a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReport.java b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReport.java index 1e36894837..78a19099e8 100644 --- a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReport.java +++ b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReport.java @@ -16,6 +16,7 @@ */ package org.apache.dubbo.metadata.store.zookeeper; +import com.google.gson.Gson; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.config.configcenter.ConfigItem; import org.apache.dubbo.common.logger.Logger; @@ -34,8 +35,6 @@ import org.apache.dubbo.remoting.zookeeper.DataListener; import org.apache.dubbo.remoting.zookeeper.EventType; import org.apache.dubbo.remoting.zookeeper.ZookeeperClient; import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; - -import com.google.gson.Gson; import org.apache.zookeeper.data.Stat; import java.util.ArrayList; @@ -59,7 +58,7 @@ public class ZookeeperMetadataReport extends AbstractMetadataReport { private final String root; - final ZookeeperClient zkClient; + ZookeeperClient zkClient; private Gson gson = new Gson(); @@ -188,6 +187,13 @@ public class ZookeeperMetadataReport extends AbstractMetadataReport { } } + @Override + public void destroy() { + super.destroy(); + // release zk client reference, but should not close it + zkClient = null; + } + private String buildPathKey(String group, String serviceKey) { return toRootDir() + group + PATH_SEPARATOR + serviceKey; } diff --git a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportFactory.java b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportFactory.java index ee3e2d97a5..c628fc858d 100644 --- a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportFactory.java +++ b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportFactory.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.extension.DisableInject; import org.apache.dubbo.metadata.report.MetadataReport; import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory; import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; +import org.apache.dubbo.rpc.model.ApplicationModel; /** * ZookeeperRegistryFactory. @@ -29,8 +30,11 @@ public class ZookeeperMetadataReportFactory extends AbstractMetadataReportFactor private ZookeeperTransporter zookeeperTransporter; - public ZookeeperMetadataReportFactory() { - this.zookeeperTransporter = ZookeeperTransporter.getExtension(); + private ApplicationModel applicationModel; + + public ZookeeperMetadataReportFactory(ApplicationModel applicationModel) { + this.applicationModel = applicationModel; + this.zookeeperTransporter = ZookeeperTransporter.getExtension(applicationModel); } @DisableInject diff --git a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/test/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportTest.java b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/test/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportTest.java index b77897cd0e..57cc6eeabb 100644 --- a/dubbo-metadata/dubbo-metadata-report-zookeeper/src/test/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportTest.java +++ b/dubbo-metadata/dubbo-metadata-report-zookeeper/src/test/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportTest.java @@ -16,6 +16,8 @@ */ package org.apache.dubbo.metadata.store.zookeeper; +import com.google.gson.Gson; +import org.apache.curator.test.TestingServer; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.config.configcenter.ConfigItem; import org.apache.dubbo.common.utils.NetUtils; @@ -27,15 +29,18 @@ import org.apache.dubbo.metadata.report.identifier.KeyTypeEnum; import org.apache.dubbo.metadata.report.identifier.MetadataIdentifier; import org.apache.dubbo.metadata.report.identifier.ServiceMetadataIdentifier; import org.apache.dubbo.metadata.report.identifier.SubscriberMetadataIdentifier; - -import com.google.gson.Gson; -import org.apache.curator.test.TestingServer; +import org.apache.dubbo.rpc.model.ApplicationModel; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; @@ -58,7 +63,7 @@ public class ZookeeperMetadataReportTest { this.zkServer = new TestingServer(zkServerPort, true); this.registryUrl = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort); - zookeeperMetadataReportFactory = new ZookeeperMetadataReportFactory(); + zookeeperMetadataReportFactory = new ZookeeperMetadataReportFactory(ApplicationModel.defaultModel()); this.zookeeperMetadataReport = (ZookeeperMetadataReport) zookeeperMetadataReportFactory.getMetadataReport(registryUrl); } diff --git a/dubbo-metadata/pom.xml b/dubbo-metadata/pom.xml index 9fdffe93f0..2eec45ed63 100644 --- a/dubbo-metadata/pom.xml +++ b/dubbo-metadata/pom.xml @@ -36,4 +36,12 @@ dubbo-metadata-report-nacos + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-metrics/pom.xml b/dubbo-metrics/pom.xml index 1f0d32ac31..8c4291aaca 100644 --- a/dubbo-metrics/pom.xml +++ b/dubbo-metrics/pom.xml @@ -33,4 +33,13 @@ false + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-monitor/dubbo-monitor-default/src/main/java/org/apache/dubbo/monitor/dubbo/DubboMonitor.java b/dubbo-monitor/dubbo-monitor-default/src/main/java/org/apache/dubbo/monitor/dubbo/DubboMonitor.java index ea32fc59aa..c46d02bd0b 100644 --- a/dubbo-monitor/dubbo-monitor-default/src/main/java/org/apache/dubbo/monitor/dubbo/DubboMonitor.java +++ b/dubbo-monitor/dubbo-monitor-default/src/main/java/org/apache/dubbo/monitor/dubbo/DubboMonitor.java @@ -20,7 +20,6 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; import org.apache.dubbo.common.utils.ExecutorUtil; -import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.monitor.Monitor; import org.apache.dubbo.monitor.MonitorService; import org.apache.dubbo.rpc.Invoker; @@ -29,7 +28,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -52,7 +50,7 @@ public class DubboMonitor implements Monitor { /** * The timer for sending statistics */ - private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(3, new NamedThreadFactory("DubboMonitorSendTimer", true)); + private final ScheduledExecutorService scheduledExecutorService; /** * The future that can cancel the scheduledExecutorService @@ -68,6 +66,7 @@ public class DubboMonitor implements Monitor { public DubboMonitor(Invoker monitorInvoker, MonitorService monitorService) { this.monitorInvoker = monitorInvoker; this.monitorService = monitorService; + scheduledExecutorService = monitorInvoker.getUrl().getOrDefaultApplicationModel().getApplicationExecutorRepository().getSharedScheduledExecutor(); // The time interval for timer scheduledExecutorService to send data final long monitorInterval = monitorInvoker.getUrl().getPositiveParameter("interval", 60000); // collect timer for collecting statistics data diff --git a/dubbo-monitor/dubbo-monitor-default/src/test/resources/log4j.properties b/dubbo-monitor/dubbo-monitor-default/src/test/resources/log4j.properties new file mode 100644 index 0000000000..8de4c4fddb --- /dev/null +++ b/dubbo-monitor/dubbo-monitor-default/src/test/resources/log4j.properties @@ -0,0 +1,7 @@ +###set log levels### +log4j.rootLogger=info, stdout +###output to the console### +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d{dd/MM/yy HH:mm:ss:SSS z}] %t %5p %c{2}: %m%n diff --git a/dubbo-monitor/pom.xml b/dubbo-monitor/pom.xml index 4e3b53ca39..735bf3743e 100644 --- a/dubbo-monitor/pom.xml +++ b/dubbo-monitor/pom.xml @@ -33,4 +33,14 @@ dubbo-monitor-api dubbo-monitor-default + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + + diff --git a/dubbo-native/pom.xml b/dubbo-native/pom.xml index fc70304c80..727e58cfab 100644 --- a/dubbo-native/pom.xml +++ b/dubbo-native/pom.xml @@ -64,7 +64,12 @@ ${project.parent.version} + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + - diff --git a/dubbo-native/src/main/java/org/apache/dubbo/common/serialize/Serialization$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/common/serialize/Serialization$Adaptive.java index 088c1bafb4..9bf19889e7 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/common/serialize/Serialization$Adaptive.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/common/serialize/Serialization$Adaptive.java @@ -21,14 +21,8 @@ public class Serialization$Adaptive implements org.apache.dubbo.common.serialize public byte getContentTypeId() { throw new UnsupportedOperationException("The method public abstract byte org.apache.dubbo.common.serialize.Serialization.getContentTypeId() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!"); } -public org.apache.dubbo.common.serialize.ObjectOutput serialize(org.apache.dubbo.common.URL arg0, java.io.OutputStream arg1) throws java.io.IOException { -if (arg0 == null) throw new IllegalArgumentException("url == null"); -org.apache.dubbo.common.URL url = arg0; -String extName = url.getParameter("serialization", "hessian2"); -if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.common.serialize.Serialization) name from url (" + url.toString() + ") use keys([serialization])"); -ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.common.serialize.Serialization.class); -org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName); -return extension.serialize(arg0, arg1); +public java.lang.String getContentType() { +throw new UnsupportedOperationException("The method public abstract java.lang.String org.apache.dubbo.common.serialize.Serialization.getContentType() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!"); } public org.apache.dubbo.common.serialize.ObjectInput deserialize(org.apache.dubbo.common.URL arg0, java.io.InputStream arg1) throws java.io.IOException { if (arg0 == null) throw new IllegalArgumentException("url == null"); @@ -39,7 +33,13 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName); return extension.deserialize(arg0, arg1); } -public java.lang.String getContentType() { -throw new UnsupportedOperationException("The method public abstract java.lang.String org.apache.dubbo.common.serialize.Serialization.getContentType() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!"); +public org.apache.dubbo.common.serialize.ObjectOutput serialize(org.apache.dubbo.common.URL arg0, java.io.OutputStream arg1) throws java.io.IOException { +if (arg0 == null) throw new IllegalArgumentException("url == null"); +org.apache.dubbo.common.URL url = arg0; +String extName = url.getParameter("serialization", "hessian2"); +if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.common.serialize.Serialization) name from url (" + url.toString() + ") use keys([serialization])"); +ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.common.serialize.Serialization.class); +org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName); +return extension.serialize(arg0, arg1); } } diff --git a/dubbo-native/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory$Adaptive.java index 3e5ad9b799..069e3b940f 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory$Adaptive.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory$Adaptive.java @@ -27,4 +27,7 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa org.apache.dubbo.metadata.report.MetadataReportFactory extension = (org.apache.dubbo.metadata.report.MetadataReportFactory)scopeModel.getExtensionLoader(org.apache.dubbo.metadata.report.MetadataReportFactory.class).getExtension(extName); return extension.getMetadataReport(arg0); } +public void destroy() { +throw new UnsupportedOperationException("The method public abstract void org.apache.dubbo.metadata.report.MetadataReportFactory.destroy() of interface org.apache.dubbo.metadata.report.MetadataReportFactory is not adaptive method!"); +} } diff --git a/dubbo-native/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter$Adaptive.java deleted file mode 100644 index aeed40713d..0000000000 --- a/dubbo-native/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter$Adaptive.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dubbo.remoting.zookeeper; -import org.apache.dubbo.common.extension.ExtensionLoader; -public class ZookeeperTransporter$Adaptive implements org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter { -public org.apache.dubbo.remoting.zookeeper.ZookeeperClient connect(org.apache.dubbo.common.URL arg0) { -if (arg0 == null) throw new IllegalArgumentException("url == null"); -org.apache.dubbo.common.URL url = arg0; -String extName = url.getParameter("client", url.getParameter("transporter", "curator")); -if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter) name from url (" + url.toString() + ") use keys([client, transporter])"); -org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter extension = (org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter)ExtensionLoader.getExtensionLoader(org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter.class).getExtension(extName); -return extension.connect(arg0); -} -} diff --git a/dubbo-native/src/main/java/org/apache/dubbo/rpc/Protocol$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/rpc/Protocol$Adaptive.java index fe16e162f3..16367fa414 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/rpc/Protocol$Adaptive.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/rpc/Protocol$Adaptive.java @@ -18,6 +18,9 @@ package org.apache.dubbo.rpc; import org.apache.dubbo.rpc.model.ScopeModel; import org.apache.dubbo.rpc.model.ScopeModelUtil; public class Protocol$Adaptive implements org.apache.dubbo.rpc.Protocol { +public int getDefaultPort() { +throw new UnsupportedOperationException("The method public abstract int org.apache.dubbo.rpc.Protocol.getDefaultPort() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!"); +} public org.apache.dubbo.rpc.Exporter export(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException { if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null"); if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null"); @@ -37,13 +40,10 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa org.apache.dubbo.rpc.Protocol extension = (org.apache.dubbo.rpc.Protocol)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.Protocol.class).getExtension(extName); return extension.refer(arg0, arg1); } -public java.util.List getServers() { -throw new UnsupportedOperationException("The method public default java.util.List org.apache.dubbo.rpc.Protocol.getServers() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!"); -} public void destroy() { throw new UnsupportedOperationException("The method public abstract void org.apache.dubbo.rpc.Protocol.destroy() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!"); } -public int getDefaultPort() { -throw new UnsupportedOperationException("The method public abstract int org.apache.dubbo.rpc.Protocol.getDefaultPort() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!"); +public java.util.List getServers() { +throw new UnsupportedOperationException("The method public default java.util.List org.apache.dubbo.rpc.Protocol.getServers() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!"); } } diff --git a/dubbo-native/src/main/java/org/apache/dubbo/rpc/ProxyFactory$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/rpc/ProxyFactory$Adaptive.java index 297287fdbb..101925d8ab 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/rpc/ProxyFactory$Adaptive.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/rpc/ProxyFactory$Adaptive.java @@ -18,14 +18,15 @@ package org.apache.dubbo.rpc; import org.apache.dubbo.rpc.model.ScopeModel; import org.apache.dubbo.rpc.model.ScopeModelUtil; public class ProxyFactory$Adaptive implements org.apache.dubbo.rpc.ProxyFactory { -public org.apache.dubbo.rpc.Invoker getInvoker(java.lang.Object arg0, java.lang.Class arg1, org.apache.dubbo.common.URL arg2) throws org.apache.dubbo.rpc.RpcException { -if (arg2 == null) throw new IllegalArgumentException("url == null"); -org.apache.dubbo.common.URL url = arg2; +public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException { +if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null"); +if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null"); +org.apache.dubbo.common.URL url = arg0.getUrl(); String extName = url.getParameter("proxy", "javassist"); if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.rpc.ProxyFactory) name from url (" + url.toString() + ") use keys([proxy])"); ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.rpc.ProxyFactory.class); org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName); -return extension.getInvoker(arg0, arg1, arg2); +return extension.getProxy(arg0); } public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0, boolean arg1) throws org.apache.dubbo.rpc.RpcException { if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null"); @@ -37,14 +38,13 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName); return extension.getProxy(arg0, arg1); } -public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException { -if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null"); -if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null"); -org.apache.dubbo.common.URL url = arg0.getUrl(); +public org.apache.dubbo.rpc.Invoker getInvoker(java.lang.Object arg0, java.lang.Class arg1, org.apache.dubbo.common.URL arg2) throws org.apache.dubbo.rpc.RpcException { +if (arg2 == null) throw new IllegalArgumentException("url == null"); +org.apache.dubbo.common.URL url = arg2; String extName = url.getParameter("proxy", "javassist"); if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.rpc.ProxyFactory) name from url (" + url.toString() + ") use keys([proxy])"); ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.rpc.ProxyFactory.class); org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName); -return extension.getProxy(arg0); +return extension.getInvoker(arg0, arg1, arg2); } } diff --git a/dubbo-native/src/main/java/org/apache/dubbo/rpc/cluster/Cluster$Adaptive.java b/dubbo-native/src/main/java/org/apache/dubbo/rpc/cluster/Cluster$Adaptive.java index 5f801663a6..e8b087d611 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/rpc/cluster/Cluster$Adaptive.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/rpc/cluster/Cluster$Adaptive.java @@ -18,12 +18,6 @@ package org.apache.dubbo.rpc.cluster; import org.apache.dubbo.rpc.model.ScopeModel; import org.apache.dubbo.rpc.model.ScopeModelUtil; public class Cluster$Adaptive implements org.apache.dubbo.rpc.cluster.Cluster { -public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1) { -throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!"); -} -public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1, boolean arg2) { -throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String,boolean) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!"); -} public org.apache.dubbo.rpc.Invoker join(org.apache.dubbo.rpc.cluster.Directory arg0, boolean arg1) throws org.apache.dubbo.rpc.RpcException { if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.cluster.Directory argument == null"); if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.cluster.Directory argument getUrl() == null"); @@ -34,4 +28,10 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa org.apache.dubbo.rpc.cluster.Cluster extension = (org.apache.dubbo.rpc.cluster.Cluster)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.cluster.Cluster.class).getExtension(extName); return extension.join(arg0, arg1); } +public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1) { +throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!"); +} +public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1, boolean arg2) { +throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String,boolean) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!"); +} } diff --git a/dubbo-native/src/main/java/org/apache/dubbo/utils/CodeGenerator.java b/dubbo-native/src/main/java/org/apache/dubbo/utils/CodeGenerator.java index a632d4aeba..f4ca7d10d0 100644 --- a/dubbo-native/src/main/java/org/apache/dubbo/utils/CodeGenerator.java +++ b/dubbo-native/src/main/java/org/apache/dubbo/utils/CodeGenerator.java @@ -17,7 +17,6 @@ package org.apache.dubbo.utils; import org.apache.commons.io.FileUtils; - import org.apache.dubbo.common.extension.Adaptive; import org.apache.dubbo.common.extension.AdaptiveClassCodeGenerator; import org.apache.dubbo.common.extension.SPI; @@ -69,7 +68,7 @@ public class CodeGenerator { value = "adaptive"; } AdaptiveClassCodeGenerator codeGenerator = new AdaptiveClassCodeGenerator(it, value); - String code = codeGenerator.generate(); + String code = codeGenerator.generate(true); System.out.println(code); System.out.println("-----:" + it.getPackage()); try { diff --git a/dubbo-plugin/dubbo-qos/src/main/java/org/apache/dubbo/qos/command/impl/ShutdownTelnet.java b/dubbo-plugin/dubbo-qos/src/main/java/org/apache/dubbo/qos/command/impl/ShutdownTelnet.java index 736e329a10..aef3983ae0 100644 --- a/dubbo-plugin/dubbo-qos/src/main/java/org/apache/dubbo/qos/command/impl/ShutdownTelnet.java +++ b/dubbo-plugin/dubbo-qos/src/main/java/org/apache/dubbo/qos/command/impl/ShutdownTelnet.java @@ -17,7 +17,6 @@ package org.apache.dubbo.qos.command.impl; import org.apache.dubbo.common.utils.StringUtils; -import org.apache.dubbo.config.deploy.DefaultApplicationDeployer; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; import org.apache.dubbo.qos.command.annotation.Cmd; @@ -60,8 +59,7 @@ public class ShutdownTelnet implements BaseCommand { StringBuilder buf = new StringBuilder(); List applicationModels = frameworkModel.getApplicationModels(); for (ApplicationModel applicationModel : new ArrayList<>(applicationModels)) { - DefaultApplicationDeployer deployer = applicationModel.getBeanFactory().getBean(DefaultApplicationDeployer.class); - deployer.destroy(); + applicationModel.destroy(); } // TODO change to ApplicationDeployer.destroy() or ApplicationModel.destroy() // DubboShutdownHook.getDubboShutdownHook().unregister(); diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ChangeTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ChangeTelnetTest.java index 0b08b87e54..5ecc7780fb 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ChangeTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ChangeTelnetTest.java @@ -22,7 +22,6 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.qos.legacy.service.DemoService; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.Protocol; @@ -30,6 +29,7 @@ import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,7 +40,7 @@ import static org.mockito.Mockito.reset; public class ChangeTelnetTest { private final DefaultAttributeMap defaultAttributeMap = new DefaultAttributeMap(); - private static final BaseCommand change = new ChangeTelnet(FrameworkModel.defaultModel()); + private BaseCommand change; private Channel mockChannel; private CommandContext mockCommandContext; @@ -48,12 +48,19 @@ public class ChangeTelnetTest { @AfterAll public static void tearDown() { + FrameworkModel.destroyAll(); + } + @BeforeAll + public static void setUp() { + FrameworkModel.destroyAll(); } @SuppressWarnings("unchecked") @BeforeEach - public void setUp() { + public void beforeEach() { + change = new ChangeTelnet(FrameworkModel.defaultModel()); + mockCommandContext = mock(CommandContext.class); mockChannel = mock(Channel.class); mockInvoker = mock(Invoker.class); @@ -66,8 +73,8 @@ public class ChangeTelnetTest { } @AfterEach - public void after() { - ProtocolUtils.closeAll(); + public void afterEach() { + FrameworkModel.destroyAll(); reset(mockCommandContext, mockChannel, mockInvoker); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/CountTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/CountTelnetTest.java index 1c0bfc613a..c28c9f562c 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/CountTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/CountTelnetTest.java @@ -21,7 +21,6 @@ import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; import org.apache.dubbo.qos.command.impl.channel.MockNettyChannel; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.qos.legacy.service.DemoService; import org.apache.dubbo.remoting.telnet.support.TelnetUtils; import org.apache.dubbo.rpc.Invoker; @@ -44,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; public class CountTelnetTest { - private static final BaseCommand count = new CountTelnet(FrameworkModel.defaultModel()); + private BaseCommand count; private MockNettyChannel mockChannel; private Invoker mockInvoker; @@ -56,6 +55,7 @@ public class CountTelnetTest { @BeforeEach public void setUp() { + count = new CountTelnet(FrameworkModel.defaultModel()); latch = new CountDownLatch(2); mockInvoker = mock(Invoker.class); mockCommandContext = mock(CommandContext.class); @@ -68,7 +68,7 @@ public class CountTelnetTest { @AfterEach public void tearDown() { - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); mockChannel.close(); reset(mockInvoker, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/InvokeTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/InvokeTelnetTest.java index 8fcda4c7c6..e8a9a99329 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/InvokeTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/InvokeTelnetTest.java @@ -21,7 +21,6 @@ import io.netty.util.DefaultAttributeMap; import org.apache.dubbo.config.bootstrap.DubboBootstrap; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.qos.legacy.service.DemoService; import org.apache.dubbo.qos.legacy.service.DemoServiceImpl; import org.apache.dubbo.remoting.RemotingException; @@ -64,7 +63,6 @@ public class InvokeTelnetTest { @AfterEach public void after() { - ProtocolUtils.closeAll(); frameworkModel.destroy(); reset(mockChannel, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PortTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PortTelnetTest.java index 27529354bb..b3b4ecf243 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PortTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PortTelnetTest.java @@ -17,11 +17,9 @@ package org.apache.dubbo.qos.command.impl; import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.common.utils.NetUtils; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.qos.legacy.service.DemoService; import org.apache.dubbo.remoting.RemotingException; import org.apache.dubbo.remoting.exchange.ExchangeClient; @@ -41,7 +39,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; public class PortTelnetTest { - private static final BaseCommand port = new PortTelnet(FrameworkModel.defaultModel()); + private BaseCommand port; private Invoker mockInvoker; private CommandContext mockCommandContext; @@ -51,17 +49,19 @@ public class PortTelnetTest { @SuppressWarnings("unchecked") @BeforeEach public void before() { + FrameworkModel frameworkModel = FrameworkModel.defaultModel(); + port = new PortTelnet(frameworkModel); mockCommandContext = mock(CommandContext.class); mockInvoker = mock(Invoker.class); given(mockInvoker.getInterface()).willReturn(DemoService.class); given(mockInvoker.getUrl()).willReturn(URL.valueOf("dubbo://127.0.0.1:" + availablePort + "/demo")); - ExtensionLoader.getExtensionLoader(Protocol.class).getExtension(DubboProtocol.NAME).export(mockInvoker); + frameworkModel.getExtensionLoader(Protocol.class).getExtension(DubboProtocol.NAME).export(mockInvoker); } @AfterEach - public void after() { - ProtocolUtils.closeAll(); + public void afterEach() { + FrameworkModel.destroyAll(); reset(mockInvoker, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PwdTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PwdTelnetTest.java index eb5046a7a4..e66fb54e2f 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PwdTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PwdTelnetTest.java @@ -16,13 +16,12 @@ */ package org.apache.dubbo.qos.command.impl; -import org.apache.dubbo.qos.command.BaseCommand; -import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; -import org.apache.dubbo.remoting.RemotingException; - import io.netty.channel.Channel; import io.netty.util.DefaultAttributeMap; +import org.apache.dubbo.qos.command.BaseCommand; +import org.apache.dubbo.qos.command.CommandContext; +import org.apache.dubbo.remoting.RemotingException; +import org.apache.dubbo.rpc.model.FrameworkModel; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,7 +48,7 @@ public class PwdTelnetTest { @AfterEach public void tearDown() { - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); mockChannel.close(); reset(mockChannel, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/SelectTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/SelectTelnetTest.java index b66ba4897c..9f2192b19c 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/SelectTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/SelectTelnetTest.java @@ -21,7 +21,6 @@ import io.netty.util.DefaultAttributeMap; import org.apache.dubbo.config.bootstrap.DubboBootstrap; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.qos.legacy.service.DemoService; import org.apache.dubbo.qos.legacy.service.DemoServiceImpl; import org.apache.dubbo.remoting.RemotingException; @@ -44,17 +43,19 @@ import static org.mockito.Mockito.reset; public class SelectTelnetTest { - private static BaseCommand select = new SelectTelnet(FrameworkModel.defaultModel()); + private BaseCommand select; private Channel mockChannel; private CommandContext mockCommandContext; - private final ModuleServiceRepository repository = ApplicationModel.defaultModel().getDefaultModule().getServiceRepository(); + private ModuleServiceRepository repository; private final DefaultAttributeMap defaultAttributeMap = new DefaultAttributeMap(); private List methods; @BeforeEach public void setup() { + repository = ApplicationModel.defaultModel().getDefaultModule().getServiceRepository(); + select = new SelectTelnet(FrameworkModel.defaultModel()); String methodName = "getPerson"; methods = new ArrayList<>(); for (Method method : DemoService.class.getMethods()) { @@ -71,7 +72,7 @@ public class SelectTelnetTest { @AfterEach public void after() { - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); reset(mockChannel, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ShutdownTelnetTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ShutdownTelnetTest.java index 4e574a8a0c..4fa797c92c 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ShutdownTelnetTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ShutdownTelnetTest.java @@ -19,7 +19,6 @@ package org.apache.dubbo.qos.command.impl; import io.netty.channel.Channel; import org.apache.dubbo.qos.command.BaseCommand; import org.apache.dubbo.qos.command.CommandContext; -import org.apache.dubbo.qos.legacy.ProtocolUtils; import org.apache.dubbo.remoting.RemotingException; import org.apache.dubbo.rpc.model.FrameworkModel; import org.junit.jupiter.api.AfterEach; @@ -33,12 +32,13 @@ import static org.mockito.Mockito.reset; public class ShutdownTelnetTest { - private static final BaseCommand shutdown = new ShutdownTelnet(FrameworkModel.defaultModel()); + private BaseCommand shutdown; private Channel mockChannel; private CommandContext mockCommandContext; @BeforeEach public void setUp() { + shutdown = new ShutdownTelnet(FrameworkModel.defaultModel()); mockCommandContext = mock(CommandContext.class); mockChannel = mock(Channel.class); given(mockCommandContext.getRemote()).willReturn(mockChannel); @@ -46,7 +46,7 @@ public class ShutdownTelnetTest { @AfterEach public void after() { - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); reset(mockChannel, mockCommandContext); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/util/CommandHelperTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/util/CommandHelperTest.java index 377d4c3859..3eb73a23f6 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/util/CommandHelperTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/util/CommandHelperTest.java @@ -53,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class CommandHelperTest { private CommandHelper commandHelper = new CommandHelper(FrameworkModel.defaultModel()); + @Test public void testHasCommand() throws Exception { assertTrue(commandHelper.hasCommand("greeting")); diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ChangeTelnetHandlerTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ChangeTelnetHandlerTest.java index 3a93d449a2..98a5c59496 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ChangeTelnetHandlerTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ChangeTelnetHandlerTest.java @@ -24,8 +24,8 @@ import org.apache.dubbo.remoting.RemotingException; import org.apache.dubbo.remoting.telnet.TelnetHandler; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.Protocol; +import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -74,7 +74,7 @@ public class ChangeTelnetHandlerTest { @AfterEach public void after() { - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); reset(mockChannel, mockInvoker); } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ProtocolUtils.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ProtocolUtils.java index eb6b59f845..c2a088f6ce 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ProtocolUtils.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ProtocolUtils.java @@ -20,25 +20,22 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.rpc.Exporter; import org.apache.dubbo.rpc.Protocol; -import org.apache.dubbo.rpc.ProtocolServer; import org.apache.dubbo.rpc.ProxyFactory; +import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol; -import java.util.Collection; - /** * TODO Comment of ProtocolUtils */ public class ProtocolUtils { - private static Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension(); - private static ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); - public static T refer(Class type, String url) { return refer(type, URL.valueOf(url)); } public static T refer(Class type, URL url) { + Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension(); + ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); return proxy.getProxy(protocol.refer(type, url)); } @@ -47,14 +44,14 @@ public class ProtocolUtils { } public static Exporter export(T instance, Class type, URL url) { + Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension(); + ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); return protocol.export(proxy.getInvoker(instance, type, url)); } public static void closeAll() { DubboProtocol.getDubboProtocol().destroy(); - Collection servers = DubboProtocol.getDubboProtocol().getServers(); - for (ProtocolServer server : servers) { - server.close(); - } + ExtensionLoader.getExtensionLoader(Protocol.class).destroy(); + FrameworkModel.destroyAll(); } } diff --git a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/TraceTelnetHandlerTest.java b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/TraceTelnetHandlerTest.java index dd74db3741..8a83021d42 100644 --- a/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/TraceTelnetHandlerTest.java +++ b/dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/TraceTelnetHandlerTest.java @@ -23,9 +23,9 @@ import org.apache.dubbo.remoting.Channel; import org.apache.dubbo.remoting.telnet.TelnetHandler; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.Protocol; +import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol; import org.apache.dubbo.rpc.protocol.dubbo.filter.TraceFilter; - import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -58,7 +58,7 @@ public class TraceTelnetHandlerTest { @AfterEach public void tearDown() { reset(mockChannel, mockInvoker); - ProtocolUtils.closeAll(); + FrameworkModel.destroyAll(); } @Test diff --git a/dubbo-plugin/pom.xml b/dubbo-plugin/pom.xml index 194525c528..ed77204583 100644 --- a/dubbo-plugin/pom.xml +++ b/dubbo-plugin/pom.xml @@ -36,4 +36,12 @@ false + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/InMemoryWritableMetadataService.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/InMemoryWritableMetadataService.java index 30ec1edd23..d563972bbe 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/InMemoryWritableMetadataService.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/InMemoryWritableMetadataService.java @@ -37,6 +37,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel; import org.apache.dubbo.rpc.model.ScopeModelAware; import org.apache.dubbo.rpc.support.ProtocolUtils; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -134,7 +135,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService, @Override public void setApplicationModel(ApplicationModel applicationModel) { this.applicationModel = applicationModel; - this.metadataPublishDelayTime = ConfigurationUtils.get(applicationModel, METADATA_PUBLISH_DELAY_KEY, DEFAULT_METADATA_PUBLISH_DELAY) * 100L; + this.metadataPublishDelayTime = ConfigurationUtils.get(applicationModel, METADATA_PUBLISH_DELAY_KEY, DEFAULT_METADATA_PUBLISH_DELAY); } @Override @@ -196,6 +197,15 @@ public class InMemoryWritableMetadataService implements WritableMetadataService, } } + public void addMetadataInfo(String key, MetadataInfo metadataInfo) { + updateLock.readLock().lock(); + try { + metadataInfos.put(key, metadataInfo); + } finally { + updateLock.readLock().unlock(); + } + } + @Override public boolean unexportURL(URL url) { if (MetadataService.class.getName().equals(url.getServiceInterface())) { @@ -286,6 +296,9 @@ public class InMemoryWritableMetadataService implements WritableMetadataService, return metadataInfo; } } + if (logger.isInfoEnabled()) { + logger.info("metadata not found for revision: " + revision); + } return null; } @@ -324,7 +337,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService, metadataSemaphore.drainPermits(); updateLock.writeLock().lock(); } catch (InterruptedException e) { - if (!applicationModel.isStopping()) { + if (!applicationModel.isDestroyed()) { logger.warn("metadata refresh thread has been interrupted unexpectedly while waiting for update.", e); } } @@ -335,7 +348,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService, } public Map getMetadataInfos() { - return metadataInfos; + return Collections.unmodifiableMap(metadataInfos); } void addMetaServiceURL(URL url) { diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/migration/MigrationRuleListener.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/migration/MigrationRuleListener.java index 1651e8cd3c..ca1f3ddc17 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/migration/MigrationRuleListener.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/migration/MigrationRuleListener.java @@ -110,7 +110,7 @@ public class MigrationRuleListener implements RegistryProtocolListener, Configur String localRawRule = moduleModel.getModelEnvironment().getLocalMigrationRule(); if (!StringUtils.isEmpty(localRawRule)) { - localRuleMigrationFuture = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMigrationRuleDelayWorker", true)) + localRuleMigrationFuture = moduleModel.getApplicationModel().getApplicationExecutorRepository().getSharedScheduledExecutor() .schedule(() -> { if (this.rawRule.equals(INIT)) { this.process(new ConfigChangedEvent(null, null, localRawRule)); diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/RegistryProtocol.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/RegistryProtocol.java index c2bedc344a..48cea930b6 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/RegistryProtocol.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/RegistryProtocol.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.config.ConfigurationUtils; import org.apache.dubbo.common.config.configcenter.DynamicConfiguration; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.threadpool.manager.ExecutorRepository; import org.apache.dubbo.common.timer.HashedWheelTimer; import org.apache.dubbo.common.url.component.ServiceConfigURL; import org.apache.dubbo.common.utils.CollectionUtils; @@ -64,10 +65,9 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import static java.util.concurrent.Executors.newSingleThreadExecutor; import static org.apache.dubbo.common.constants.CommonConstants.APPLICATION_KEY; import static org.apache.dubbo.common.constants.CommonConstants.CLUSTER_KEY; import static org.apache.dubbo.common.constants.CommonConstants.COMMA_SPLIT_PATTERN; @@ -592,6 +592,7 @@ public class RegistryProtocol implements Protocol, ScopeModelAware { @Override public void destroy() { + // FIXME all application models in framework are removed at this moment for (ApplicationModel applicationModel : frameworkModel.getApplicationModels()) { for (ModuleModel moduleModel : applicationModel.getModuleModels()) { List listeners = moduleModel.getExtensionLoader(RegistryProtocolListener.class) @@ -849,7 +850,7 @@ public class RegistryProtocol implements Protocol, ScopeModelAware { */ private class ExporterChangeableWrapper implements Exporter { - private final ExecutorService executor = newSingleThreadExecutor(new NamedThreadFactory("Exporter-Unexport", true)); + private final ScheduledExecutorService executor; private final Invoker originInvoker; private Exporter exporter; @@ -859,6 +860,9 @@ public class RegistryProtocol implements Protocol, ScopeModelAware { public ExporterChangeableWrapper(Exporter exporter, Invoker originInvoker) { this.exporter = exporter; this.originInvoker = originInvoker; + ExecutorRepository executorRepository = originInvoker.getUrl().getOrDefaultApplicationModel() + .getDefaultExtension(ExecutorRepository.class); + this.executor = executorRepository.getSharedScheduledExecutor(); } public Invoker getOriginInvoker() { @@ -907,19 +911,15 @@ public class RegistryProtocol implements Protocol, ScopeModelAware { logger.warn(t.getMessage(), t); } - executor.submit(() -> { + //TODO wait for shutdown timeout is a bit strange + int timeout = ConfigurationUtils.getServerShutdownTimeout(subscribeUrl.getScopeModel()); + executor.schedule(() -> { try { - int timeout = ConfigurationUtils.getServerShutdownTimeout(subscribeUrl.getScopeModel()); - if (timeout > 0) { - logger.info("Waiting " + timeout + "ms for registry to notify all consumers before unexport. " + - "Usually, this is called when you use dubbo API"); - Thread.sleep(timeout); - } exporter.unexport(); } catch (Throwable t) { logger.warn(t.getMessage(), t); } - }); + }, timeout, TimeUnit.MILLISECONDS); } public void setSubscribeUrl(URL subscribeUrl) { diff --git a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java index 21b5bd9e8a..7965cbdb80 100644 --- a/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java +++ b/dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java @@ -19,10 +19,10 @@ package org.apache.dubbo.registry.support; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.threadpool.manager.ExecutorRepository; import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.common.utils.ConcurrentHashSet; import org.apache.dubbo.common.utils.ConfigUtils; -import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.common.utils.StringUtils; import org.apache.dubbo.common.utils.UrlUtils; import org.apache.dubbo.registry.NotifyListener; @@ -48,7 +48,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -82,7 +81,7 @@ public abstract class AbstractRegistry implements Registry { // Local disk cache, where the special key value.registries records the list of registry centers, and the others are the list of notified service providers private final Properties properties = new Properties(); // File cache timing writing - private final ExecutorService registryCacheExecutor = Executors.newFixedThreadPool(1, new NamedThreadFactory("DubboSaveRegistryCache", true)); + private final ExecutorService registryCacheExecutor; private final AtomicLong lastCacheChanged = new AtomicLong(); private final AtomicInteger savePropertiesRetryTimes = new AtomicInteger(); private final Set registered = new ConcurrentHashSet<>(); @@ -100,6 +99,7 @@ public abstract class AbstractRegistry implements Registry { setUrl(url); registryManager = url.getOrDefaultApplicationModel().getBeanFactory().getBean(RegistryManager.class); localCacheEnabled = url.getParameter(REGISTRY_LOCAL_FILE_CACHE_ENABLED, true); + registryCacheExecutor = url.getOrDefaultApplicationModel().getDefaultExtension(ExecutorRepository.class).getSharedExecutor(); if (localCacheEnabled) { // Start file save timer syncSaveFile = url.getParameter(REGISTRY_FILESAVE_SYNC_KEY, false); diff --git a/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/ServiceInstanceMetadataUtilsTest.java b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/ServiceInstanceMetadataUtilsTest.java index 3f08c8dea2..7a83c1d899 100644 --- a/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/ServiceInstanceMetadataUtilsTest.java +++ b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/ServiceInstanceMetadataUtilsTest.java @@ -26,6 +26,7 @@ import org.apache.dubbo.registry.client.DefaultServiceInstance; import org.apache.dubbo.registry.client.InMemoryServiceDiscovery; import org.apache.dubbo.registry.client.ServiceDiscovery; import org.apache.dubbo.registry.client.ServiceDiscoveryRegistry; +import org.apache.dubbo.registry.client.metadata.store.InMemoryWritableMetadataService; import org.apache.dubbo.registry.support.RegistryManager; import org.apache.dubbo.rpc.model.ApplicationModel; @@ -172,17 +173,16 @@ public class ServiceInstanceMetadataUtilsTest { ServiceDiscovery serviceDiscovery = Mockito.mock(ServiceDiscovery.class); - WritableMetadataService writableMetadataService = WritableMetadataService.getDefaultExtension(serviceInstance.getApplicationModel()); - Map metadataInfoMap = writableMetadataService.getMetadataInfos(); + InMemoryWritableMetadataService writableMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(serviceInstance.getApplicationModel()); MetadataInfo metadataInfo = new MetadataInfo("demo"); metadataInfo.addService(new MetadataInfo.ServiceInfo(url1)); - metadataInfoMap.put(DEFAULT_KEY, metadataInfo); + writableMetadataService.addMetadataInfo(DEFAULT_KEY, metadataInfo); ServiceInstanceMetadataUtils.calInstanceRevision(serviceDiscovery, serviceInstance); Assertions.assertEquals(metadataInfo.calAndGetRevision(), serviceInstance.getMetadata().get(EXPORTED_SERVICES_REVISION_PROPERTY_NAME)); Assertions.assertNull(serviceInstance.getExtendParams().get(INSTANCE_REVISION_UPDATED_KEY)); - metadataInfoMap.get(DEFAULT_KEY).addService(new MetadataInfo.ServiceInfo(url2)); + writableMetadataService.getMetadataInfos().get(DEFAULT_KEY).addService(new MetadataInfo.ServiceInfo(url2)); ServiceInstanceMetadataUtils.calInstanceRevision(serviceDiscovery, serviceInstance); Assertions.assertEquals(metadataInfo.calAndGetRevision(), serviceInstance.getMetadata().get(EXPORTED_SERVICES_REVISION_PROPERTY_NAME)); Assertions.assertEquals(serviceInstance.getExtendParams().get(INSTANCE_REVISION_UPDATED_KEY), "true"); diff --git a/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/migration/MigrationInvokerTest.java b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/migration/MigrationInvokerTest.java index 3a4cc359cd..c21a2e9f53 100644 --- a/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/migration/MigrationInvokerTest.java +++ b/dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/migration/MigrationInvokerTest.java @@ -20,12 +20,13 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.config.ApplicationConfig; import org.apache.dubbo.registry.client.migration.model.MigrationRule; import org.apache.dubbo.registry.client.migration.model.MigrationStep; +import org.apache.dubbo.registry.integration.DemoService; import org.apache.dubbo.registry.integration.DynamicDirectory; import org.apache.dubbo.registry.integration.RegistryProtocol; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.cluster.ClusterInvoker; import org.apache.dubbo.rpc.model.ApplicationModel; - +import org.apache.dubbo.rpc.model.FrameworkModel; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -39,7 +40,7 @@ import java.util.List; public class MigrationInvokerTest { @BeforeEach public void before() { - ApplicationModel.reset(); + FrameworkModel.destroyAll(); ApplicationConfig applicationConfig = new ApplicationConfig(); applicationConfig.setName("Test"); ApplicationModel.defaultModel().getApplicationConfigManager().setApplication(applicationConfig); @@ -47,7 +48,7 @@ public class MigrationInvokerTest { @AfterEach public void after() { - ApplicationModel.reset(); + FrameworkModel.destroyAll(); } @Test @@ -88,7 +89,7 @@ public class MigrationInvokerTest { Mockito.when(invoker.getUrl()).thenReturn(consumerURL); Mockito.when(serviceDiscoveryInvoker.getUrl()).thenReturn(consumerURL); - MigrationInvoker migrationInvoker = new MigrationInvoker(registryProtocol, null, null, null, null, consumerURL); + MigrationInvoker migrationInvoker = new MigrationInvoker(registryProtocol, null, null, DemoService.class, null, consumerURL); MigrationRule migrationRule = Mockito.mock(MigrationRule.class); Mockito.when(migrationRule.getForce(Mockito.any())).thenReturn(true); diff --git a/dubbo-registry/dubbo-registry-api/src/test/resources/log4j.xml b/dubbo-registry/dubbo-registry-api/src/test/resources/log4j.xml index de4c580e98..3a4fc863c7 100644 --- a/dubbo-registry/dubbo-registry-api/src/test/resources/log4j.xml +++ b/dubbo-registry/dubbo-registry-api/src/test/resources/log4j.xml @@ -23,7 +23,7 @@ - + - \ No newline at end of file + diff --git a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistry.java b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistry.java index e4940b0aeb..f3b8f72bf1 100644 --- a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistry.java +++ b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistry.java @@ -67,7 +67,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { private final ConcurrentMap> zkListeners = new ConcurrentHashMap<>(); - private final ZookeeperClient zkClient; + private ZookeeperClient zkClient; public ZookeeperRegistry(URL url, ZookeeperTransporter zookeeperTransporter) { super(url); @@ -106,22 +106,27 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { @Override public boolean isAvailable() { - return zkClient.isConnected(); + return zkClient != null && zkClient.isConnected(); } @Override public void destroy() { super.destroy(); - try { - zkClient.close(); - } catch (Exception e) { - logger.warn("Failed to close zookeeper client " + getUrl() + ", cause: " + e.getMessage(), e); + // Just release zkClient reference, but can not close zk client here for zk client is shared somewhere else. + // See org.apache.dubbo.remoting.zookeeper.AbstractZookeeperTransporter#destroy() + zkClient = null; + } + + private void checkDestroyed() { + if (zkClient == null) { + throw new IllegalStateException("registry is destroyed"); } } @Override public void doRegister(URL url) { try { + checkDestroyed(); zkClient.create(toUrlPath(url), url.getParameter(DYNAMIC_KEY, true)); } catch (Throwable e) { throw new RpcException("Failed to register " + url + " to zookeeper " + getUrl() + ", cause: " + e.getMessage(), e); @@ -131,6 +136,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { @Override public void doUnregister(URL url) { try { + checkDestroyed(); zkClient.delete(toUrlPath(url)); } catch (Throwable e) { throw new RpcException("Failed to unregister " + url + " to zookeeper " + getUrl() + ", cause: " + e.getMessage(), e); @@ -140,6 +146,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { @Override public void doSubscribe(final URL url, final NotifyListener listener) { try { + checkDestroyed(); if (ANY_VALUE.equals(url.getServiceInterface())) { String root = toRootPath(); boolean check = url.getParameter(CHECK_KEY, false); @@ -193,6 +200,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { @Override public void doUnsubscribe(URL url, NotifyListener listener) { + checkDestroyed(); ConcurrentMap listeners = zkListeners.get(url); if (listeners != null) { ChildListener zkListener = listeners.remove(listener); @@ -219,6 +227,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry { throw new IllegalArgumentException("lookup url == null"); } try { + checkDestroyed(); List providers = new ArrayList<>(); for (String path : toCategoriesPath(url)) { List children = zkClient.getChildren(path); diff --git a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryFactory.java b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryFactory.java index 011683f964..07858b68e1 100644 --- a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryFactory.java +++ b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryFactory.java @@ -21,6 +21,7 @@ import org.apache.dubbo.common.extension.DisableInject; import org.apache.dubbo.registry.Registry; import org.apache.dubbo.registry.support.AbstractRegistryFactory; import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; +import org.apache.dubbo.rpc.model.ApplicationModel; /** * ZookeeperRegistryFactory. @@ -29,8 +30,16 @@ public class ZookeeperRegistryFactory extends AbstractRegistryFactory { private ZookeeperTransporter zookeeperTransporter; + private ApplicationModel applicationModel; + + // for compatible usage public ZookeeperRegistryFactory() { - this.zookeeperTransporter = ZookeeperTransporter.getExtension(); + this(ApplicationModel.defaultModel()); + } + + public ZookeeperRegistryFactory(ApplicationModel applicationModel) { + this.applicationModel = applicationModel; + this.zookeeperTransporter = ZookeeperTransporter.getExtension(applicationModel); } @DisableInject diff --git a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperServiceDiscovery.java b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperServiceDiscovery.java index 0cc65da9b4..fc8e59641f 100644 --- a/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperServiceDiscovery.java +++ b/dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperServiceDiscovery.java @@ -87,6 +87,7 @@ public class ZookeeperServiceDiscovery extends AbstractServiceDiscovery { @Override public void doDestroy() throws Exception { serviceDiscovery.close(); + curatorFramework.close(); } @Override diff --git a/dubbo-registry/dubbo-registry-zookeeper/src/test/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryTest.java b/dubbo-registry/dubbo-registry-zookeeper/src/test/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryTest.java index 2052c3beb5..3cf5d80f29 100644 --- a/dubbo-registry/dubbo-registry-zookeeper/src/test/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryTest.java +++ b/dubbo-registry/dubbo-registry-zookeeper/src/test/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistryTest.java @@ -16,6 +16,7 @@ */ package org.apache.dubbo.registry.zookeeper; +import org.apache.curator.test.TestingServer; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.status.Status; import org.apache.dubbo.common.utils.NetUtils; @@ -24,8 +25,6 @@ import org.apache.dubbo.registry.Registry; import org.apache.dubbo.registry.status.RegistryStatusChecker; import org.apache.dubbo.rpc.RpcException; import org.apache.dubbo.rpc.model.ApplicationModel; - -import org.apache.curator.test.TestingServer; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -62,7 +61,7 @@ public class ZookeeperRegistryTest { this.zkServer.start(); this.registryUrl = URL.valueOf("zookeeper://localhost:" + zkServerPort); - zookeeperRegistryFactory = new ZookeeperRegistryFactory(); + zookeeperRegistryFactory = new ZookeeperRegistryFactory(ApplicationModel.defaultModel()); this.zookeeperRegistry = (ZookeeperRegistry) zookeeperRegistryFactory.createRegistry(registryUrl); } @@ -75,7 +74,7 @@ public class ZookeeperRegistryTest { public void testAnyHost() { Assertions.assertThrows(IllegalStateException.class, () -> { URL errorUrl = URL.valueOf("multicast://0.0.0.0/"); - new ZookeeperRegistryFactory().createRegistry(errorUrl); + new ZookeeperRegistryFactory(ApplicationModel.defaultModel()).createRegistry(errorUrl); }); } diff --git a/dubbo-registry/pom.xml b/dubbo-registry/pom.xml index a652eb894c..0f9a7871e3 100644 --- a/dubbo-registry/pom.xml +++ b/dubbo-registry/pom.xml @@ -39,4 +39,13 @@ dubbo-registry-dns dubbo-registry-xds + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/RemotingScopeModelInitializer.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/RemotingScopeModelInitializer.java new file mode 100644 index 0000000000..706d7ccbfc --- /dev/null +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/RemotingScopeModelInitializer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.remoting; + +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; +import org.apache.dubbo.rpc.model.ApplicationModel; +import org.apache.dubbo.rpc.model.FrameworkModel; +import org.apache.dubbo.rpc.model.ModuleModel; +import org.apache.dubbo.rpc.model.ScopeModelInitializer; + +import java.util.List; + +/** + * Scope model initializer for remoting-api + */ +public class RemotingScopeModelInitializer implements ScopeModelInitializer { + + private static final Logger logger = LoggerFactory.getLogger(RemotingScopeModelInitializer.class); + + @Override + public void initializeFrameworkModel(FrameworkModel frameworkModel) { + + } + + @Override + public void initializeApplicationModel(ApplicationModel applicationModel) { + applicationModel.addDestroyListener(m -> { + // destroy zookeeper clients if any + try { + List transporters = applicationModel.getExtensionLoader(ZookeeperTransporter.class).getLoadedExtensionInstances(); + for (ZookeeperTransporter zkTransporter : transporters) { + zkTransporter.destroy(); + } + } catch (Exception e) { + logger.error("Error encountered while destroying ZookeeperTransporter: " + e.getMessage(), e); + } + }); + } + + @Override + public void initializeModuleModel(ModuleModel moduleModel) { + + } +} diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/Connection.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/Connection.java index a675ef4016..9fc91154e5 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/Connection.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/Connection.java @@ -90,7 +90,7 @@ public class Connection extends AbstractReferenceCounted { private Bootstrap create() { final Bootstrap bootstrap = new Bootstrap(); - bootstrap.group(NettyEventLoopFactory.NIO_EVENT_LOOP_GROUP) + bootstrap.group(NettyEventLoopFactory.NIO_EVENT_LOOP_GROUP.get()) .option(ChannelOption.SO_KEEPALIVE, true) .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/NettyEventLoopFactory.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/NettyEventLoopFactory.java index c766c073a9..09d4baff04 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/NettyEventLoopFactory.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/api/NettyEventLoopFactory.java @@ -16,8 +16,6 @@ */ package org.apache.dubbo.remoting.api; -import org.apache.dubbo.remoting.Constants; - import io.netty.channel.EventLoopGroup; import io.netty.channel.epoll.Epoll; import io.netty.channel.epoll.EpollEventLoopGroup; @@ -29,6 +27,8 @@ import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.util.concurrent.DefaultThreadFactory; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; +import org.apache.dubbo.remoting.Constants; import java.util.concurrent.ThreadFactory; @@ -40,7 +40,10 @@ public class NettyEventLoopFactory { /** * netty client bootstrap */ - public static final EventLoopGroup NIO_EVENT_LOOP_GROUP = eventLoopGroup(Constants.DEFAULT_IO_THREADS, "NettyClientWorker"); + public static final GlobalResourceInitializer NIO_EVENT_LOOP_GROUP = new GlobalResourceInitializer<>(() -> + eventLoopGroup(Constants.DEFAULT_IO_THREADS, "NettyClientWorker"), + eventLoopGroup -> eventLoopGroup.shutdownGracefully() + ); public static EventLoopGroup eventLoopGroup(int threads, String threadFactoryName) { ThreadFactory threadFactory = new DefaultThreadFactory(threadFactoryName, true); diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture.java index f97a0f7b7c..f1f0ed19fc 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture.java @@ -18,6 +18,7 @@ package org.apache.dubbo.remoting.exchange.support; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; import org.apache.dubbo.common.threadpool.ThreadlessExecutor; import org.apache.dubbo.common.timer.HashedWheelTimer; import org.apache.dubbo.common.timer.Timeout; @@ -52,10 +53,9 @@ public class DefaultFuture extends CompletableFuture { private static final Map FUTURES = new ConcurrentHashMap<>(); - public static final Timer TIME_OUT_TIMER = new HashedWheelTimer( - new NamedThreadFactory("dubbo-future-timeout", true), - 30, - TimeUnit.MILLISECONDS); + private static GlobalResourceInitializer TIME_OUT_TIMER = new GlobalResourceInitializer<>(() -> new HashedWheelTimer( + new NamedThreadFactory("dubbo-future-timeout", true), 30, TimeUnit.MILLISECONDS), + () -> destroy()); // invoke id. private final Long id; @@ -91,7 +91,13 @@ public class DefaultFuture extends CompletableFuture { */ private static void timeoutCheck(DefaultFuture future) { TimeoutCheckTask task = new TimeoutCheckTask(future.getId()); - future.timeoutCheckTask = TIME_OUT_TIMER.newTimeout(task, future.getTimeout(), TimeUnit.MILLISECONDS); + future.timeoutCheckTask = TIME_OUT_TIMER.get().newTimeout(task, future.getTimeout(), TimeUnit.MILLISECONDS); + } + + public static void destroy() { + TIME_OUT_TIMER.remove(timer-> timer.stop()); + FUTURES.clear(); + CHANNELS.clear(); } /** diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture2.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture2.java index 2e995ee56c..de8baf1a59 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture2.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/DefaultFuture2.java @@ -18,6 +18,7 @@ package org.apache.dubbo.remoting.exchange.support; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; import org.apache.dubbo.common.threadpool.ThreadlessExecutor; import org.apache.dubbo.common.timer.HashedWheelTimer; import org.apache.dubbo.common.timer.Timeout; @@ -46,12 +47,14 @@ import java.util.concurrent.TimeUnit; */ public class DefaultFuture2 extends CompletableFuture { - public static final Timer TIME_OUT_TIMER = new HashedWheelTimer( - new NamedThreadFactory("dubbo-future-timeout", true), - 30, - TimeUnit.MILLISECONDS); + private static GlobalResourceInitializer TIME_OUT_TIMER = new GlobalResourceInitializer<>( + () -> new HashedWheelTimer(new NamedThreadFactory("dubbo-future-timeout", true), + 30, TimeUnit.MILLISECONDS), + () -> destroy()); + private static final Logger logger = LoggerFactory.getLogger(DefaultFuture2.class); private static final Map FUTURES = new ConcurrentHashMap<>(); + // invoke id. private final Request request; private final Connection connection; @@ -90,7 +93,12 @@ public class DefaultFuture2 extends CompletableFuture { */ private static void timeoutCheck(DefaultFuture2 future) { TimeoutCheckTask task = new TimeoutCheckTask(future.getId()); - future.timeoutCheckTask = TIME_OUT_TIMER.newTimeout(task, future.getTimeout(), TimeUnit.MILLISECONDS); + future.timeoutCheckTask = TIME_OUT_TIMER.get().newTimeout(task, future.getTimeout(), TimeUnit.MILLISECONDS); + } + + public static void destroy() { + TIME_OUT_TIMER.remove(timer-> timer.stop()); + FUTURES.clear(); } /** diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeClient.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeClient.java index ced7dbab94..81df8653ed 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeClient.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeClient.java @@ -17,7 +17,9 @@ package org.apache.dubbo.remoting.exchange.support.header; import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; import org.apache.dubbo.common.timer.HashedWheelTimer; +import org.apache.dubbo.common.timer.Timeout; import org.apache.dubbo.common.utils.Assert; import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.remoting.ChannelHandler; @@ -48,10 +50,13 @@ public class HeaderExchangeClient implements ExchangeClient { private final Client client; private final ExchangeChannel channel; - private static final HashedWheelTimer IDLE_CHECK_TIMER = new HashedWheelTimer( - new NamedThreadFactory("dubbo-client-idleCheck", true), 1, TimeUnit.SECONDS, TICKS_PER_WHEEL); - private HeartbeatTimerTask heartBeatTimerTask; - private ReconnectTimerTask reconnectTimerTask; + public static GlobalResourceInitializer IDLE_CHECK_TIMER = new GlobalResourceInitializer<>(() -> + new HashedWheelTimer(new NamedThreadFactory("dubbo-client-idleCheck", true), 1, + TimeUnit.SECONDS, TICKS_PER_WHEEL), + timer -> timer.stop()); + + private Timeout reconnectTimer; + private Timeout heartBeatTimer; public HeaderExchangeClient(Client client, boolean startTimer) { Assert.notNull(client, "Client can't be null"); @@ -191,8 +196,8 @@ public class HeaderExchangeClient implements ExchangeClient { AbstractTimerTask.ChannelProvider cp = () -> Collections.singletonList(HeaderExchangeClient.this); int heartbeat = getHeartbeat(url); long heartbeatTick = calculateLeastDuration(heartbeat); - this.heartBeatTimerTask = new HeartbeatTimerTask(cp, heartbeatTick, heartbeat); - IDLE_CHECK_TIMER.newTimeout(heartBeatTimerTask, heartbeatTick, TimeUnit.MILLISECONDS); + HeartbeatTimerTask heartBeatTimerTask = new HeartbeatTimerTask(cp, heartbeatTick, heartbeat); + heartBeatTimer = IDLE_CHECK_TIMER.get().newTimeout(heartBeatTimerTask, heartbeatTick, TimeUnit.MILLISECONDS); } } @@ -201,18 +206,19 @@ public class HeaderExchangeClient implements ExchangeClient { AbstractTimerTask.ChannelProvider cp = () -> Collections.singletonList(HeaderExchangeClient.this); int idleTimeout = getIdleTimeout(url); long heartbeatTimeoutTick = calculateLeastDuration(idleTimeout); - this.reconnectTimerTask = new ReconnectTimerTask(cp, heartbeatTimeoutTick, idleTimeout); - IDLE_CHECK_TIMER.newTimeout(reconnectTimerTask, heartbeatTimeoutTick, TimeUnit.MILLISECONDS); + ReconnectTimerTask reconnectTimerTask = new ReconnectTimerTask(cp, heartbeatTimeoutTick, idleTimeout); + reconnectTimer = IDLE_CHECK_TIMER.get().newTimeout(reconnectTimerTask, heartbeatTimeoutTick, TimeUnit.MILLISECONDS); } } private void doClose() { - if (heartBeatTimerTask != null) { - heartBeatTimerTask.cancel(); + if (heartBeatTimer != null) { + heartBeatTimer.cancel(); + heartBeatTimer = null; } - - if (reconnectTimerTask != null) { - reconnectTimerTask.cancel(); + if (reconnectTimer != null) { + reconnectTimer.cancel(); + reconnectTimer = null; } } diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java index 2367465002..d0d940d6d0 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/exchange/support/header/HeaderExchangeServer.java @@ -20,7 +20,9 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.Version; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; import org.apache.dubbo.common.timer.HashedWheelTimer; +import org.apache.dubbo.common.timer.Timeout; import org.apache.dubbo.common.utils.Assert; import org.apache.dubbo.common.utils.CollectionUtils; import org.apache.dubbo.common.utils.NamedThreadFactory; @@ -58,10 +60,12 @@ public class HeaderExchangeServer implements ExchangeServer { private final RemotingServer server; private AtomicBoolean closed = new AtomicBoolean(false); - private static final HashedWheelTimer IDLE_CHECK_TIMER = new HashedWheelTimer(new NamedThreadFactory("dubbo-server-idleCheck", true), 1, - TimeUnit.SECONDS, TICKS_PER_WHEEL); + public static GlobalResourceInitializer IDLE_CHECK_TIMER = new GlobalResourceInitializer<>(() -> + new HashedWheelTimer(new NamedThreadFactory("dubbo-server-idleCheck", true), 1, + TimeUnit.SECONDS, TICKS_PER_WHEEL), + timer -> timer.stop()); - private CloseTimerTask closeTimerTask; + private Timeout closeTimer; public HeaderExchangeServer(RemotingServer server) { Assert.notNull(server, "server == null"); @@ -160,8 +164,8 @@ public class HeaderExchangeServer implements ExchangeServer { } private void cancelCloseTask() { - if (closeTimerTask != null) { - closeTimerTask.cancel(); + if (closeTimer != null) { + closeTimer.cancel(); } } @@ -272,10 +276,9 @@ public class HeaderExchangeServer implements ExchangeServer { int idleTimeout = getIdleTimeout(url); long idleTimeoutTick = calculateLeastDuration(idleTimeout); CloseTimerTask closeTimerTask = new CloseTimerTask(cp, idleTimeoutTick, idleTimeout); - this.closeTimerTask = closeTimerTask; // init task and start timer. - IDLE_CHECK_TIMER.newTimeout(closeTimerTask, idleTimeoutTick, TimeUnit.MILLISECONDS); + this.closeTimer = IDLE_CHECK_TIMER.get().newTimeout(closeTimerTask, idleTimeoutTick, TimeUnit.MILLISECONDS); } } } diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/transport/dispatcher/WrappedChannelHandler.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/transport/dispatcher/WrappedChannelHandler.java index dadada1db6..7742d046b3 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/transport/dispatcher/WrappedChannelHandler.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/transport/dispatcher/WrappedChannelHandler.java @@ -19,6 +19,7 @@ package org.apache.dubbo.remoting.transport.dispatcher; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourcesRepository; import org.apache.dubbo.common.threadpool.manager.ExecutorRepository; import org.apache.dubbo.remoting.Channel; import org.apache.dubbo.remoting.ChannelHandler; @@ -27,6 +28,7 @@ import org.apache.dubbo.remoting.exchange.Request; import org.apache.dubbo.remoting.exchange.Response; import org.apache.dubbo.remoting.exchange.support.DefaultFuture; import org.apache.dubbo.remoting.transport.ChannelHandlerDelegate; +import org.apache.dubbo.rpc.model.ApplicationModel; import java.util.concurrent.ExecutorService; @@ -130,11 +132,17 @@ public class WrappedChannelHandler implements ChannelHandlerDelegate { * @return */ public ExecutorService getSharedExecutorService() { + ApplicationModel applicationModel = url.getOrDefaultApplicationModel(); ExecutorRepository executorRepository = - url.getOrDefaultApplicationModel().getExtensionLoader(ExecutorRepository.class).getDefaultExtension(); + applicationModel.getExtensionLoader(ExecutorRepository.class).getDefaultExtension(); ExecutorService executor = executorRepository.getExecutor(url); if (executor == null) { - executor = executorRepository.createExecutorIfAbsent(url); + // if application is destroyed, use global executor service + if (applicationModel.isDestroyed()) { + executor = GlobalResourcesRepository.getGlobalExecutorService(); + }else { + executor = executorRepository.createExecutorIfAbsent(url); + } } return executor; } diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/AbstractZookeeperClient.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/AbstractZookeeperClient.java index ca620cb985..309a85b646 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/AbstractZookeeperClient.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/AbstractZookeeperClient.java @@ -206,7 +206,10 @@ public abstract class AbstractZookeeperClient getZookeeperClientMap() { return zookeeperClientMap; } + + @Override + public void destroy() { + // only destroy zk clients here + for (ZookeeperClient client : zookeeperClientMap.values()) { + client.close(); + } + zookeeperClientMap.clear(); + } } diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter.java b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter.java index 136d8d3472..f6bc79c2cd 100644 --- a/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter.java +++ b/dubbo-remoting/dubbo-remoting-api/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter.java @@ -20,10 +20,9 @@ import org.apache.dubbo.common.URL; import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.common.extension.ExtensionScope; import org.apache.dubbo.common.extension.SPI; +import org.apache.dubbo.rpc.model.ApplicationModel; -import static org.apache.dubbo.common.extension.ExtensionLoader.getExtensionLoader; - -@SPI(scope = ExtensionScope.FRAMEWORK) +@SPI(scope = ExtensionScope.APPLICATION) public interface ZookeeperTransporter { String CURATOR_5 = "curator5"; @@ -32,8 +31,10 @@ public interface ZookeeperTransporter { ZookeeperClient connect(URL url); - static ZookeeperTransporter getExtension() { - ExtensionLoader extensionLoader = getExtensionLoader(ZookeeperTransporter.class); + void destroy(); + + static ZookeeperTransporter getExtension(ApplicationModel applicationModel) { + ExtensionLoader extensionLoader = applicationModel.getExtensionLoader(ZookeeperTransporter.class); boolean isHighVersion = isHighVersionCurator(); if (isHighVersion) { return extensionLoader.getExtension(CURATOR_5); diff --git a/dubbo-remoting/dubbo-remoting-api/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.model.ScopeModelInitializer b/dubbo-remoting/dubbo-remoting-api/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.model.ScopeModelInitializer new file mode 100644 index 0000000000..1568955be7 --- /dev/null +++ b/dubbo-remoting/dubbo-remoting-api/src/main/resources/META-INF/dubbo/internal/org.apache.dubbo.rpc.model.ScopeModelInitializer @@ -0,0 +1 @@ +dubbo-remoting-api=org.apache.dubbo.remoting.RemotingScopeModelInitializer diff --git a/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyClient.java b/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyClient.java index f18893b3a8..998f5e3fbe 100644 --- a/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyClient.java +++ b/dubbo-remoting/dubbo-remoting-netty4/src/main/java/org/apache/dubbo/remoting/transport/netty4/NettyClient.java @@ -31,6 +31,7 @@ import org.apache.dubbo.common.Version; import org.apache.dubbo.common.config.ConfigurationUtils; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.common.resource.GlobalResourceInitializer; import org.apache.dubbo.common.utils.NetUtils; import org.apache.dubbo.common.utils.StringUtils; import org.apache.dubbo.remoting.ChannelHandler; @@ -53,18 +54,21 @@ import static org.apache.dubbo.remoting.api.NettyEventLoopFactory.socketChannelC */ public class NettyClient extends AbstractClient { - private static final Logger logger = LoggerFactory.getLogger(NettyClient.class); - /** - * netty client bootstrap - */ - private static final EventLoopGroup EVENT_LOOP_GROUP = eventLoopGroup(Constants.DEFAULT_IO_THREADS, "NettyClientWorker"); - private static final String SOCKS_PROXY_HOST = "socksProxyHost"; private static final String SOCKS_PROXY_PORT = "socksProxyPort"; private static final String DEFAULT_SOCKS_PROXY_PORT = "1080"; + private static final Logger logger = LoggerFactory.getLogger(NettyClient.class); + + /** + * netty client bootstrap + */ + private static final GlobalResourceInitializer EVENT_LOOP_GROUP = new GlobalResourceInitializer<>(() -> + eventLoopGroup(Constants.DEFAULT_IO_THREADS, "NettyClientWorker"), + eventLoopGroup -> eventLoopGroup.shutdownGracefully()); + private Bootstrap bootstrap; /** @@ -93,7 +97,7 @@ public class NettyClient extends AbstractClient { protected void doOpen() throws Throwable { final NettyClientHandler nettyClientHandler = new NettyClientHandler(getUrl(), this); bootstrap = new Bootstrap(); - bootstrap.group(EVENT_LOOP_GROUP) + bootstrap.group(EVENT_LOOP_GROUP.get()) .option(ChannelOption.SO_KEEPALIVE, true) .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT) diff --git a/dubbo-remoting/dubbo-remoting-zookeeper-curator5/src/main/java/org/apache/dubbo/remoting/zookeeper/curator5/Curator5ZookeeperClient.java b/dubbo-remoting/dubbo-remoting-zookeeper-curator5/src/main/java/org/apache/dubbo/remoting/zookeeper/curator5/Curator5ZookeeperClient.java index 9fc626a7a7..168ce14aa6 100644 --- a/dubbo-remoting/dubbo-remoting-zookeeper-curator5/src/main/java/org/apache/dubbo/remoting/zookeeper/curator5/Curator5ZookeeperClient.java +++ b/dubbo-remoting/dubbo-remoting-zookeeper-curator5/src/main/java/org/apache/dubbo/remoting/zookeeper/curator5/Curator5ZookeeperClient.java @@ -16,16 +16,6 @@ */ package org.apache.dubbo.remoting.zookeeper.curator5; -import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.config.configcenter.ConfigItem; -import org.apache.dubbo.common.logger.Logger; -import org.apache.dubbo.common.logger.LoggerFactory; -import org.apache.dubbo.remoting.zookeeper.ChildListener; -import org.apache.dubbo.remoting.zookeeper.DataListener; -import org.apache.dubbo.remoting.zookeeper.EventType; -import org.apache.dubbo.remoting.zookeeper.StateListener; -import org.apache.dubbo.remoting.zookeeper.AbstractZookeeperClient; - import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.framework.api.CuratorWatcher; @@ -35,6 +25,15 @@ import org.apache.curator.framework.recipes.cache.NodeCacheListener; import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.curator.retry.RetryNTimes; +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.config.configcenter.ConfigItem; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.remoting.zookeeper.AbstractZookeeperClient; +import org.apache.dubbo.remoting.zookeeper.ChildListener; +import org.apache.dubbo.remoting.zookeeper.DataListener; +import org.apache.dubbo.remoting.zookeeper.EventType; +import org.apache.dubbo.remoting.zookeeper.StateListener; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; @@ -252,6 +251,7 @@ public class Curator5ZookeeperClient extends AbstractZookeeperClientdubbo-remoting-zookeeper-curator5 dubbo-remoting-netty4 + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/filter/AccessLogFilter.java b/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/filter/AccessLogFilter.java index 653a4fcd3a..6c9aa26167 100644 --- a/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/filter/AccessLogFilter.java +++ b/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/filter/AccessLogFilter.java @@ -21,7 +21,6 @@ import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; import org.apache.dubbo.common.utils.ConcurrentHashSet; import org.apache.dubbo.common.utils.ConfigUtils; -import org.apache.dubbo.common.utils.NamedThreadFactory; import org.apache.dubbo.rpc.Filter; import org.apache.dubbo.rpc.Invocation; import org.apache.dubbo.rpc.Invoker; @@ -39,9 +38,8 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.dubbo.common.constants.CommonConstants.PROVIDER; import static org.apache.dubbo.rpc.Constants.ACCESS_LOG_KEY; @@ -74,18 +72,17 @@ public class AccessLogFilter implements Filter { private static final String FILE_DATE_FORMAT = "yyyyMMdd"; // It's safe to declare it as singleton since it runs on single thread only - private static final DateFormat FILE_NAME_FORMATTER = new SimpleDateFormat(FILE_DATE_FORMAT); + private final DateFormat fileNameFormatter = new SimpleDateFormat(FILE_DATE_FORMAT); - private static final Map> LOG_ENTRIES = new ConcurrentHashMap<>(); + private final Map> logEntries = new ConcurrentHashMap<>(); - private static final ScheduledExecutorService LOG_SCHEDULED = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Dubbo-Access-Log", true)); + private AtomicBoolean scheduled = new AtomicBoolean(); /** * Default constructor initialize demon thread for writing into access log file with names with access log key * defined in url accesslog */ public AccessLogFilter() { - LOG_SCHEDULED.scheduleWithFixedDelay(this::writeLogToFile, LOG_OUTPUT_INTERVAL, LOG_OUTPUT_INTERVAL, TimeUnit.MILLISECONDS); } /** @@ -98,6 +95,10 @@ public class AccessLogFilter implements Filter { */ @Override public Result invoke(Invoker invoker, Invocation inv) throws RpcException { + if (scheduled.compareAndSet(false, true)) { + inv.getModuleModel().getApplicationModel().getApplicationExecutorRepository().getSharedScheduledExecutor() + .scheduleWithFixedDelay(this::writeLogToFile, LOG_OUTPUT_INTERVAL, LOG_OUTPUT_INTERVAL, TimeUnit.MILLISECONDS); + } try { String accessLogKey = invoker.getUrl().getParameter(ACCESS_LOG_KEY); if (ConfigUtils.isNotEmpty(accessLogKey)) { @@ -112,7 +113,7 @@ public class AccessLogFilter implements Filter { } private void log(String accessLog, AccessLogData accessLogData) { - Set logSet = LOG_ENTRIES.computeIfAbsent(accessLog, k -> new ConcurrentHashSet<>()); + Set logSet = logEntries.computeIfAbsent(accessLog, k -> new ConcurrentHashSet<>()); if (logSet.size() < LOG_MAX_BUFFER) { logSet.add(accessLogData); @@ -144,8 +145,8 @@ public class AccessLogFilter implements Filter { } private void writeLogToFile() { - if (!LOG_ENTRIES.isEmpty()) { - for (Map.Entry> entry : LOG_ENTRIES.entrySet()) { + if (!logEntries.isEmpty()) { + for (Map.Entry> entry : logEntries.entrySet()) { String accessLog = entry.getKey(); Set logSet = entry.getValue(); writeLogSetToFile(accessLog, logSet); @@ -195,8 +196,8 @@ public class AccessLogFilter implements Filter { private void renameFile(File file) { if (file.exists()) { - String now = FILE_NAME_FORMATTER.format(new Date()); - String last = FILE_NAME_FORMATTER.format(new Date(file.lastModified())); + String now = fileNameFormatter.format(new Date()); + String last = fileNameFormatter.format(new Date(file.lastModified())); if (!now.equals(last)) { File archive = new File(file.getAbsolutePath() + "." + last); file.renameTo(archive); diff --git a/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/protocol/AbstractProtocol.java b/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/protocol/AbstractProtocol.java index 4192ba7217..388aa60d54 100644 --- a/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/protocol/AbstractProtocol.java +++ b/dubbo-rpc/dubbo-rpc-api/src/main/java/org/apache/dubbo/rpc/protocol/AbstractProtocol.java @@ -94,7 +94,6 @@ public abstract class AbstractProtocol implements Protocol, ScopeModelAware { public void destroy() { for (Invoker invoker : invokers) { if (invoker != null) { - invokers.remove(invoker); try { if (logger.isInfoEnabled()) { logger.info("Destroy reference: " + invoker.getUrl()); @@ -105,8 +104,9 @@ public abstract class AbstractProtocol implements Protocol, ScopeModelAware { } } } - for (String key : new ArrayList<>(exporterMap.keySet())) { - Exporter exporter = exporterMap.remove(key); + invokers.clear(); + + exporterMap.forEach((key, exporter)-> { if (exporter != null) { try { if (logger.isInfoEnabled()) { @@ -117,7 +117,8 @@ public abstract class AbstractProtocol implements Protocol, ScopeModelAware { logger.warn(t.getMessage(), t); } } - } + }); + exporterMap.clear(); } @Override diff --git a/dubbo-rpc/dubbo-rpc-api/src/test/java/org/apache/dubbo/rpc/filter/AccessLogFilterTest.java b/dubbo-rpc/dubbo-rpc-api/src/test/java/org/apache/dubbo/rpc/filter/AccessLogFilterTest.java index 53e1537ed5..5b87b88777 100644 --- a/dubbo-rpc/dubbo-rpc-api/src/test/java/org/apache/dubbo/rpc/filter/AccessLogFilterTest.java +++ b/dubbo-rpc/dubbo-rpc-api/src/test/java/org/apache/dubbo/rpc/filter/AccessLogFilterTest.java @@ -25,7 +25,6 @@ import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.support.AccessLogData; import org.apache.dubbo.rpc.support.MockInvocation; import org.apache.dubbo.rpc.support.MyInvoker; - import org.junit.jupiter.api.Test; import java.lang.reflect.Field; @@ -63,13 +62,13 @@ public class AccessLogFilterTest { Invoker invoker = new MyInvoker(url); Invocation invocation = new MockInvocation(); - Field field = AccessLogFilter.class.getDeclaredField("LOG_ENTRIES"); + Field field = AccessLogFilter.class.getDeclaredField("logEntries"); field.setAccessible(true); - assertTrue(((Map) field.get(AccessLogFilter.class)).isEmpty()); + assertTrue(((Map) field.get(accessLogFilter)).isEmpty()); accessLogFilter.invoke(invoker, invocation); - Map> logs = (Map>) field.get(AccessLogFilter.class); + Map> logs = (Map>) field.get(accessLogFilter); assertFalse(logs.isEmpty()); assertFalse(logs.get("true").isEmpty()); AccessLogData log = logs.get("true").iterator().next(); diff --git a/dubbo-rpc/dubbo-rpc-dubbo/src/main/java/org/apache/dubbo/rpc/protocol/dubbo/DubboProtocol.java b/dubbo-rpc/dubbo-rpc-dubbo/src/main/java/org/apache/dubbo/rpc/protocol/dubbo/DubboProtocol.java index 2555aab9e1..7605959ea1 100644 --- a/dubbo-rpc/dubbo-rpc-dubbo/src/main/java/org/apache/dubbo/rpc/protocol/dubbo/DubboProtocol.java +++ b/dubbo-rpc/dubbo-rpc-dubbo/src/main/java/org/apache/dubbo/rpc/protocol/dubbo/DubboProtocol.java @@ -57,6 +57,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import static org.apache.dubbo.common.constants.CommonConstants.GROUP_KEY; @@ -104,6 +105,8 @@ public class DubboProtocol extends AbstractProtocol { private static final Object PENDING_OBJECT = new Object(); private final Set optimizers = new ConcurrentHashSet<>(); + private AtomicBoolean destroyed = new AtomicBoolean(); + private ExchangeHandler requestHandler = new ExchangeHandlerAdapter() { @Override @@ -286,6 +289,7 @@ public class DubboProtocol extends AbstractProtocol { @Override public Exporter export(Invoker invoker) throws RpcException { + checkDestroyed(); URL url = invoker.getUrl(); // export service. @@ -314,6 +318,7 @@ public class DubboProtocol extends AbstractProtocol { } private void openServer(URL url) { + checkDestroyed(); // find server. String key = url.getAddress(); //client can export a service which's only for server to invoke @@ -336,6 +341,12 @@ public class DubboProtocol extends AbstractProtocol { } } + private void checkDestroyed() { + if (destroyed.get()) { + throw new IllegalStateException( getClass().getSimpleName() + " is destroyed"); + } + } + private ProtocolServer createServer(URL url) { url = URLBuilder.from(url) // send readonly event when server closes, it's enabled by default @@ -407,11 +418,13 @@ public class DubboProtocol extends AbstractProtocol { @Override public Invoker refer(Class type, URL url) throws RpcException { + checkDestroyed(); return protocolBindingRefer(type, url); } @Override public Invoker protocolBindingRefer(Class serviceType, URL url) throws RpcException { + checkDestroyed(); optimizeSerialization(url); // create rpc invoker. @@ -644,6 +657,12 @@ public class DubboProtocol extends AbstractProtocol { @Override @SuppressWarnings("unchecked") public void destroy() { + if (!destroyed.compareAndSet(false, true)) { + return; + } + if (logger.isInfoEnabled()) { + logger.info("Destroying protocol [" + this.getClass().getSimpleName() + "] ..."); + } for (String key : new ArrayList<>(serverMap.keySet())) { ProtocolServer protocolServer = serverMap.remove(key); @@ -664,6 +683,7 @@ public class DubboProtocol extends AbstractProtocol { logger.warn("Close dubbo server [" + server.getLocalAddress()+ "] failed: " + t.getMessage(), t); } } + serverMap.clear(); for (String key : new ArrayList<>(referenceClientMap.keySet())) { Object clients = referenceClientMap.remove(key); @@ -679,6 +699,7 @@ public class DubboProtocol extends AbstractProtocol { } } } + referenceClientMap.clear(); super.destroy(); } diff --git a/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/decode/DubboTelnetDecodeTest.java b/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/decode/DubboTelnetDecodeTest.java index 1ef1702cb7..b11cab09b3 100644 --- a/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/decode/DubboTelnetDecodeTest.java +++ b/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/decode/DubboTelnetDecodeTest.java @@ -52,6 +52,8 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.dubbo.rpc.Constants.SERIALIZATION_SECURITY_CHECK_KEY; + /** * These junit tests aim to test unpack and stick pack of dubbo and telnet */ @@ -72,11 +74,17 @@ public class DubboTelnetDecodeTest { public static void setup() { ModuleServiceRepository serviceRepository = ApplicationModel.defaultModel().getDefaultModule().getServiceRepository(); serviceRepository.registerService(DemoService.class); + + // disable org.apache.dubbo.remoting.transport.CodecSupport.checkSerialization to avoid error: + // java.io.IOException: Service org.apache.dubbo.rpc.protocol.dubbo.support.DemoService with version 0.0.0 not found, invocation rejected. + System.setProperty(SERIALIZATION_SECURITY_CHECK_KEY, "false"); } @AfterAll public static void teardown() { FrameworkModel.defaultModel().destroy(); + System.clearProperty(SERIALIZATION_SECURITY_CHECK_KEY); + } /** @@ -470,6 +478,9 @@ public class DubboTelnetDecodeTest { DubboCodec dubboCodec = new DubboCodec(FrameworkModel.defaultModel()); dubboCodec.encode(new MockChannel(), buffer, request); + // register + // frameworkModel.getServiceRepository().registerProviderUrl(); + return dubboByteBuf; } diff --git a/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/support/ProtocolUtils.java b/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/support/ProtocolUtils.java index 533e408be7..e614b2652e 100644 --- a/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/support/ProtocolUtils.java +++ b/dubbo-rpc/dubbo-rpc-dubbo/src/test/java/org/apache/dubbo/rpc/protocol/dubbo/support/ProtocolUtils.java @@ -17,33 +17,34 @@ package org.apache.dubbo.rpc.protocol.dubbo.support; import org.apache.dubbo.common.URL; -import org.apache.dubbo.common.extension.ExtensionLoader; import org.apache.dubbo.rpc.Exporter; import org.apache.dubbo.rpc.Invoker; import org.apache.dubbo.rpc.Protocol; -import org.apache.dubbo.rpc.ProtocolServer; import org.apache.dubbo.rpc.ProxyFactory; +import org.apache.dubbo.rpc.model.FrameworkModel; import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol; -import java.util.List; - /** * TODO Comment of ProtocolUtils */ public class ProtocolUtils { - public static ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); - private static Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension(); public static T refer(Class type, String url) { return refer(type, URL.valueOf(url)); } public static T refer(Class type, URL url) { + FrameworkModel frameworkModel = url.getOrDefaultFrameworkModel(); + ProxyFactory proxy = frameworkModel.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); + Protocol protocol = frameworkModel.getExtensionLoader(Protocol.class).getAdaptiveExtension(); return proxy.getProxy(protocol.refer(type, url)); } public static Invoker referInvoker(Class type, URL url) { + FrameworkModel frameworkModel = url.getOrDefaultFrameworkModel(); + ProxyFactory proxy = frameworkModel.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); + Protocol protocol = frameworkModel.getExtensionLoader(Protocol.class).getAdaptiveExtension(); return (Invoker) protocol.refer(type, url); } @@ -52,14 +53,14 @@ public class ProtocolUtils { } public static Exporter export(T instance, Class type, URL url) { + FrameworkModel frameworkModel = url.getOrDefaultFrameworkModel(); + ProxyFactory proxy = frameworkModel.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension(); + Protocol protocol = frameworkModel.getExtensionLoader(Protocol.class).getAdaptiveExtension(); return protocol.export(proxy.getInvoker(instance, type, url)); } public static void closeAll() { DubboProtocol.getDubboProtocol().destroy(); - List servers = DubboProtocol.getDubboProtocol().getServers(); - for (ProtocolServer server : servers) { - server.close(); - } + FrameworkModel.destroyAll(); } } diff --git a/dubbo-rpc/dubbo-rpc-dubbo/src/test/resources/log4j.xml b/dubbo-rpc/dubbo-rpc-dubbo/src/test/resources/log4j.xml index 09dba05e6c..4d4f4b6969 100644 --- a/dubbo-rpc/dubbo-rpc-dubbo/src/test/resources/log4j.xml +++ b/dubbo-rpc/dubbo-rpc-dubbo/src/test/resources/log4j.xml @@ -31,8 +31,16 @@ --> + + + + + + + + diff --git a/dubbo-rpc/dubbo-rpc-grpc/src/main/java/org/apache/dubbo/rpc/protocol/grpc/GrpcProtocol.java b/dubbo-rpc/dubbo-rpc-grpc/src/main/java/org/apache/dubbo/rpc/protocol/grpc/GrpcProtocol.java index 0e80932391..c3128ae522 100644 --- a/dubbo-rpc/dubbo-rpc-grpc/src/main/java/org/apache/dubbo/rpc/protocol/grpc/GrpcProtocol.java +++ b/dubbo-rpc/dubbo-rpc-grpc/src/main/java/org/apache/dubbo/rpc/protocol/grpc/GrpcProtocol.java @@ -16,6 +16,12 @@ */ package org.apache.dubbo.rpc.protocol.grpc; +import io.grpc.BindableService; +import io.grpc.CallOptions; +import io.grpc.Channel; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.netty.NettyServerBuilder; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; @@ -27,13 +33,6 @@ import org.apache.dubbo.rpc.model.FrameworkServiceRepository; import org.apache.dubbo.rpc.model.ProviderModel; import org.apache.dubbo.rpc.protocol.AbstractProxyProtocol; -import io.grpc.BindableService; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ManagedChannel; -import io.grpc.Server; -import io.grpc.netty.NettyServerBuilder; - import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -190,6 +189,9 @@ public class GrpcProtocol extends AbstractProxyProtocol { @Override public void destroy() { + if (logger.isInfoEnabled()) { + logger.info("Destroying protocol [" + this.getClass().getSimpleName() + "] ..."); + } serverMap.values().forEach(ProtocolServer::close); channelMap.values().forEach(ReferenceCountManagedChannel::shutdown); serverMap.clear(); diff --git a/dubbo-rpc/dubbo-rpc-rest/src/main/java/org/apache/dubbo/rpc/protocol/rest/RestProtocol.java b/dubbo-rpc/dubbo-rpc-rest/src/main/java/org/apache/dubbo/rpc/protocol/rest/RestProtocol.java index 1e658b6548..0c8720c517 100644 --- a/dubbo-rpc/dubbo-rpc-rest/src/main/java/org/apache/dubbo/rpc/protocol/rest/RestProtocol.java +++ b/dubbo-rpc/dubbo-rpc-rest/src/main/java/org/apache/dubbo/rpc/protocol/rest/RestProtocol.java @@ -24,7 +24,6 @@ import org.apache.dubbo.remoting.http.servlet.ServletManager; import org.apache.dubbo.rpc.ProtocolServer; import org.apache.dubbo.rpc.RpcException; import org.apache.dubbo.rpc.protocol.AbstractProxyProtocol; - import org.apache.http.HeaderElement; import org.apache.http.HeaderElementIterator; import org.apache.http.client.config.RequestConfig; @@ -205,6 +204,9 @@ public class RestProtocol extends AbstractProxyProtocol { @Override public void destroy() { + if (logger.isInfoEnabled()) { + logger.info("Destroying protocol [" + this.getClass().getSimpleName() + "] ..."); + } super.destroy(); if (connectionMonitor != null) { diff --git a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleProtocol.java b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleProtocol.java index 8a5a6c8e0d..940a824690 100644 --- a/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleProtocol.java +++ b/dubbo-rpc/dubbo-rpc-triple/src/main/java/org/apache/dubbo/rpc/protocol/tri/TripleProtocol.java @@ -16,6 +16,7 @@ */ package org.apache.dubbo.rpc.protocol.tri; +import grpc.health.v1.HealthCheckResponse; import org.apache.dubbo.common.URL; import org.apache.dubbo.common.logger.Logger; import org.apache.dubbo.common.logger.LoggerFactory; @@ -32,10 +33,6 @@ import org.apache.dubbo.rpc.protocol.AbstractExporter; import org.apache.dubbo.rpc.protocol.AbstractProtocol; import org.apache.dubbo.rpc.protocol.tri.service.TriBuiltinService; -import grpc.health.v1.HealthCheckResponse; - -import java.util.ArrayList; - import static org.apache.dubbo.common.constants.CommonConstants.DEFAULT_CLIENT_THREADPOOL; import static org.apache.dubbo.common.constants.CommonConstants.THREADPOOL_KEY; @@ -110,21 +107,11 @@ public class TripleProtocol extends AbstractProtocol implements Protocol { @Override public void destroy() { + if (logger.isInfoEnabled()) { + logger.info("Destroying protocol [" + this.getClass().getSimpleName() + "] ..."); + } PortUnificationExchanger.close(); pathResolver.destroy(); - for (String key : new ArrayList<>(exporterMap.keySet())) { - Exporter exporter = exporterMap.remove(key); - if (exporter != null) { - try { - if (logger.isInfoEnabled()) { - logger.info("Unexport service: " + exporter.getInvoker().getUrl()); - } - exporter.unexport(); - } catch (Throwable t) { - logger.warn(t.getMessage(), t); - } - } - } - + super.destroy(); } } diff --git a/dubbo-rpc/pom.xml b/dubbo-rpc/pom.xml index ef384d536d..091f8e9a7a 100644 --- a/dubbo-rpc/pom.xml +++ b/dubbo-rpc/pom.xml @@ -37,4 +37,13 @@ dubbo-rpc-grpc dubbo-rpc-triple + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-serialization/pom.xml b/dubbo-serialization/pom.xml index 2d23812292..6f759958f4 100644 --- a/dubbo-serialization/pom.xml +++ b/dubbo-serialization/pom.xml @@ -34,4 +34,13 @@ dubbo-serialization-hessian2 dubbo-serialization-jdk + + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + + diff --git a/dubbo-spring-boot/dubbo-spring-boot-compatible/autoconfigure/src/main/java/org/apache/dubbo/spring/boot/context/event/AwaitingNonWebApplicationListener.java b/dubbo-spring-boot/dubbo-spring-boot-compatible/autoconfigure/src/main/java/org/apache/dubbo/spring/boot/context/event/AwaitingNonWebApplicationListener.java index 8dd42662a2..deab7fb054 100644 --- a/dubbo-spring-boot/dubbo-spring-boot-compatible/autoconfigure/src/main/java/org/apache/dubbo/spring/boot/context/event/AwaitingNonWebApplicationListener.java +++ b/dubbo-spring-boot/dubbo-spring-boot-compatible/autoconfigure/src/main/java/org/apache/dubbo/spring/boot/context/event/AwaitingNonWebApplicationListener.java @@ -57,26 +57,26 @@ public class AwaitingNonWebApplicationListener implements SmartApplicationListen private static final Class[] SUPPORTED_APPLICATION_EVENTS = of(ApplicationReadyEvent.class, ContextClosedEvent.class); - private static final AtomicBoolean awaited = new AtomicBoolean(false); + private final AtomicBoolean awaited = new AtomicBoolean(false); private static final Integer UNDEFINED_ID = Integer.valueOf(-1); /** * Target the application id */ - private static final AtomicInteger applicationContextId = new AtomicInteger(UNDEFINED_ID); + private final AtomicInteger applicationContextId = new AtomicInteger(UNDEFINED_ID); - private static final Lock lock = new ReentrantLock(); + private final Lock lock = new ReentrantLock(); - private static final Condition condition = lock.newCondition(); + private final Condition condition = lock.newCondition(); - private static final ExecutorService executorService = newSingleThreadExecutor(); + private final ExecutorService executorService = newSingleThreadExecutor(); private static T[] of(T... values) { return values; } - static AtomicBoolean getAwaited() { + AtomicBoolean getAwaited() { return awaited; } diff --git a/dubbo-spring-boot/pom.xml b/dubbo-spring-boot/pom.xml index f3ea332510..2f52e8a22f 100644 --- a/dubbo-spring-boot/pom.xml +++ b/dubbo-spring-boot/pom.xml @@ -78,6 +78,12 @@ test + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + test + diff --git a/dubbo-test/dubbo-test-check/pom.xml b/dubbo-test/dubbo-test-check/pom.xml new file mode 100644 index 0000000000..0af14d7f4c --- /dev/null +++ b/dubbo-test/dubbo-test-check/pom.xml @@ -0,0 +1,51 @@ + + + + + dubbo-test + org.apache.dubbo + ${revision} + ../pom.xml + + 4.0.0 + + dubbo-test-check + + + 8 + 8 + + + + + org.apache.dubbo + dubbo-common + + + org.junit.platform + junit-platform-launcher + 1.6.2 + + + org.apache.commons + commons-lang3 + + + diff --git a/dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/DubboTestChecker.java b/dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/DubboTestChecker.java new file mode 100644 index 0000000000..e8233602f0 --- /dev/null +++ b/dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/DubboTestChecker.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.test.check; + +import org.apache.commons.lang3.StringUtils; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.rpc.model.FrameworkModel; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.TestSource; +import org.junit.platform.engine.support.descriptor.ClassSource; +import org.junit.platform.engine.support.descriptor.MethodSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * A test listener to check unclosed threads of test. + * + * 
+ * Usages:
+ *  # enable thread checking
+ *  mvn test -DcheckThreads=true
+ *
+ *  # change thread dump wait time (ms)
+ *  mvn test -DcheckThreads=true -DthreadDumpWaitTime=5000
+ *
+ *  # print test reports of all sub modules to single file
+ *  mvn test -DcheckThreads=true -DthreadDumpWaitTime=5000 -DreportFile=/path/test-check-report.txt
+ *
+ */ +public class DubboTestChecker implements TestExecutionListener { + + private static final String CONFIG_CHECK_MODE = "checkMode"; + private static final String CONFIG_CHECK_THREADS = "checkThreads"; + private static final String CONFIG_THREAD_DUMP_WAIT_TIME = "threadDumpWaitTime"; + private static final String CONFIG_FORCE_DESTROY = "forceDestroy"; + private static final String CONFIG_REPORT_FILE = "reportFile"; + private static final String MODE_CLASS = "class"; + private static final String MODE_METHOD = "method"; + + private static final Logger logger = LoggerFactory.getLogger(DubboTestChecker.class); + + /** + * check mode: + * class - check after class execution finished + * method - check after method execution finished + */ + private String checkMode; + /** + * whether check unclosed threads + */ + private boolean checkThreads; + /** + * sleep time before dump threads + */ + private long threadDumpWaitTimeMs; + /** + * whether force destroy dubbo engine, default value is true. + */ + private boolean forceDestroyDubboAfterClass; + + /** + * Check report file + */ + private File reportFile; + + /** + * thread -> stacktrace + */ + private Map unclosedThreadMap = new ConcurrentHashMap<>(); + // test class name -> thread list + private Map> unclosedThreadsOfTestMap = new ConcurrentHashMap<>(); + private String identifier; + private PrintWriter reportWriter; + private String projectDir; + private FileOutputStream reportFileOut; + + @Override + public void testPlanExecutionStarted(TestPlan testPlan) { + try { + init(System.getProperties()); + } catch (IOException e) { + throw new IllegalStateException("Test checker init failed", e); + } + } + + public void init(Properties properties) throws IOException { + if (properties == null) { + properties = new Properties(); + } + // log prefix + identifier = "[" + this.getClass().getSimpleName() + "] "; + + // checkMode: class/method + checkMode = StringUtils.lowerCase(properties.getProperty(CONFIG_CHECK_MODE, MODE_CLASS)); + // checkThreads: true/false + checkThreads = Boolean.parseBoolean(properties.getProperty(CONFIG_CHECK_THREADS, "false")); + // threadDumpWaitTime + threadDumpWaitTimeMs = Long.parseLong(properties.getProperty(CONFIG_THREAD_DUMP_WAIT_TIME, "5000")); + // force destroy dubbo + forceDestroyDubboAfterClass = Boolean.parseBoolean(properties.getProperty(CONFIG_FORCE_DESTROY, "true")); + + // project dir + projectDir = new File(".").getCanonicalPath(); + + // report file + String reportFileCanonicalPath = ""; + String defaultReportDir = "target/"; + String defaultReportFileName = "test-check-report.txt"; + if (checkThreads) { + String reportFilePath = properties.getProperty(CONFIG_REPORT_FILE, defaultReportDir + defaultReportFileName); + this.reportFile = new File(reportFilePath); + if (reportFile.isDirectory()) { + reportFile.mkdirs(); + reportFile = new File(reportFile, defaultReportFileName); + } + reportFileOut = new FileOutputStream(this.reportFile); + reportWriter = new PrintWriter(reportFileOut); + reportFileCanonicalPath = reportFile.getCanonicalPath(); + } + + log("Project dir: " + projectDir); + log(String.format("Dubbo test checker configs: checkMode=%s, checkThreads=%s, threadDumpWaitTimeMs=%s, forceDestroy=%s, reportFile=%s", + checkMode, checkThreads, threadDumpWaitTimeMs, forceDestroyDubboAfterClass, reportFileCanonicalPath)); + flushReportFile(); + } + + @Override + public void testPlanExecutionFinished(TestPlan testPlan) { + + // print all unclosed threads + if (checkThreads) { + printThreadCheckingSummaryReport(); + } else { + log("Thread checking is disabled, use -DcheckThreads=true to check unclosed threads."); + } + if (reportWriter != null) { + reportWriter.close(); + } + } + + private void printThreadCheckingSummaryReport() { + log("===== Thread Checking Summary Report ======"); + log("Project dir: " + projectDir); + log("Total found " + unclosedThreadMap.size() + " unclosed threads in " + unclosedThreadsOfTestMap.size() + " tests."); + log(""); + unclosedThreadsOfTestMap.forEach((testClassName, threads) -> { + printUnclosedThreads(threads, testClassName); + }); + flushReportFile(); + } + + private void flushReportFile() { + try { + reportWriter.flush(); + if (reportFileOut != null) { + reportFileOut.getFD().sync(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public Map getUnclosedThreadMap() { + return unclosedThreadMap; + } + + @Override + public void executionStarted(TestIdentifier testIdentifier) { +// TestSource testSource = testIdentifier.getSource().orElse(null); +// if (testSource instanceof ClassSource) { +// ClassSource source = (ClassSource) testSource; +// log("Run test class: " + source.getClassName()); +// } else if (testSource instanceof MethodSource) { +// MethodSource source = (MethodSource) testSource; +// log("Run test method: " + source.getClassName() + "#" + source.getMethodName()); +// } + } + + @Override + public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult testExecutionResult) { + + TestSource testSource = testIdentifier.getSource().orElse(null); + String testClassName; + if (testSource instanceof MethodSource) { + if (!StringUtils.contains(checkMode, MODE_METHOD)) { + return; + } + MethodSource methodSource = (MethodSource) testSource; + testClassName = methodSource.getClassName(); + //log("Finish test method: " + methodSource.getClassName() + "#" + methodSource.getMethodName()); + } else if (testSource instanceof ClassSource) { + if (forceDestroyDubboAfterClass) { + // make sure destroy dubbo engine + FrameworkModel.destroyAll(); + } + + if (!StringUtils.contains(checkMode, MODE_CLASS)) { + return; + } + + ClassSource source = (ClassSource) testSource; + testClassName = source.getClassName(); + //log("Finish test class: " + source.getClassName()); + } else { + return; + } + + if (checkThreads) { + checkUnclosedThreads(testClassName, threadDumpWaitTimeMs); + } + } + + public Map checkUnclosedThreads(String testClassName, long waitMs) { + // wait for shutdown + log("Wait " + waitMs + "ms to check threads of " + testClassName + " ..."); + try { + Thread.sleep(waitMs); + } catch (InterruptedException e) { + } + + Map threadStacks = Thread.getAllStackTraces(); + List unclosedThreads = threadStacks.keySet().stream() + .filter(thread -> !StringUtils.startsWithAny(thread.getName(), + "Reference Handler", "Finalizer", "Signal Dispatcher", "Attach Listener", "process reaper", "main" // jvm + , "surefire-forkedjvm-" // surefire plugin + )) + .filter(thread -> !unclosedThreadMap.containsKey(thread)) + .collect(Collectors.toList()); + unclosedThreads.sort(Comparator.comparing(Thread::getName)); + if (unclosedThreads.size() > 0) { + for (Thread thread : unclosedThreads) { + unclosedThreadMap.put(thread, threadStacks.get(thread)); + } + unclosedThreadsOfTestMap.put(testClassName, unclosedThreads); + printUnclosedThreads(unclosedThreads, testClassName); + } + + // return new unclosed thread map + Map unclosedThreadMap = new LinkedHashMap<>(); + for (Thread thread : unclosedThreads) { + unclosedThreadMap.put(thread, threadStacks.get(thread)); + } + return unclosedThreadMap; + } + + private void printUnclosedThreads(List threads, String testClassName) { + if (threads.size() > 0) { + log("Found " + threads.size() + " unclosed threads in test: " + testClassName); + for (Thread thread : threads) { + StackTraceElement[] stackTrace = unclosedThreadMap.get(thread); + log(getFullStacktrace(thread, stackTrace)); + } + flushReportFile(); + } + } + + private void log(String msg) { + // logger.info(identifier + msg); + String s = identifier + msg; + System.out.println(s); + if (reportWriter != null) { + reportWriter.println(s); + } + } + + public static String getFullStacktrace(Thread thread, StackTraceElement[] stackTrace) { + StringBuilder sb = new StringBuilder("Thread: \"" + thread.getName() + "\"" + " Id=" + + thread.getId()); + sb.append(" ").append(thread.getState()); + sb.append('\n'); + if (stackTrace == null) { + stackTrace = thread.getStackTrace(); + } + for (StackTraceElement ste : stackTrace) { + sb.append(" at ").append(ste.toString()); + sb.append('\n'); + } + return sb.toString(); + } + +} diff --git a/dubbo-test/dubbo-test-check/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/dubbo-test/dubbo-test-check/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..0b6e457b5b --- /dev/null +++ b/dubbo-test/dubbo-test-check/src/main/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +org.apache.dubbo.test.check.DubboTestChecker diff --git a/dubbo-test/dubbo-test-common/pom.xml b/dubbo-test/dubbo-test-common/pom.xml index 0cbfa346a9..69b53ff853 100644 --- a/dubbo-test/dubbo-test-common/pom.xml +++ b/dubbo-test/dubbo-test-common/pom.xml @@ -68,5 +68,11 @@ + + + org.apache.dubbo + dubbo-test-check + ${project.parent.version} + diff --git a/dubbo-test/pom.xml b/dubbo-test/pom.xml index bc6fee2b6c..ee20080694 100644 --- a/dubbo-test/pom.xml +++ b/dubbo-test/pom.xml @@ -28,6 +28,7 @@ pom + dubbo-test-check dubbo-test-common dubbo-test-spring dubbo-test-spring3.2