[3.0] Manage global resources and executor services, fix zk client connections (#9033)

* shutdown scheduledExecutors and executorServiceRing

* fix NPE of configuration

* Fix notify loop after executor service is shutdown

* improve registry center in tests

* polish unregister shutdown hook log

* Fix systemConfiguration NPE

* enable surefire reuseForks for fast testing

* Fix checking DubboShutdownHook is alive

* fix SPI/Bean instantiation constructor matching

* shutdown share executor

* release zk client

* Improve application destroy processing

* Add GlobalResourcesRepository to manager static resources, including ExecutorService and HashedWheelTimer

* revert zk client address mapping

* Fix zk client early close problem

* improve executor service

* improve both start by module and by application

* release sharedScheduledExecutor and fix AccessLogFilter

* Fix tests

* checking zk registry if destroyed

* fix testSystemPropertyOverrideReferenceConfig config error

* improve KeepRunningOnSpringClosedTest

* SPI extension/scope bean prefers parameterized constructor instead of default constructor

* Support compatible usage of ZookeeperRegistryFactory

* Improve the application/module destroy process, the deployer destruction is divided into pre-destroy and post-destroy

* Destroy NacosDynamicConfiguration

* Destroy MetadataReportFactory

* Recreate adaptive classes

* Destroy global resources on dubbo shutdown

* Fix zk thread leaks when create client failed

* Improve protocol destroying

* Improve thread name of protocol port

* Manage global EventLoopGroup, HashedWheelTimer, etc.

* Protected metadata info in memory

* Support checking unclosed threads when finish test class

* Improve application check started

* add test log4j.properties

* add pom.xml of dubbo-test-check

* fix CodecSupport.checkSerialization error

* Fix ReferenceCountExchangeClientTest log msg checking

* Fix ServiceInstanceMetadataUtilsTest and MigrationInvokerTest

* Fix protocol destroy problem

* rename test check jvm args

* cancel asyncMetadataFuture and unregisterServiceInstance in pre-destroy application

* Ignore refresh metadata if application is stopping

* Remove unused destroyDynamicConfigurations method

* Change ZookeeperTransporter to application scope

* Add testMultiProviderApplicationsStopOneByOne

* Destroy ZookeeperTransporter in RemotingScopeModelInitializer

* support specify test check report file

dubbo-cluster/pom.xml

@@ -60,5 +60,11 @@
             <version>${project.parent.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
-</project>
+</project>

dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/loadbalance/ShortestResponseLoadBalance.java

@@ -17,7 +17,6 @@
 package org.apache.dubbo.rpc.cluster.loadbalance;
 
 import org.apache.dubbo.common.URL;
-import org.apache.dubbo.common.utils.NamedThreadFactory;
 import org.apache.dubbo.rpc.Invocation;
 import org.apache.dubbo.rpc.Invoker;
 import org.apache.dubbo.rpc.RpcStatus;
@@ -29,7 +28,6 @@ import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -46,7 +44,7 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements
 
     public static final String NAME = "shortestresponse";
 
-    private int SLIDE_PERIOD = 30_000;
+    private int slidePeriod = 30_000;
 
     private ConcurrentMap<RpcStatus, SlideWindowData> methodMap = new ConcurrentHashMap<>();
 
@@ -54,13 +52,15 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements
 
     private volatile long lastUpdateTime = System.currentTimeMillis();
 
+    private ExecutorService executorService;
+
     @Override
     public void setApplicationModel(ApplicationModel applicationModel) {
-        SLIDE_PERIOD = applicationModel.getModelEnvironment().getConfiguration().getInt(Constants.SHORTEST_RESPONSE_SLIDE_PERIOD, 30_000);
+        slidePeriod = applicationModel.getModelEnvironment().getConfiguration().getInt(Constants.SHORTEST_RESPONSE_SLIDE_PERIOD, 30_000);
+        executorService = applicationModel.getApplicationExecutorRepository().getSharedExecutor();
     }
 
     protected static class SlideWindowData {
-        private final static ExecutorService EXECUTOR_SERVICE = Executors.newSingleThreadExecutor((new NamedThreadFactory("Dubbo-slidePeriod-reset")));
 
         private long succeededOffset;
         private long succeededElapsedOffset;
@@ -138,10 +138,10 @@ public class ShortestResponseLoadBalance extends AbstractLoadBalance implements
             }
         }
 
-        if (System.currentTimeMillis() - lastUpdateTime > SLIDE_PERIOD
+        if (System.currentTimeMillis() - lastUpdateTime > slidePeriod
             && onResetSlideWindow.compareAndSet(false, true)) {
             //reset slideWindowData in async way
-            SlideWindowData.EXECUTOR_SERVICE.execute(() -> {
+            executorService.execute(() -> {
                 methodMap.values().forEach(SlideWindowData::reset);
                 lastUpdateTime = System.currentTimeMillis();
                 onResetSlideWindow.set(false);

dubbo-cluster/src/main/java/org/apache/dubbo/rpc/cluster/support/ForkingClusterInvoker.java

@@ -30,7 +30,6 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -53,11 +52,11 @@ public class ForkingClusterInvoker<T> extends AbstractClusterInvoker<T> {
      * Use {@link NamedInternalThreadFactory} to produce {@link org.apache.dubbo.common.threadlocal.InternalThread}
      * which with the use of {@link org.apache.dubbo.common.threadlocal.InternalThreadLocal} in {@link RpcContext}.
      */
-    private final ExecutorService executor = Executors.newCachedThreadPool(
-        new NamedInternalThreadFactory("forking-cluster-timer", true));
+    private final ExecutorService executor;
 
     public ForkingClusterInvoker(Directory<T> directory) {
         super(directory);
+        executor = directory.getUrl().getOrDefaultApplicationModel().getApplicationExecutorRepository().getSharedExecutor();
     }
 
     @Override

dubbo-common/pom.xml

@@ -94,7 +94,6 @@
             <scope>test</scope>
         </dependency>
 
-
     </dependencies>
 
 </project>

dubbo-common/src/main/java/org/apache/dubbo/common/URL.java

@@ -1577,6 +1577,11 @@ class URL implements Serializable {
         return attributes.get(key);
     }
 
+    public Object getAttribute(String key, Object defaultValue) {
+        Object val = attributes.get(key);
+        return val != null ? val : defaultValue;
+    }
+
     public URL putAttribute(String key, Object obj) {
         attributes.put(key, obj);
         return this;

dubbo-common/src/main/java/org/apache/dubbo/common/beans/support/InstantiationStrategy.java

@@ -33,7 +33,6 @@ import java.util.List;
  */
 public class InstantiationStrategy {
 
-    private boolean supportConstructorWithArguments;
     private ScopeModelAccessor scopeModelAccessor;
 
     public InstantiationStrategy() {
@@ -42,19 +41,17 @@ public class InstantiationStrategy {
 
     public InstantiationStrategy(ScopeModelAccessor scopeModelAccessor) {
         this.scopeModelAccessor = scopeModelAccessor;
-        this.supportConstructorWithArguments = (this.scopeModelAccessor != null);
     }
 
     public <T> T instantiate(Class<T> type) throws ReflectiveOperationException {
 
-        // 1. try default constructor
+        // should not use default constructor directly, maybe also has another constructor matched scope model arguments
+        // 1. try get default constructor
+        Constructor<T> defaultConstructor = null;
         try {
-            return type.getConstructor().newInstance();
+            defaultConstructor = type.getConstructor();
         } catch (NoSuchMethodException e) {
             // ignore no default constructor
-            if (!supportConstructorWithArguments) {
-                throw new IllegalArgumentException("Default constructor was not found for type: " + type.getName());
-            }
         }
 
         // 2. use matched constructor if found
@@ -65,22 +62,35 @@ public class InstantiationStrategy {
                 matchedConstructors.add(constructor);
             }
         }
+        // remove default constructor from matchedConstructors
+        if (defaultConstructor != null) {
+            matchedConstructors.remove(defaultConstructor);
+        }
+
+        // match order:
+        // 1. the only matched constructor with parameters
+        // 2. default constructor if absent
+
+        Constructor targetConstructor;
         if (matchedConstructors.size() > 1) {
             throw new IllegalArgumentException("Expect only one but found " +
                 matchedConstructors.size() + " matched constructors for type: " + type.getName() +
                 ", matched constructors: " + matchedConstructors);
-        } else if (matchedConstructors.size() == 0) {
+        } else if (matchedConstructors.size() == 1) {
+            targetConstructor = matchedConstructors.get(0);
+        } else if (defaultConstructor != null) {
+            targetConstructor = defaultConstructor;
+        } else {
             throw new IllegalArgumentException("None matched constructor was found for type: " + type.getName());
         }
 
         // create instance with arguments
-        Constructor constructor = matchedConstructors.get(0);
-        Class[] parameterTypes = constructor.getParameterTypes();
+        Class[] parameterTypes = targetConstructor.getParameterTypes();
         Object[] args = new Object[parameterTypes.length];
         for (int i = 0; i < parameterTypes.length; i++) {
             args[i] = getArgumentValueForType(parameterTypes[i]);
         }
-        return (T) constructor.newInstance(args);
+        return (T) targetConstructor.newInstance(args);
     }
 
     private boolean isMatched(Constructor<?> constructor) {

dubbo-common/src/main/java/org/apache/dubbo/common/concurrent/CallableSafeInitializer.java

@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.concurrent;
+
+import org.apache.dubbo.common.resource.Disposable;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+
+/**
+ * <p>
+ * A safe and lazy and removable initializer implementation that wraps a
+ * {@code Callable} object.
+ * </p>
+ * @see org.apache.commons.lang3.concurrent.AtomicSafeInitializer
+ */
+public class CallableSafeInitializer<T> {
+    /** A guard which ensures that initialize() is called only once. */
+    private final AtomicReference<CallableSafeInitializer<T>> factory =
+            new AtomicReference<>();
+
+    /** Holds the reference to the managed object. */
+    private final AtomicReference<T> reference = new AtomicReference<>();
+
+    /** The Callable to be executed. */
+    private final Callable<T> callable;
+
+    public CallableSafeInitializer(Callable<T> callable) {
+        this.callable = callable;
+    }
+
+    /**
+     * Get (and initialize, if not initialized yet) the required object
+     *
+     * @return lazily initialized object
+     * exception
+     */
+    //@Override
+    public final T get() {
+        T result;
+
+        while ((result = reference.get()) == null) {
+            if (factory.compareAndSet(null, this)) {
+                reference.set(initialize());
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * Creates and initializes the object managed by this
+     * {@code AtomicInitializer}. This method is called by {@link #get()} when
+     * the managed object is not available yet. An implementation can focus on
+     * the creation of the object. No synchronization is needed, as this is
+     * already handled by {@code get()}. This method is guaranteed to be called
+     * only once.
+     *
+     * @return the managed data object
+     */
+    protected T initialize() {
+        try {
+            return callable.call();
+        } catch (Exception e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
+    }
+
+    public T remove() {
+        return this.remove(null);
+    }
+
+    public T remove(Consumer<? super T> action) {
+        // release
+        T value = reference.getAndSet(null);
+        if (value != null && action != null) {
+            if (action != null) {
+                action.accept(value);
+            } else if (value instanceof Disposable) {
+                ((Disposable) value).destroy();
+            }
+        }
+        factory.set(null);
+        return value;
+    }
+}

dubbo-common/src/main/java/org/apache/dubbo/common/config/Environment.java

@@ -246,7 +246,14 @@ public class Environment extends LifecycleAdapter implements ApplicationExt {
         globalConfiguration = null;
         globalConfigurationMaps = null;
         defaultDynamicGlobalConfiguration = null;
-        defaultDynamicConfiguration = null;
+        if (defaultDynamicConfiguration != null) {
+            try {
+                defaultDynamicConfiguration.close();
+            } catch (Exception e) {
+                logger.warn("close dynamic configuration failed: " + e.getMessage(), e);
+            }
+            defaultDynamicConfiguration = null;
+        }
     }
 
     /**

dubbo-common/src/main/java/org/apache/dubbo/common/config/ModuleEnvironment.java

@@ -119,6 +119,7 @@ public class ModuleEnvironment extends Environment implements ModuleExt {
 
     @Override
     public void destroy() throws IllegalStateException {
+        super.destroy();
         this.orderedPropertiesConfiguration = null;
         this.globalConfiguration = null;
         this.dynamicGlobalConfiguration = null;

dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/nop/NopDynamicConfiguration.java

@@ -20,10 +20,10 @@ import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
 import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
 
-import static java.util.Collections.emptySortedSet;
-
 import java.util.SortedSet;
 
+import static java.util.Collections.emptySortedSet;
+
 /**
  * The default extension of {@link DynamicConfiguration}. If user does not specify a config center, or specifies one
  * that is not a valid extension, it will default to this one.
@@ -71,4 +71,9 @@ public class NopDynamicConfiguration implements DynamicConfiguration {
     public SortedSet<String> getConfigKeys(String group) {
         return emptySortedSet();
     }
+
+    @Override
+    public void close() throws Exception {
+        // no-op
+    }
 }

dubbo-common/src/main/java/org/apache/dubbo/common/config/configcenter/wrapper/CompositeDynamicConfiguration.java

@@ -18,6 +18,8 @@ package org.apache.dubbo.common.config.configcenter.wrapper;
 
 import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
 import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -32,6 +34,8 @@ public class CompositeDynamicConfiguration implements DynamicConfiguration {
 
     public static final String NAME = "COMPOSITE";
 
+    private static final Logger logger = LoggerFactory.getLogger(CompositeDynamicConfiguration.class);
+
     private Set<DynamicConfiguration> configurations = new HashSet<>();
 
     public void addConfiguration(DynamicConfiguration configuration) {
@@ -87,6 +91,18 @@ public class CompositeDynamicConfiguration implements DynamicConfiguration {
         return (SortedSet<String>) iterateConfigOperation(configuration -> configuration.getConfigKeys(group));
     }
 
+    @Override
+    public void close() throws Exception {
+        for (DynamicConfiguration configuration : configurations) {
+            try {
+                configuration.close();
+            } catch (Exception e) {
+                logger.warn("close dynamic configuration failed: " + e.getMessage(), e);
+            }
+        }
+        configurations.clear();
+    }
+
     private void iterateListenerOperation(Consumer<DynamicConfiguration> consumer) {
         for (DynamicConfiguration configuration : configurations) {
             consumer.accept(configuration);

dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ApplicationDeployer.java

@@ -19,7 +19,7 @@ package org.apache.dubbo.common.deploy;
 import org.apache.dubbo.common.config.ReferenceCache;
 import org.apache.dubbo.rpc.model.ApplicationModel;
 
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 
 /**
  * initialize and start application instance
@@ -33,17 +33,29 @@ public interface ApplicationDeployer extends Deployer<ApplicationModel> {
 
     /**
      * Starts the component.
+     * @return
      */
-    CompletableFuture start() throws IllegalStateException;
+    Future start() throws IllegalStateException;
 
     /**
      * Stops the component.
      */
     void stop() throws IllegalStateException;
 
+    /**
+     * Register application instance and start internal services
+     */
     void prepareApplicationInstance();
 
-    void destroy();
+    /**
+     * Pre-processing before destroy model
+     */
+    void preDestroy();
+
+    /**
+     * Post-processing after destroy model
+     */
+    void postDestroy();
 
     /**
      * Indicates that the Application is initialized or not.
@@ -61,6 +73,6 @@ public interface ApplicationDeployer extends Deployer<ApplicationModel> {
 
     void checkStarting();
 
-    void checkStarted(CompletableFuture checkerStartFuture);
+    void checkStarted();
 
 }

dubbo-common/src/main/java/org/apache/dubbo/common/deploy/DeployState.java

@@ -20,5 +20,38 @@ package org.apache.dubbo.common.deploy;
  * Deploy state enum
  */
 public enum DeployState {
-    PENDING, STARTING, STARTED, STOPPING, STOPPED, FAILED
+    /**
+     * Unknown state
+     */
+    UNKNOWN,
+
+    /**
+     * Pending, wait for start
+     */
+    PENDING,
+
+    /**
+     * Starting
+     */
+    STARTING,
+
+    /**
+     * Started
+     */
+    STARTED,
+
+    /**
+     * Stopping
+     */
+    STOPPING,
+
+    /**
+     * Stopped
+     */
+    STOPPED,
+
+    /**
+     * Failed
+     */
+    FAILED
 }

dubbo-common/src/main/java/org/apache/dubbo/common/deploy/Deployer.java

@@ -18,7 +18,7 @@ package org.apache.dubbo.common.deploy;
 
 import org.apache.dubbo.rpc.model.ScopeModel;
 
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 
 /**
  */
@@ -31,8 +31,9 @@ public interface Deployer<E extends ScopeModel> {
 
     /**
      * Starts the component.
+     * @return
      */
-    CompletableFuture start() throws IllegalStateException;
+    Future start() throws IllegalStateException;
 
     /**
      * Stops the component.

dubbo-common/src/main/java/org/apache/dubbo/common/deploy/ModuleDeployer.java

@@ -20,7 +20,7 @@ import org.apache.dubbo.common.config.ReferenceCache;
 import org.apache.dubbo.config.ServiceConfigBase;
 import org.apache.dubbo.rpc.model.ModuleModel;
 
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 
 /**
  * Export/refer services of module
@@ -29,11 +29,13 @@ public interface ModuleDeployer extends Deployer<ModuleModel> {
 
     void initialize() throws IllegalStateException;
 
-    CompletableFuture start() throws IllegalStateException;
+    Future start() throws IllegalStateException;
 
     void stop() throws IllegalStateException;
 
-    void destroy() throws IllegalStateException;
+    void preDestroy() throws IllegalStateException;
+
+    void postDestroy() throws IllegalStateException;
 
     boolean isInitialized();
 

dubbo-common/src/main/java/org/apache/dubbo/common/extension/AdaptiveClassCodeGenerator.java

@@ -26,6 +26,7 @@ import org.apache.dubbo.rpc.model.ScopeModelUtil;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -90,6 +91,14 @@ public class AdaptiveClassCodeGenerator {
      * generate and return class code
      */
     public String generate() {
+        return this.generate(false);
+    }
+
+    /**
+     * generate and return class code
+     * @param sort - whether sort methods
+     */
+    public String generate(boolean sort) {
         // no need to generate adaptive class since there's no adaptive method found.
         if (!hasAdaptiveMethod()) {
             throw new IllegalStateException("No adaptive method exist on extension " + type.getName() + ", refuse to create the adaptive class!");
@@ -101,6 +110,9 @@ public class AdaptiveClassCodeGenerator {
         code.append(generateClassDeclaration());
 
         Method[] methods = type.getMethods();
+        if (sort) {
+            Arrays.sort(methods, Comparator.comparing(Method::toString));
+        }
         for (Method method : methods) {
             code.append(generateMethod(method));
         }

dubbo-common/src/main/java/org/apache/dubbo/common/resource/Disposable.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.resource;
+
+/**
+ * An interface for destroying resources
+ */
+public interface Disposable {
+
+    void destroy();
+
+}

dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourceInitializer.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.resource;
+
+import org.apache.dubbo.common.concurrent.CallableSafeInitializer;
+
+import java.util.concurrent.Callable;
+import java.util.function.Consumer;
+
+/**
+ * A initializer to release resource automatically on dubbo shutdown
+ */
+public class GlobalResourceInitializer<T> extends CallableSafeInitializer<T> {
+
+    /**
+     * The Dispose action to be executed on shutdown.
+     */
+    private Consumer<T> disposeAction;
+
+    private Disposable disposable;
+
+    public GlobalResourceInitializer(Callable<T> initializer) {
+        super(initializer);
+    }
+
+    public GlobalResourceInitializer(Callable initializer, Consumer<T> disposeAction) {
+        super(initializer);
+        this.disposeAction = disposeAction;
+    }
+
+    public GlobalResourceInitializer(Callable<T> callable, Disposable disposable) {
+        super(callable);
+        this.disposable = disposable;
+    }
+
+    @Override
+    protected T initialize() {
+        T value = super.initialize();
+        // register disposable to release automatically
+        if (this.disposable != null) {
+            GlobalResourcesRepository.getInstance().registerDisposable(this.disposable);
+        } else {
+            GlobalResourcesRepository.getInstance().registerDisposable(() -> this.remove(disposeAction));
+        }
+        return value;
+    }
+
+    public interface DestroyHandler<T> {
+        void dispose(GlobalResourceInitializer<T> initializer);
+    }
+}

dubbo-common/src/main/java/org/apache/dubbo/common/resource/GlobalResourcesRepository.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.common.resource;
+
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.utils.NamedThreadFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Global resources repository between all framework models.
+ * It will be destroyed only after all framework model is destroyed.
+ */
+public class GlobalResourcesRepository {
+
+    private static final Logger logger = LoggerFactory.getLogger(GlobalResourcesRepository.class);
+
+    private volatile static GlobalResourcesRepository instance;
+    private volatile ExecutorService executorService;
+    private final List<Disposable> oneoffDisposables = Collections.synchronizedList(new ArrayList<>());
+    private final List<Disposable> reusedDisposables = Collections.synchronizedList(new ArrayList<>());
+
+    private GlobalResourcesRepository() {
+    }
+
+    public static GlobalResourcesRepository getInstance() {
+        if (instance == null) {
+            synchronized (GlobalResourcesRepository.class) {
+                if (instance == null) {
+                    instance = new GlobalResourcesRepository();
+                }
+            }
+        }
+        return instance;
+    }
+
+    public static ExecutorService getGlobalExecutorService() {
+        return getInstance().getExecutorService();
+    }
+
+    public ExecutorService getExecutorService() {
+        if (executorService == null || executorService.isShutdown()) {
+            synchronized (this) {
+                if (executorService == null || executorService.isShutdown()) {
+                    executorService = Executors.newCachedThreadPool(new NamedThreadFactory("Dubbo-global-shared-handler", true));
+                }
+            }
+        }
+        return executorService;
+    }
+
+    public void destroy() {
+        if (logger.isInfoEnabled()) {
+            logger.info("Destroying global resources ...");
+        }
+        if (executorService != null) {
+            executorService.shutdownNow();
+            executorService = null;
+        }
+
+        // notify disposables
+        // NOTE: don't clear reused disposables for reuse purpose
+        for (Disposable disposable : new ArrayList<>(reusedDisposables)) {
+            try {
+                disposable.destroy();
+            } catch (Exception e) {
+                logger.warn("destroy resources failed: " + e.getMessage(), e);
+            }
+        }
+
+        for (Disposable disposable : new ArrayList<>(oneoffDisposables)) {
+            try {
+                disposable.destroy();
+            } catch (Exception e) {
+                logger.warn("destroy resources failed: " + e.getMessage(), e);
+            }
+        }
+        // clear one-off disposable
+        oneoffDisposables.clear();
+
+        if (logger.isInfoEnabled()) {
+            logger.info("Dubbo is completely destroyed");
+        }
+    }
+
+    /**
+     * Register a one-off disposable, the disposable is removed automatically on first shutdown.
+     * @param disposable
+     */
+    public void registerDisposable(Disposable disposable) {
+        this.registerDisposable(disposable, false);
+    }
+
+    /**
+     * Register a disposable
+     * @param disposable
+     * @param reused true - the disposable is keep and reused. false - the disposable is removed automatically on first shutdown
+     */
+    public void registerDisposable(Disposable disposable, boolean reused) {
+        List<Disposable> disposables = reused ? reusedDisposables : oneoffDisposables;
+        if (!disposables.contains(disposable)) {
+            disposables.add(disposable);
+        }
+    }
+
+    public void removeDisposable(Disposable disposable) {
+        this.reusedDisposables.remove(disposable);
+        this.oneoffDisposables.remove(disposable);
+    }
+
+}

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/DefaultExecutorRepository.java

@@ -32,6 +32,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.apache.dubbo.rpc.model.ModuleModel;
 import org.apache.dubbo.rpc.model.ScopeModelAware;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -48,6 +49,7 @@ import static org.apache.dubbo.common.constants.CommonConstants.DEFAULT_REFER_TH
 import static org.apache.dubbo.common.constants.CommonConstants.EXECUTOR_SERVICE_COMPONENT_KEY;
 import static org.apache.dubbo.common.constants.CommonConstants.SIDE_KEY;
 import static org.apache.dubbo.common.constants.CommonConstants.THREADS_KEY;
+import static org.apache.dubbo.common.constants.CommonConstants.THREAD_NAME_KEY;
 
 /**
  * Consider implementing {@code Licycle} to enable executors shutdown when the process stops.
@@ -57,17 +59,18 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
 
     private int DEFAULT_SCHEDULER_SIZE = Runtime.getRuntime().availableProcessors();
 
-    private final ExecutorService SHARED_EXECUTOR = Executors.newCachedThreadPool(new NamedThreadFactory("DubboSharedHandler", true));
+    private final ExecutorService sharedExecutor;
+    private final ScheduledExecutorService sharedScheduledExecutor;
 
     private Ring<ScheduledExecutorService> scheduledExecutors = new Ring<>();
 
-    private volatile ExecutorService serviceExportExecutor;
+    private volatile ScheduledExecutorService serviceExportExecutor;
 
     private volatile ExecutorService serviceReferExecutor;
 
     private ScheduledExecutorService reconnectScheduledExecutor;
 
-    public  Ring<ScheduledExecutorService> registryNotificationExecutorRing = new Ring<>();
+    public Ring<ScheduledExecutorService> registryNotificationExecutorRing = new Ring<>();
 
     private Ring<ScheduledExecutorService> serviceDiscoveryAddressNotificationExecutorRing = new Ring<>();
 
@@ -77,25 +80,28 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
 
     private ExecutorService poolRouterExecutor;
 
-    private static Ring<ExecutorService> executorServiceRing = new Ring<ExecutorService>();
+    private Ring<ExecutorService> executorServiceRing = new Ring<ExecutorService>();
 
-    private static final Object LOCK = new Object();
+    private final Object LOCK = new Object();
     private ExtensionAccessor extensionAccessor;
 
     private ApplicationModel applicationModel;
 
     public DefaultExecutorRepository() {
+        sharedExecutor = Executors.newCachedThreadPool(new NamedThreadFactory("Dubbo-shared-handler", true));
+        sharedScheduledExecutor = Executors.newScheduledThreadPool(8, new NamedThreadFactory("Dubbo-shared-scheduler", true));
+
         for (int i = 0; i < DEFAULT_SCHEDULER_SIZE; i++) {
             ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(
-                new NamedThreadFactory("Dubbo-framework-scheduler", true));
+                new NamedThreadFactory("Dubbo-framework-scheduler-" + i, true));
             scheduledExecutors.addItem(scheduler);
 
             executorServiceRing.addItem(new ThreadPoolExecutor(1, 1,
                 0L, TimeUnit.MILLISECONDS,
-                new LinkedBlockingQueue<Runnable>(1024), new NamedInternalThreadFactory("Dubbo-state-router-loop", true)
+                new LinkedBlockingQueue<Runnable>(1024), new NamedInternalThreadFactory("Dubbo-state-router-loop-" + i, true)
                 , new ThreadPoolExecutor.AbortPolicy()));
         }
-//
+
 //        reconnectScheduledExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("Dubbo-reconnect-scheduler"));
         poolRouterExecutor = new ThreadPoolExecutor(1, 10, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(1024),
             new NamedInternalThreadFactory("Dubbo-state-router-pool-router", true), new ThreadPoolExecutor.AbortPolicy());
@@ -123,7 +129,11 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
         Map<Integer, ExecutorService> executors = data.computeIfAbsent(EXECUTOR_SERVICE_COMPONENT_KEY, k -> new ConcurrentHashMap<>());
         // Consumer's executor is sharing globally, key=Integer.MAX_VALUE. Provider's executor is sharing by protocol.
         Integer portKey = CONSUMER_SIDE.equalsIgnoreCase(url.getParameter(SIDE_KEY)) ? Integer.MAX_VALUE : url.getPort();
-        ExecutorService executor = executors.computeIfAbsent(portKey, k -> createExecutor(url));
+        if (url.getParameter(THREAD_NAME_KEY) == null) {
+            url = url.putAttribute(THREAD_NAME_KEY, "Dubbo-protocol-"+portKey);
+        }
+        URL finalUrl = url;
+        ExecutorService executor = executors.computeIfAbsent(portKey, k -> createExecutor(finalUrl));
         // If executor has been shut down, create a new one
         if (executor.isShutdown() || executor.isTerminated()) {
             executors.remove(portKey);
@@ -156,7 +166,7 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
             logger.info("Executor for " + url + " is shutdown.");
         }
         if (executor == null) {
-            return SHARED_EXECUTOR;
+            return sharedExecutor;
         } else {
             return executor;
         }
@@ -201,17 +211,16 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
     }
 
     @Override
-    public ExecutorService getServiceExportExecutor() {
+    public ScheduledExecutorService getServiceExportExecutor() {
         if (serviceExportExecutor == null) {
             synchronized (LOCK) {
                 if (serviceExportExecutor == null) {
                     int coreSize = getExportThreadNum();
-                    serviceExportExecutor = Executors.newFixedThreadPool(coreSize,
+                    serviceExportExecutor = Executors.newScheduledThreadPool(coreSize,
                         new NamedThreadFactory("Dubbo-service-export", true));
                 }
             }
         }
-
         return serviceExportExecutor;
     }
 
@@ -219,14 +228,13 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
     public void shutdownServiceExportExecutor() {
         synchronized (LOCK) {
             if (serviceExportExecutor != null && !serviceExportExecutor.isShutdown()) {
-                try{
+                try {
                     serviceExportExecutor.shutdown();
-                }catch (Throwable ignored){
+                } catch (Throwable ignored) {
                     // ignored
-                    logger.warn(ignored.getMessage(),ignored);
+                    logger.warn(ignored.getMessage(), ignored);
                 }
             }
-
             serviceExportExecutor = null;
         }
     }
@@ -242,7 +250,6 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
                 }
             }
         }
-
         return serviceReferExecutor;
     }
 
@@ -250,13 +257,12 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
     public void shutdownServiceReferExecutor() {
         synchronized (LOCK) {
             if (serviceReferExecutor != null && !serviceReferExecutor.isShutdown()) {
-                try{
+                try {
                     serviceReferExecutor.shutdown();
-                }catch (Throwable ignored){
-                    logger.warn(ignored.getMessage(),ignored);
+                } catch (Throwable ignored) {
+                    logger.warn(ignored.getMessage(), ignored);
                 }
             }
-
             serviceReferExecutor = null;
         }
     }
@@ -341,7 +347,12 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
 
     @Override
     public ExecutorService getSharedExecutor() {
-        return SHARED_EXECUTOR;
+        return sharedExecutor;
+    }
+
+    @Override
+    public ScheduledExecutorService getSharedScheduledExecutor() {
+        return sharedScheduledExecutor;
     }
 
     private ExecutorService createExecutor(URL url) {
@@ -355,20 +366,10 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
 
     @Override
     public void destroyAll() {
-        try{
-            poolRouterExecutor.shutdown();
-        }catch (Throwable ignored){
-            // ignored
-            logger.warn(ignored.getMessage(),ignored);
-        }
-//        serviceDiscoveryAddressNotificationExecutor.shutdown();
-//        registryNotificationExecutor.shutdown();
-        try{
-            metadataRetryExecutor.shutdown();
-        }catch (Throwable ignored){
-            // ignored
-            logger.warn(ignored.getMessage(),ignored);
-        }
+        logger.info("destroying executor repository ..");
+        shutdownExecutorService(poolRouterExecutor, "poolRouterExecutor");
+        shutdownExecutorService(metadataRetryExecutor, "metadataRetryExecutor");
+
         shutdownServiceExportExecutor();
         shutdownServiceReferExecutor();
 
@@ -376,32 +377,50 @@ public class DefaultExecutorRepository implements ExecutorRepository, ExtensionA
             if (executors != null) {
                 executors.values().forEach(executor -> {
                     if (executor != null && !executor.isShutdown()) {
-                        try{
+                        try {
                             ExecutorUtil.shutdownNow(executor, 100);
-                        }catch (Throwable ignored){
+                        } catch (Throwable ignored) {
                             // ignored
-                            logger.warn(ignored.getMessage(),ignored);
+                            logger.warn(ignored.getMessage(), ignored);
                         }
                     }
                 });
             }
         });
+        data.clear();
 
-        // shutdown all executor services
-        for (ScheduledExecutorService executorService : scheduledExecutors.listItems()) {
-            try {
-                executorService.shutdown();
-            } catch (Exception e) {
-                logger.warn("shutdown scheduledExecutors failed: " + e.getMessage(), e);
-            }
+        // scheduledExecutors
+        shutdownExecutorServices(scheduledExecutors.listItems(), "scheduledExecutors");
+
+        // executorServiceRing
+        shutdownExecutorServices(executorServiceRing.listItems(), "executorServiceRing");
+
+        // shutdown share executor
+        shutdownExecutorService(sharedExecutor, "sharedExecutor");
+        shutdownExecutorService(sharedScheduledExecutor, "sharedScheduledExecutor");
+
+        // serviceDiscoveryAddressNotificationExecutorRing
+        shutdownExecutorServices(serviceDiscoveryAddressNotificationExecutorRing.listItems(),
+            "serviceDiscoveryAddressNotificationExecutorRing");
+
+        // registryNotificationExecutorRing
+        shutdownExecutorServices(registryNotificationExecutorRing.listItems(),
+            "registryNotificationExecutorRing");
+
+    }
+
+    private void shutdownExecutorServices(List<? extends ExecutorService> executorServices, String msg) {
+        for (ExecutorService executorService : executorServices) {
+            shutdownExecutorService(executorService, msg);
         }
+    }
 
-        for (ExecutorService executorService : executorServiceRing.listItems()) {
-            try {
-                executorService.shutdown();
-            } catch (Exception e) {
-                logger.warn("shutdown executorServiceRing failed: " + e.getMessage(), e);
-            }
+    private void shutdownExecutorService(ExecutorService executorService, String name) {
+        try {
+            executorService.shutdownNow();
+        } catch (Exception e) {
+            String msg = "shutdown executor service [" + name + "] failed: ";
+            logger.warn(msg + e.getMessage(), e);
         }
     }
 

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepository.java

@@ -58,7 +58,7 @@ public interface ExecutorRepository {
 
     ExecutorService nextExecutorExecutor();
 
-    ExecutorService getServiceExportExecutor();
+    ScheduledExecutorService getServiceExportExecutor();
 
     /**
      * The executor only used in bootstrap currently, we should call this method to release the resource
@@ -92,6 +92,12 @@ public interface ExecutorRepository {
      */
     ExecutorService getSharedExecutor();
 
+    /**
+     * Get the shared schedule executor
+     * @return
+     */
+    ScheduledExecutorService getSharedScheduledExecutor();
+
     ExecutorService getPoolRouterExecutor();
 
     /**

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/cached/CachedThreadPool.java

@@ -47,7 +47,7 @@ public class CachedThreadPool implements ThreadPool {
 
     @Override
     public Executor getExecutor(URL url) {
-        String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME);
+        String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME));
         int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS);
         int threads = url.getParameter(THREADS_KEY, Integer.MAX_VALUE);
         int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES);

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/eager/EagerThreadPool.java

@@ -44,7 +44,7 @@ public class EagerThreadPool implements ThreadPool {
 
     @Override
     public Executor getExecutor(URL url) {
-        String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME);
+        String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME));
         int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS);
         int threads = url.getParameter(THREADS_KEY, Integer.MAX_VALUE);
         int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES);

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/fixed/FixedThreadPool.java

@@ -43,7 +43,7 @@ public class FixedThreadPool implements ThreadPool {
 
     @Override
     public Executor getExecutor(URL url) {
-        String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME);
+        String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME));
         int threads = url.getParameter(THREADS_KEY, DEFAULT_THREADS);
         int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES);
         return new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS,

dubbo-common/src/main/java/org/apache/dubbo/common/threadpool/support/limited/LimitedThreadPool.java

@@ -45,7 +45,7 @@ public class LimitedThreadPool implements ThreadPool {
 
     @Override
     public Executor getExecutor(URL url) {
-        String name = url.getParameter(THREAD_NAME_KEY, DEFAULT_THREAD_NAME);
+        String name = url.getParameter(THREAD_NAME_KEY, (String) url.getAttribute(THREAD_NAME_KEY, DEFAULT_THREAD_NAME));
         int cores = url.getParameter(CORE_THREADS_KEY, DEFAULT_CORE_THREADS);
         int threads = url.getParameter(THREADS_KEY, DEFAULT_THREADS);
         int queues = url.getParameter(QUEUES_KEY, DEFAULT_QUEUES);

dubbo-common/src/main/java/org/apache/dubbo/common/utils/ClassLoaderResourceLoader.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.common.utils;
 
+import org.apache.dubbo.common.resource.GlobalResourcesRepository;
+
 import java.io.IOException;
 import java.lang.ref.SoftReference;
 import java.lang.reflect.Field;
@@ -30,25 +32,21 @@ import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 public class ClassLoaderResourceLoader {
-    private final static ExecutorService executorService =
-        new ThreadPoolExecutor(0, Integer.MAX_VALUE,
-            60L, TimeUnit.SECONDS,
-            new SynchronousQueue<>(),
-            new NamedThreadFactory("DubboClassLoaderResourceLoader", true));
 
     private static SoftReference<Map<ClassLoader, Map<String, Set<URL>>>> classLoaderResourcesCache = null;
 
+    static {
+        // register resources destroy listener
+        GlobalResourcesRepository.getInstance().registerDisposable(()-> destroy(), true);
+    }
+
     public static Map<ClassLoader, Set<java.net.URL>> loadResources(String fileName, List<ClassLoader> classLoaders) {
         Map<ClassLoader, Set<java.net.URL>> resources = new ConcurrentHashMap<>();
         CountDownLatch countDownLatch = new CountDownLatch(classLoaders.size());
         for (ClassLoader classLoader : classLoaders) {
-            executorService.submit(()->{
+            GlobalResourcesRepository.getGlobalExecutorService().submit(() -> {
                 resources.put(classLoader, loadResources(fileName, classLoader));
                 countDownLatch.countDown();
             });
@@ -99,6 +97,11 @@ public class ClassLoaderResourceLoader {
         return urlCache.get(fileName);
     }
 
+    public static void destroy() {
+        if (classLoaderResourcesCache != null) {
+            classLoaderResourcesCache.clear();
+        }
+    }
 
     private static void setRef(URL url) {
         try {

dubbo-common/src/main/java/org/apache/dubbo/config/ServiceConfigBase.java

@@ -185,13 +185,14 @@ public abstract class ServiceConfigBase<T> extends AbstractServiceConfig {
         }
         if (!interfaceClass.isInstance(ref)) {
             throw new IllegalStateException("The class "
-                + ref.getClass().getName() + getClassloaderDesc(ref.getClass()) + " unimplemented interface "
-                + interfaceClass + getClassloaderDesc(interfaceClass) + "!");
+                + getClassDesc(ref.getClass()) + " unimplemented interface "
+                + getClassDesc(interfaceClass) + "!");
         }
     }
 
-    private String getClassloaderDesc(Class clazz) {
-        return "[classloader=" + clazz.getClassLoader().getClass().getName() + "@" + clazz.getClassLoader().hashCode() + "]";
+    private String getClassDesc(Class clazz) {
+        ClassLoader classLoader = clazz.getClassLoader();
+        return clazz.getName() + "[classloader=" + classLoader.getClass().getName() + "@" + classLoader.hashCode() + "]";
     }
 
     public Optional<String> getContextPath(ProtocolConfig protocolConfig) {

dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ApplicationModel.java

@@ -201,13 +201,20 @@ public class ApplicationModel extends ScopeModel {
     }
 
     @Override
-    public void onDestroy() {
-        // TODO destroy application resources
-        for (ModuleModel moduleModel : new ArrayList<>(moduleModels)) {
-            moduleModel.destroy();
+    protected void onDestroy() {
+
+        if (deployer != null) {
+            deployer.preDestroy();
         }
 
-        notifyDestroy();
+        // destroy application resources
+        for (ModuleModel moduleModel : new ArrayList<>(moduleModels)) {
+            if (moduleModel != internalModule) {
+                moduleModel.destroy();
+            }
+        }
+        // destroy internal module later
+        internalModule.destroy();
 
         if (defaultInstance == this) {
             synchronized (ApplicationModel.class) {
@@ -219,10 +226,12 @@ public class ApplicationModel extends ScopeModel {
         }
 
         if (deployer != null) {
-            deployer.destroy();
-            deployer = null;
+            deployer.postDestroy();
         }
 
+        // destroy other resources (e.g. ZookeeperTransporter )
+        notifyDestroy();
+
         if (environment != null) {
             environment.destroy();
             environment = null;
@@ -235,6 +244,8 @@ public class ApplicationModel extends ScopeModel {
             serviceRepository.destroy();
             serviceRepository = null;
         }
+        // try destroy framework if no any application
+        frameworkModel.tryDestroy();
     }
 
     public FrameworkModel getFrameworkModel() {
@@ -301,12 +312,13 @@ public class ApplicationModel extends ScopeModel {
             if (moduleModel == defaultModule) {
                 defaultModule = findDefaultModule();
             }
-            if (this.moduleModels.size() == 1 && this.moduleModels.get(0) == internalModule) {
-                this.internalModule.destroy();
-            }
-            if (this.moduleModels.isEmpty()) {
-                destroy();
-            }
+        }
+    }
+
+    void tryDestroy() {
+        if (this.moduleModels.isEmpty()
+            || (this.moduleModels.size() == 1 && this.moduleModels.get(0) == internalModule)) {
+            destroy();
         }
     }
 

dubbo-common/src/main/java/org/apache/dubbo/rpc/model/FrameworkModel.java

@@ -21,6 +21,7 @@ import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.common.extension.ExtensionScope;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.resource.GlobalResourcesRepository;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -49,7 +50,6 @@ public class FrameworkModel extends ScopeModel {
     private FrameworkServiceRepository serviceRepository;
 
 
-
     public FrameworkModel() {
         super(null, ExtensionScope.FRAMEWORK);
         initialize();
@@ -70,20 +70,38 @@ public class FrameworkModel extends ScopeModel {
     }
 
     @Override
-    public void onDestroy() {
-        //TODO destroy framework model
+    protected void onDestroy() {
+        // destroy all application model
         for (ApplicationModel applicationModel : new ArrayList<>(applicationModels)) {
             applicationModel.destroy();
         }
 
-        allInstances.remove(this);
-        if (defaultInstance == this) {
-            synchronized (FrameworkModel.class) {
+        if (LOGGER.isInfoEnabled()) {
+            LOGGER.info("Dubbo framework[" + getInternalId() + "] is destroying ...");
+        }
+        synchronized (FrameworkModel.class) {
+            allInstances.remove(this);
+            if (defaultInstance == this) {
                 defaultInstance = null;
             }
         }
 
+        // notify destroy and clean framework resources
+        // see org.apache.dubbo.config.deploy.FrameworkModelCleaner
         notifyDestroy();
+
+        if (LOGGER.isInfoEnabled()) {
+            LOGGER.info("Dubbo framework[" + getInternalId() + "] is destroyed");
+        }
+
+        // if all FrameworkModels are destroyed, clean global static resources, shutdown dubbo completely
+        if (allInstances.isEmpty()) {
+            destroyGlobalResources();
+        }
+    }
+
+    private void destroyGlobalResources() {
+        GlobalResourcesRepository.getInstance().destroy();
     }
 
     public static FrameworkModel defaultModel() {
@@ -120,6 +138,9 @@ public class FrameworkModel extends ScopeModel {
 
     synchronized void removeApplication(ApplicationModel model) {
         this.applicationModels.remove(model);
+    }
+
+    synchronized void tryDestroy() {
         if (applicationModels.size() == 0) {
             destroy();
         }

dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ModuleModel.java

@@ -81,14 +81,20 @@ public class ModuleModel extends ScopeModel {
     }
 
     @Override
-    public void onDestroy() {
-        notifyDestroy();
+    protected void onDestroy() {
+        if (deployer != null) {
+            deployer.preDestroy();
+        }
+
         applicationModel.removeModule(this);
 
         if (deployer != null) {
-            deployer.destroy();
-            deployer = null;
+            deployer.postDestroy();
         }
+
+        // destroy other resources
+        notifyDestroy();
+
         if (serviceRepository != null) {
             serviceRepository.destroy();
             serviceRepository = null;
@@ -98,6 +104,7 @@ public class ModuleModel extends ScopeModel {
             moduleEnvironment.destroy();
             moduleEnvironment = null;
         }
+        applicationModel.tryDestroy();
     }
 
     public ApplicationModel getApplicationModel() {

dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModel.java

@@ -69,7 +69,6 @@ public abstract class ScopeModel implements ExtensionAccessor {
 
     private Map<String, Object> attributes;
     private AtomicBoolean destroyed = new AtomicBoolean(false);
-    private volatile boolean stopping;
 
     public ScopeModel(ScopeModel parent, ExtensionScope scope) {
         this.parent = parent;
@@ -92,7 +91,6 @@ public abstract class ScopeModel implements ExtensionAccessor {
         this.destroyListeners = new LinkedList<>();
         this.attributes = new ConcurrentHashMap<>();
         this.classLoaders = new ConcurrentHashSet<>();
-        this.stopping = false;
 
         // Add Framework's ClassLoader by default
         ClassLoader dubboClassLoader = ScopeModel.class.getClassLoader();
@@ -119,21 +117,13 @@ public abstract class ScopeModel implements ExtensionAccessor {
         return destroyed.get();
     }
 
-    public void setStopping() {
-        stopping = true;
-    }
-
-    public boolean isStopping() {
-        return stopping;
-    }
-
     protected void notifyDestroy() {
         for (ScopeModelDestroyListener destroyListener : destroyListeners) {
             destroyListener.onDestroy(this);
         }
     }
 
-    public abstract void onDestroy();
+    protected abstract void onDestroy();
 
     public final void addDestroyListener(ScopeModelDestroyListener listener) {
         destroyListeners.add(listener);

dubbo-common/src/main/java/org/apache/dubbo/rpc/model/ScopeModelDestroyListener.java

@@ -16,6 +16,6 @@
  */
 package org.apache.dubbo.rpc.model;
 
-public interface ScopeModelDestroyListener {
-    void onDestroy(ScopeModel scopeModel);
+public interface ScopeModelDestroyListener<T extends ScopeModel> {
+    void onDestroy(T scopeModel);
 }

dubbo-common/src/test/java/org/apache/dubbo/common/config/configcenter/file/FileSystemDynamicConfigurationTest.java

@@ -16,23 +16,22 @@
  */
 package org.apache.dubbo.common.config.configcenter.file;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
-
-import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.TreeSet;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Arrays.asList;
-import static org.apache.commons.io.FileUtils.deleteQuietly;
 import static org.apache.dubbo.common.URL.valueOf;
 import static org.apache.dubbo.common.config.configcenter.DynamicConfiguration.DEFAULT_GROUP;
 import static org.apache.dubbo.common.config.configcenter.file.FileSystemDynamicConfiguration.CONFIG_CENTER_DIR_PARAM_NAME;
@@ -60,13 +59,18 @@ public class FileSystemDynamicConfigurationTest {
     public void init() {
         File rootDirectory = new File(getClassPath(), "config-center");
         rootDirectory.mkdirs();
+        try {
+            FileUtils.cleanDirectory(rootDirectory);
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
         URL url = valueOf("dubbo://127.0.0.1:20880").addParameter(CONFIG_CENTER_DIR_PARAM_NAME, rootDirectory.getAbsolutePath());
         configuration = new FileSystemDynamicConfiguration(url);
     }
 
     @AfterEach
     public void destroy() throws Exception {
-        deleteQuietly(configuration.getRootDirectory());
+        FileUtils.deleteQuietly(configuration.getRootDirectory());
         configuration.close();
     }
 

dubbo-common/src/test/java/org/apache/dubbo/common/threadpool/manager/ExecutorRepositoryTest.java

@@ -23,16 +23,12 @@ import org.junit.jupiter.api.Test;
 
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public class ExecutorRepositoryTest {
     private ExecutorRepository executorRepository = ApplicationModel.defaultModel().getExtensionLoader(ExecutorRepository.class).getDefaultExtension();
 
     @Test
-    public void test() {
-        testGetExecutor();
-        testUpdateExecutor();
-    }
-
     public void testGetExecutor() {
         testGet(URL.valueOf("dubbo://127.0.0.1:23456"));
         testGet(URL.valueOf("dubbo://127.0.0.1:23456?side=consumer"));
@@ -54,6 +50,7 @@ public class ExecutorRepositoryTest {
         Assertions.assertNotEquals(executorService, executorRepository.getExecutor(url));
     }
 
+    @Test
     public void testUpdateExecutor() {
         URL url = URL.valueOf("dubbo://127.0.0.1:23456?threads=5");
         ThreadPoolExecutor executorService = (ThreadPoolExecutor) executorRepository.createExecutorIfAbsent(url);
@@ -75,7 +72,57 @@ public class ExecutorRepositoryTest {
 
         executorService.setCorePoolSize(5);
         executorRepository.updateThreadpool(url, executorService);
+    }
 
+    @Test
+    public void testSharedExecutor() throws Exception {
+        ExecutorService sharedExecutor = executorRepository.getSharedExecutor();
+        MockTask task1 = new MockTask(2000);
+        MockTask task2 = new MockTask(100);
+        MockTask task3 = new MockTask(200);
+        sharedExecutor.execute(task1);
+        sharedExecutor.execute(task2);
+        sharedExecutor.submit(task3);
 
+        Thread.sleep(150);
+        Assertions.assertTrue(task1.isRunning());
+        Assertions.assertFalse(task1.isDone());
+        Assertions.assertTrue(task2.isRunning());
+        Assertions.assertTrue(task2.isDone());
+        Assertions.assertTrue(task3.isRunning());
+        Assertions.assertFalse(task3.isDone());
+
+        Thread.sleep(200);
+        Assertions.assertTrue(task3.isDone());
+        Assertions.assertFalse(task1.isDone());
+    }
+
+    private static class MockTask implements Runnable {
+        private long waitTimeMS;
+        private AtomicBoolean running = new AtomicBoolean();
+        private AtomicBoolean done = new AtomicBoolean();
+
+        public MockTask(long waitTimeMS) {
+            this.waitTimeMS = waitTimeMS;
+        }
+
+        @Override
+        public void run() {
+            running.set(true);
+            try {
+                Thread.sleep(waitTimeMS);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            done.set(true);
+        }
+
+        public boolean isDone() {
+            return done.get();
+        }
+
+        public boolean isRunning() {
+            return running.get();
+        }
     }
 }

dubbo-compatible/pom.xml

@@ -98,5 +98,11 @@
             <artifactId>fastjson</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ConfigScopeModelInitializer.java

@@ -21,6 +21,7 @@ import org.apache.dubbo.common.deploy.ApplicationDeployer;
 import org.apache.dubbo.common.deploy.ModuleDeployer;
 import org.apache.dubbo.config.deploy.DefaultApplicationDeployer;
 import org.apache.dubbo.config.deploy.DefaultModuleDeployer;
+import org.apache.dubbo.config.deploy.FrameworkModelCleaner;
 import org.apache.dubbo.config.utils.DefaultConfigValidator;
 import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.apache.dubbo.rpc.model.FrameworkModel;
@@ -31,7 +32,7 @@ public class ConfigScopeModelInitializer implements ScopeModelInitializer {
 
     @Override
     public void initializeFrameworkModel(FrameworkModel frameworkModel) {
-
+        frameworkModel.addDestroyListener(new FrameworkModelCleaner());
     }
 
     @Override

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/DubboShutdownHook.java

@@ -87,8 +87,8 @@ public class DubboShutdownHook extends Thread {
      */
     public void unregister() {
         if (registered.compareAndSet(true, false)) {
-            if (this.isAlive() || destroyed.get()) {
-                // DubboShutdownHook is running
+            if (this.isAlive()) {
+                // DubboShutdownHook thread is running
                 return;
             }
             try {

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/ServiceConfig.java

@@ -23,11 +23,11 @@ import org.apache.dubbo.common.deploy.ModuleDeployer;
 import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.threadpool.manager.ExecutorRepository;
 import org.apache.dubbo.common.url.component.ServiceConfigURL;
 import org.apache.dubbo.common.utils.ClassUtils;
 import org.apache.dubbo.common.utils.CollectionUtils;
 import org.apache.dubbo.common.utils.ConfigUtils;
-import org.apache.dubbo.common.utils.NamedThreadFactory;
 import org.apache.dubbo.common.utils.StringUtils;
 import org.apache.dubbo.config.annotation.Service;
 import org.apache.dubbo.config.invoker.DelegateProviderMetaDataInvoker;
@@ -57,8 +57,6 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
@@ -108,11 +106,6 @@ public class ServiceConfig<T> extends ServiceConfigBase<T> {
      */
     private static final Map<String, Integer> RANDOM_PORT_MAP = new HashMap<String, Integer>();
 
-    /**
-     * A delayed exposure service timer
-     */
-    private static final ScheduledExecutorService DELAY_EXPORT_EXECUTOR = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboServiceDelayExporter", true));
-
     private Protocol protocolSPI;
 
     /**
@@ -259,13 +252,14 @@ public class ServiceConfig<T> extends ServiceConfigBase<T> {
     }
 
     protected void doDelayExport() {
-        DELAY_EXPORT_EXECUTOR.schedule(() -> {
-            try {
-                doExport();
-            } catch (Exception e) {
-                logger.error("Failed to export service config: " + interfaceName, e);
-            }
-        }, getDelay(), TimeUnit.MILLISECONDS);
+        getScopeModel().getDefaultExtension(ExecutorRepository.class).getServiceExportExecutor()
+            .schedule(() -> {
+                try {
+                    doExport();
+                } catch (Exception e) {
+                    logger.error("Failed to export service config: " + interfaceName, e);
+                }
+            }, getDelay(), TimeUnit.MILLISECONDS);
     }
 
     protected void exported() {

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/bootstrap/DubboBootstrap.java

@@ -46,16 +46,14 @@ import org.apache.dubbo.config.bootstrap.builders.ReferenceBuilder;
 import org.apache.dubbo.config.bootstrap.builders.RegistryBuilder;
 import org.apache.dubbo.config.bootstrap.builders.ServiceBuilder;
 import org.apache.dubbo.config.context.ConfigManager;
-import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory;
-import org.apache.dubbo.rpc.Protocol;
 import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.model.ModuleModel;
 
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
@@ -127,10 +125,6 @@ public final class DubboBootstrap {
         return getInstance(frameworkModel.newApplication());
     }
 
-    public static DubboBootstrap newInstance(ApplicationModel applicationModel) {
-        return getInstance(applicationModel);
-    }
-
     /**
      * Try reset dubbo status for new instance.
      *
@@ -153,8 +147,6 @@ public final class DubboBootstrap {
                 instance.destroy();
                 instance = null;
             }
-            AbstractMetadataReportFactory.destroy();
-            destroyAllProtocols();
             FrameworkModel.destroyAll();
         } else {
             instance = null;
@@ -227,7 +219,7 @@ public final class DubboBootstrap {
      * @return
      */
     public DubboBootstrap start(boolean wait) {
-        CompletableFuture future = applicationDeployer.start();
+        Future future = applicationDeployer.start();
         if (wait) {
             try {
                 future.get();
@@ -238,6 +230,14 @@ public final class DubboBootstrap {
         return this;
     }
 
+    /**
+     * Start dubbo application but no wait for finish.
+     * @return the future object
+     */
+    public Future asyncStart() {
+        return applicationDeployer.start();
+    }
+
     /**
      * Stop dubbo application
      * @return
@@ -249,7 +249,7 @@ public final class DubboBootstrap {
     }
 
     public void destroy() {
-        applicationDeployer.destroy();
+        applicationModel.destroy();
     }
 
     public boolean isInitialized() {
@@ -330,30 +330,6 @@ public final class DubboBootstrap {
         return applicationDeployer.getReferenceCache();
     }
 
-    /**
-     * Destroy all the protocols.
-     */
-    private static void destroyProtocols(FrameworkModel frameworkModel) {
-        //TODO destroy protocol in framework scope
-        ExtensionLoader<Protocol> loader = frameworkModel.getExtensionLoader(Protocol.class);
-        for (String protocolName : loader.getLoadedExtensions()) {
-            try {
-                Protocol protocol = loader.getLoadedExtension(protocolName);
-                if (protocol != null) {
-                    protocol.destroy();
-                }
-            } catch (Throwable t) {
-                logger.warn(t.getMessage(), t);
-            }
-        }
-    }
-
-    private static void destroyAllProtocols() {
-        for (FrameworkModel frameworkModel : FrameworkModel.getAllInstances()) {
-            destroyProtocols(frameworkModel);
-        }
-    }
-
     private void executeMutually(Runnable runnable) {
         try {
             lock.lock();

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultApplicationDeployer.java

@@ -26,6 +26,7 @@ import org.apache.dubbo.common.config.configcenter.wrapper.CompositeDynamicConfi
 import org.apache.dubbo.common.deploy.AbstractDeployer;
 import org.apache.dubbo.common.deploy.ApplicationDeployListener;
 import org.apache.dubbo.common.deploy.ApplicationDeployer;
+import org.apache.dubbo.common.deploy.DeployState;
 import org.apache.dubbo.common.deploy.ModuleDeployer;
 import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.common.lang.ShutdownHookCallbacks;
@@ -48,16 +49,13 @@ import org.apache.dubbo.metadata.MetadataServiceExporter;
 import org.apache.dubbo.metadata.WritableMetadataService;
 import org.apache.dubbo.metadata.report.MetadataReportFactory;
 import org.apache.dubbo.metadata.report.MetadataReportInstance;
-import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory;
 import org.apache.dubbo.registry.client.DefaultServiceInstance;
 import org.apache.dubbo.registry.client.ServiceInstance;
 import org.apache.dubbo.registry.client.metadata.ServiceInstanceMetadataUtils;
 import org.apache.dubbo.registry.client.metadata.store.InMemoryWritableMetadataService;
 import org.apache.dubbo.registry.client.metadata.store.RemoteMetadataServiceImpl;
 import org.apache.dubbo.registry.support.RegistryManager;
-import org.apache.dubbo.rpc.Protocol;
 import org.apache.dubbo.rpc.model.ApplicationModel;
-import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.model.ModuleModel;
 import org.apache.dubbo.rpc.model.ScopeModel;
 import org.apache.dubbo.rpc.model.ScopeModelUtil;
@@ -70,6 +68,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -115,8 +114,9 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
 
     private ScheduledFuture<?> asyncMetadataFuture;
     private String identifier;
-    private CompletableFuture startFuture;
+    private volatile CompletableFuture startFuture;
     private DubboShutdownHook dubboShutdownHook;
+    private Object startedLock = new Object();
 
     public DefaultApplicationDeployer(ApplicationModel applicationModel) {
         super(applicationModel);
@@ -512,25 +512,26 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
      * @return
      */
     @Override
-    public synchronized CompletableFuture start() {
+    public synchronized Future start() {
+        CompletableFuture startFuture = getStartFuture();
+
+        // maybe call start again after add new module, check if any new module
+        boolean hasPendingModule = hasPendingModule();
+
         if (isStarting()) {
+            // currently is starting, maybe both start by module and application
+            // if has new modules, start them
+            if (hasPendingModule) {
+                startModules();
+            }
+            // if is starting, reuse previous startFuture
             return startFuture;
         }
-        startFuture = new CompletableFuture();
-        if (isStarted()) {
-            // maybe call start again after add new module, check if any new module
-            boolean hasNewModule = false;
-            for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
-                if (moduleModel.getDeployer().isPending()) {
-                    hasNewModule = true;
-                    break;
-                }
-            }
-            // if no new module, just return
-            if (!hasNewModule) {
-                startFuture.complete(false);
-                return startFuture;
-            }
+
+        // if is started and no new module, just return
+        if (isStarted() && !hasPendingModule) {
+            completeStartFuture(false);
+            return startFuture;
         }
 
         onStarting();
@@ -542,35 +543,69 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
         return startFuture;
     }
 
-    private void doStart() {
-        // copy current modules, ignore new module during starting
-        List<ModuleModel> moduleModels = new ArrayList<>(applicationModel.getModuleModels());
-        List<CompletableFuture> futures = new ArrayList<>(moduleModels.size());
-
-        for (ModuleModel moduleModel : moduleModels) {
-            // export services in module
+    private boolean hasPendingModule() {
+        boolean found = false;
+        for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
             if (moduleModel.getDeployer().isPending()) {
-                CompletableFuture moduleFuture = moduleModel.getDeployer().start();
-                futures.add(moduleFuture);
+                found = true;
+                break;
             }
         }
+        return found;
+    }
+
+    private CompletableFuture getStartFuture() {
+        if (startFuture == null) {
+            synchronized (this) {
+                if (startFuture == null) {
+                    startFuture = new CompletableFuture();
+                }
+            }
+        }
+        return startFuture;
+    }
+
+
+    private void doStart() {
+        startModules();
 
         // prepare application instance
         prepareApplicationInstance();
 
-        // notify on each module started
-//        executorRepository.getSharedExecutor().submit(()-> {
-//            awaitDeployFinished(futures);
-//            onStarted();
-//        });
+        executorRepository.getSharedExecutor().submit(() -> {
+            while (true) {
+                // notify on each module started
+                synchronized (startedLock) {
+                    try {
+                        startedLock.wait(500);
+                    } catch (InterruptedException e) {
+                        // ignore
+                    }
+                }
+
+                // if has new module, do start again
+                if (hasPendingModule()) {
+                    startModules();
+                    continue;
+                }
+
+                DeployState newState = checkState();
+                if (!(newState == DeployState.STARTING || newState == DeployState.PENDING)) {
+                    // start finished or error
+                    break;
+                }
+            }
+        });
     }
 
-    private void awaitDeployFinished(List<CompletableFuture> futures) {
-        try {
-            CompletableFuture mergedFuture = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
-            mergedFuture.get();
-        } catch (Exception e) {
-            logger.error(getIdentifier() + " await deploy finished failed", e);
+    private void startModules() {
+        // copy current modules, ignore new module during starting
+        List<ModuleModel> moduleModels = new ArrayList<>(applicationModel.getModuleModels());
+        for (ModuleModel moduleModel : moduleModels) {
+            // export services in module
+            if (moduleModel.getDeployer().isPending()) {
+                moduleModel.getDeployer().start();
+            }
         }
     }
 
@@ -595,8 +630,14 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
         exportMetadataService();
         // start internal module
         ModuleDeployer internalModuleDeployer = applicationModel.getInternalModule().getDeployer();
-        if (!internalModuleDeployer.isRunning()) {
-            internalModuleDeployer.start();
+        if (!internalModuleDeployer.isStarted()) {
+            Future future = internalModuleDeployer.start();
+            // wait for internal module start finished
+            try {
+                future.get();
+            } catch (Exception e) {
+                logger.warn("wait for internal module started failed: " + e.getMessage(), e);
+            }
         }
     }
 
@@ -707,15 +748,23 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
         }
         if (registered) {
             // scheduled task for updating Metadata and ServiceInstance
-            asyncMetadataFuture = executorRepository.nextScheduledExecutor().scheduleAtFixedRate(() -> {
-                InMemoryWritableMetadataService localMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(applicationModel);
-                if (!applicationModel.getDeployer().isStopping() || !applicationModel.getDeployer().isStopped()) {
-                    localMetadataService.blockUntilUpdated();
+            asyncMetadataFuture = executorRepository.getSharedScheduledExecutor().scheduleAtFixedRate(() -> {
+
+                // ignore refresh metadata on stopping
+                if (applicationModel.isDestroyed()) {
+                    return;
                 }
+
+                InMemoryWritableMetadataService localMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(applicationModel);
+                localMetadataService.blockUntilUpdated();
                 try {
-                    ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance);
+                    if (!applicationModel.isDestroyed()) {
+                        ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance);
+                    }
                 } catch (Exception e) {
-                    logger.error("Refresh instance and metadata error", e);
+                    if (!applicationModel.isDestroyed()) {
+                        logger.error("Refresh instance and metadata error", e);
+                    }
                 } finally {
                     localMetadataService.releaseBlock();
                 }
@@ -780,36 +829,45 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
 
     @Override
     public void stop() {
-        destroy();
+        applicationModel.destroy();
     }
 
     @Override
-    public synchronized void destroy() {
+    public void preDestroy() {
         if (isStopping() || isStopped()) {
             return;
         }
+        onStopping();
+
+        unRegisterShutdownHook();
+        if (asyncMetadataFuture != null) {
+            asyncMetadataFuture.cancel(true);
+        }
+        unregisterServiceInstance();
+        unexportMetadataService();
+
+    }
+
+    @Override
+    public synchronized void postDestroy() {
+        // expect application model is destroyed before here
+        if (isStopped()) {
+            return;
+        }
         try {
-            onStopping();
-            unRegisterShutdownHook();
-            unregisterServiceInstance();
-            unexportMetadataService();
-            if (asyncMetadataFuture != null) {
-                asyncMetadataFuture.cancel(true);
-            }
-
             executeShutdownCallbacks();
 
-            applicationModel.destroy();
-
-            destroyProtocols();
-
             destroyRegistries();
             destroyServiceDiscoveries();
             destroyMetadataReports();
 
-            destroyServiceDiscoveries();
+            // TODO should we close unused protocol server which only used by this application?
+            // protocol server will be closed on all applications of same framework are stopped currently, but no associate to application
+            // see org.apache.dubbo.config.deploy.FrameworkModelCleaner#destroyProtocols
+            // see org.apache.dubbo.config.bootstrap.DubboBootstrapMultiInstanceTest#testMultiProviderApplicationStopOneByOne
+
+            // destroy all executor services
             destroyExecutorRepository();
-            destroyDynamicConfigurations();
 
             onStopped();
         } catch (Throwable ex) {
@@ -828,44 +886,129 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
         if (isStarting()) {
             return;
         }
-        onStarting();
+        for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
+            if (moduleModel.getDeployer().isStarting()) {
+                onStarting();
+                break;
+            }
+        }
     }
 
     @Override
-    public void checkStarted(CompletableFuture checkerStartFuture) {
-        for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
-            if (moduleModel.getDeployer().isPending()) {
+    public void checkStarted() {
+        // TODO improve newState checking
+        DeployState newState = checkState();
+        switch (newState) {
+            case STARTED:
+                onStarted();
+                break;
+            case STARTING:
+                onStarting();
+                break;
+            case PENDING:
                 setPending();
-            } else if (moduleModel.getDeployer().isStarting()) {
-                return;
+                break;
+        }
+
+        // notify started
+        synchronized (startedLock) {
+            startedLock.notifyAll();
+        }
+    }
+
+    private DeployState checkState() {
+        DeployState newState = DeployState.UNKNOWN;
+        int pending = 0, starting = 0, started = 0, stopping = 0, stopped = 0;
+        for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
+            ModuleDeployer deployer = moduleModel.getDeployer();
+            if (deployer.isPending()) {
+                pending++;
+            } else if (deployer.isStarting()) {
+                starting++;
+            } else if (deployer.isStarted()) {
+                started++;
+            } else if (deployer.isStopping()) {
+                stopping++;
+            } else if (deployer.isStopped()) {
+                stopped++;
             }
         }
-        // all modules has been started
-        onStarted(checkerStartFuture);
+
+        if (started > 0) {
+            if (pending + starting + stopping + stopped == 0) {
+                // all modules have been started
+                newState = DeployState.STARTED;
+            } else if (pending + starting > 0) {
+                // some module is pending and some is started
+                newState = DeployState.STARTING;
+            } else if (stopping + stopped > 0) {
+                newState = DeployState.STOPPING;
+            }
+        } else if (starting > 0) {
+            // any module is starting
+            newState = DeployState.STARTING;
+        } else if (pending > 0) {
+            if (starting + starting + stopping + stopped == 0) {
+                // all modules have not starting or started
+                newState = DeployState.PENDING;
+            } else if (stopping + stopped > 0) {
+                // some is pending and some is stopping or stopped
+                newState = DeployState.STOPPING;
+            }
+        } else if (stopping > 0) {
+            // some is stopping and some stopped
+            newState = DeployState.STOPPING;
+        } else if (stopped > 0) {
+            // all modules are stopped
+            newState = DeployState.STOPPED;
+        }
+        return newState;
     }
 
     private void onStarting() {
+        if (isStarting()) {
+            return;
+        }
         setStarting();
         if (logger.isInfoEnabled()) {
             logger.info(getIdentifier() + " is starting.");
         }
     }
 
-    private void onStarted(CompletableFuture checkerStartFuture) {
+    private void onStarted() {
+        if (isStarted()) {
+            return;
+        }
         setStarted();
         if (logger.isInfoEnabled()) {
             logger.info(getIdentifier() + " is ready.");
         }
-        if (startFuture != null) {
-            startFuture.complete(true);
+        // refresh metadata
+        try {
+            if (serviceInstance != null) {
+                ServiceInstanceMetadataUtils.refreshMetadataAndInstance(serviceInstance);
+            }
+        } catch (Exception e) {
+            logger.error("refresh metadata failed: " + e.getMessage(), e);
         }
-        if (checkerStartFuture != null) {
-            checkerStartFuture.complete(true);
+        // complete future
+        completeStartFuture(true);
+        // shutdown export/refer executor after started
+        executorRepository.shutdownServiceExportExecutor();
+        executorRepository.shutdownServiceReferExecutor();
+    }
+
+    private void completeStartFuture(boolean success) {
+        if (startFuture != null) {
+            startFuture.complete(success);
+            startFuture = null;
         }
     }
 
     private void onStopping() {
-        applicationModel.setStopping();
+        if (isStopping()) {
+            return;
+        }
         setStopping();
         if (logger.isInfoEnabled()) {
             logger.info(getIdentifier() + " is stopping.");
@@ -873,6 +1016,9 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
     }
 
     private void onStopped() {
+        if (isStopped()) {
+            return;
+        }
         setStopped();
         if (logger.isInfoEnabled()) {
             logger.info(getIdentifier() + " has stopped.");
@@ -888,27 +1034,6 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
         RegistryManager.getInstance(applicationModel).destroyAll();
     }
 
-    /**
-     * Destroy all the protocols.
-     */
-    private void destroyProtocols() {
-        FrameworkModel frameworkModel = applicationModel.getFrameworkModel();
-        if (frameworkModel.getApplicationModels().isEmpty()) {
-            //TODO destroy protocol in framework scope
-            ExtensionLoader<Protocol> loader = frameworkModel.getExtensionLoader(Protocol.class);
-            for (String protocolName : loader.getLoadedExtensions()) {
-                try {
-                    Protocol protocol = loader.getLoadedExtension(protocolName);
-                    if (protocol != null) {
-                        protocol.destroy();
-                    }
-                } catch (Throwable t) {
-                    logger.warn(t.getMessage(), t);
-                }
-            }
-        }
-    }
-
     private void destroyServiceDiscoveries() {
         RegistryManager.getInstance(applicationModel).getServiceDiscoveries().forEach(serviceDiscovery -> {
             try {
@@ -923,15 +1048,11 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
     }
 
     private void destroyMetadataReports() {
-        // TODO only destroy MetadataReport of this application
-        AbstractMetadataReportFactory.destroy();
-    }
-
-    private void destroyDynamicConfigurations() {
-        // TODO only destroy DynamicConfiguration of this application
-        // DynamicConfiguration may be cached somewhere, and maybe used during destroy
-        // destroy them may cause some troubles, so just clear instances cache
-        // ExtensionLoader.resetExtensionLoader(DynamicConfigurationFactory.class);
+        // only destroy MetadataReport of this application
+        List<MetadataReportFactory> metadataReportFactories = getExtensionLoader(MetadataReportFactory.class).getLoadedExtensionInstances();
+        for (MetadataReportFactory metadataReportFactory : metadataReportFactories) {
+            metadataReportFactory.destroy();
+        }
     }
 
     private ApplicationConfig getApplication() {
@@ -940,10 +1061,10 @@ public class DefaultApplicationDeployer extends AbstractDeployer<ApplicationMode
 
     private String getIdentifier() {
         if (identifier == null) {
-            if (applicationModel.getModelName() != null && !StringUtils.isEquals(applicationModel.getModelName(), applicationModel.getInternalName())) {
-                identifier = applicationModel.getModelName() + "[" + applicationModel.getInternalId() + "]";
-            } else {
-                identifier = "Dubbo Application" + "[" + applicationModel.getInternalId() + "]";
+            identifier = "Dubbo application[" + applicationModel.getInternalId() + "]";
+            if (applicationModel.getModelName() != null
+                && !StringUtils.isEquals(applicationModel.getModelName(), applicationModel.getInternalName())) {
+                identifier += "(" + applicationModel.getModelName() + ")";
             }
         }
         return identifier;

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/DefaultModuleDeployer.java

@@ -43,6 +43,7 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 
 /**
  * Export/refer services of module
@@ -120,7 +121,7 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
     }
 
     @Override
-    public synchronized CompletableFuture start() throws IllegalStateException {
+    public synchronized Future start() throws IllegalStateException {
         if (isStarting() || isStarted()) {
             return startFuture;
         }
@@ -164,15 +165,22 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
 
     @Override
     public void stop() throws IllegalStateException {
-        destroy();
+        moduleModel.destroy();
     }
 
     @Override
-    public synchronized void destroy() throws IllegalStateException {
+    public void preDestroy() throws IllegalStateException {
         if (isStopping() || isStopped()) {
             return;
         }
         onModuleStopping();
+    }
+
+    @Override
+    public synchronized void postDestroy() throws IllegalStateException {
+        if (isStopped()) {
+            return;
+        }
         unexportServices();
         unreferServices();
 
@@ -206,7 +214,6 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
             }
             serviceRepository.destroy();
         }
-        moduleModel.destroy();
         onModuleStopped();
     }
 
@@ -219,7 +226,9 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
     private void onModuleStarted(CompletableFuture startFuture) {
         setStarted();
         logger.info(getIdentifier() + " has started.");
-        applicationDeployer.checkStarted(startFuture);
+        applicationDeployer.checkStarted();
+        // complete module start future after application state changed, fix #9012 ?
+        startFuture.complete(true);
     }
 
     private void onModuleStopping() {
@@ -345,7 +354,6 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
         } catch (Exception e) {
             logger.warn(getIdentifier() + " export services occurred an exception.");
         } finally {
-            executorRepository.shutdownServiceExportExecutor();
             logger.info(getIdentifier() + " export services finished.");
             asyncExportingFutures.clear();
         }
@@ -359,7 +367,6 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
         } catch (Exception e) {
             logger.warn(getIdentifier() + " refer services occurred an exception.");
         } finally {
-            executorRepository.shutdownServiceReferExecutor();
             logger.info(getIdentifier() + " refer services finished.");
             asyncReferringFutures.clear();
         }
@@ -390,10 +397,10 @@ public class DefaultModuleDeployer extends AbstractDeployer<ModuleModel> impleme
 
     private String getIdentifier() {
         if (identifier == null) {
-            if (moduleModel.getModelName() != null && !StringUtils.isEquals(moduleModel.getModelName(), moduleModel.getInternalName())) {
-                identifier = moduleModel.getModelName() + "[" + moduleModel.getInternalId() + "]";
-            } else {
-                identifier = "Dubbo Module" + "[" + moduleModel.getInternalId() + "]";
+            identifier = "Dubbo module[" + moduleModel.getInternalId() + "]";
+            if (moduleModel.getModelName() != null
+                && !StringUtils.isEquals(moduleModel.getModelName(), moduleModel.getInternalName())) {
+                identifier += "(" + moduleModel.getModelName() + ")";
             }
         }
         return identifier;

dubbo-config/dubbo-config-api/src/main/java/org/apache/dubbo/config/deploy/FrameworkModelCleaner.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dubbo.config.deploy;
+
+import org.apache.dubbo.common.extension.ExtensionLoader;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.rpc.Protocol;
+import org.apache.dubbo.rpc.model.FrameworkModel;
+import org.apache.dubbo.rpc.model.ScopeModelDestroyListener;
+
+/**
+ * A cleaner to release resources of framework model
+ */
+public class FrameworkModelCleaner implements ScopeModelDestroyListener<FrameworkModel> {
+
+    private static final Logger logger = LoggerFactory.getLogger(FrameworkModelCleaner.class);
+
+    @Override
+    public void onDestroy(FrameworkModel frameworkModel) {
+        destroyFrameworkResources(frameworkModel);
+    }
+
+    /**
+     * Destroy all framework resources.
+     */
+    private void destroyFrameworkResources(FrameworkModel frameworkModel) {
+        // destroy protocol in framework scope
+        destroyProtocols(frameworkModel);
+    }
+
+    /**
+     * Destroy all the protocols.
+     */
+    private void destroyProtocols(FrameworkModel frameworkModel) {
+        ExtensionLoader<Protocol> loader = frameworkModel.getExtensionLoader(Protocol.class);
+        for (String protocolName : loader.getLoadedExtensions()) {
+            try {
+                Protocol protocol = loader.getLoadedExtension(protocolName);
+                if (protocol != null) {
+                    protocol.destroy();
+                }
+            } catch (Throwable t) {
+                logger.warn(t.getMessage(), t);
+            }
+        }
+    }
+
+
+}

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractMethodConfigTest.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.config;
 
+import org.apache.dubbo.config.bootstrap.DubboBootstrap;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 
 import java.util.HashMap;
@@ -28,6 +30,12 @@ import static org.hamcrest.Matchers.isEmptyOrNullString;
 import static org.hamcrest.Matchers.sameInstance;
 
 public class AbstractMethodConfigTest {
+
+    @AfterAll
+    public static void afterAll() {
+        DubboBootstrap.reset();
+    }
+
     @Test
     public void testTimeout() throws Exception {
         MethodConfig methodConfig = new MethodConfig();

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/AbstractReferenceConfigTest.java

@@ -24,6 +24,8 @@ import org.apache.dubbo.rpc.cluster.RouterFactory;
 import org.apache.dubbo.rpc.cluster.router.condition.ConditionRouterFactory;
 import org.apache.dubbo.rpc.cluster.router.condition.config.AppRouterFactory;
 import org.apache.dubbo.rpc.cluster.router.tag.TagRouterFactory;
+import org.apache.dubbo.rpc.model.FrameworkModel;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -47,6 +49,11 @@ import static org.mockito.Mockito.when;
 
 public class AbstractReferenceConfigTest {
 
+    @AfterAll
+    public static void afterAll() throws Exception {
+        FrameworkModel.destroyAll();
+    }
+
     @Test
     public void testCheck() throws Exception {
         ReferenceConfig referenceConfig = new ReferenceConfig();

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ConfigCenterConfigTest.java

@@ -91,6 +91,7 @@ public class ConfigCenterConfigTest {
                         .initialize();
             } catch (Exception e) {
                 // ignore
+                e.printStackTrace();
             }
 
             Collection<ConfigCenterConfig> configCenters = ApplicationModel.defaultModel().getApplicationConfigManager().getConfigCenters();
@@ -100,6 +101,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             SysProps.clear();
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -128,6 +130,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             SysProps.clear();
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -155,6 +158,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             SysProps.clear();
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -183,6 +187,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             ApplicationModel.defaultModel().getModelEnvironment().getPropertiesConfiguration().refresh();
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -211,6 +216,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             SysProps.clear();
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -240,6 +246,7 @@ public class ConfigCenterConfigTest {
             Assertions.assertEquals(false, configCenter.isCheck());
         } finally {
             ApplicationModel.defaultModel().getModelEnvironment().getPropertiesConfiguration().refresh();
+            DubboBootstrap.getInstance().stop();
         }
     }
 

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProtocolConfigTest.java

@@ -20,6 +20,7 @@ package org.apache.dubbo.config;
 import org.apache.dubbo.common.utils.NetUtils;
 import org.apache.dubbo.config.bootstrap.DubboBootstrap;
 import org.apache.dubbo.config.context.ConfigManager;
+import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -31,7 +32,9 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.*;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.is;
 
 public class ProtocolConfigTest {
 
@@ -45,6 +48,11 @@ public class ProtocolConfigTest {
         SysProps.clear();
     }
 
+    @AfterAll
+    public static void afterAll() {
+        DubboBootstrap.reset();
+    }
+
     @Test
     public void testName() throws Exception {
         ProtocolConfig protocol = new ProtocolConfig();
@@ -245,6 +253,7 @@ public class ProtocolConfigTest {
             Assertions.assertEquals("rest", protocolConfig.getName());
             Assertions.assertEquals(port, protocolConfig.getPort());
         } finally {
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -265,6 +274,7 @@ public class ProtocolConfigTest {
             Assertions.assertEquals("rest", protocolConfig.getName());
             Assertions.assertEquals(port, protocolConfig.getPort());
         } finally {
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -287,6 +297,7 @@ public class ProtocolConfigTest {
             Assertions.assertEquals("rest", protocolConfig.getName());
             Assertions.assertEquals(port, protocolConfig.getPort());
         } finally {
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -314,6 +325,7 @@ public class ProtocolConfigTest {
             Assertions.assertEquals("rest", protocol.getName());
             Assertions.assertEquals(port1, protocol.getPort());
         } finally {
+            DubboBootstrap.getInstance().stop();
         }
     }
 
@@ -340,6 +352,7 @@ public class ProtocolConfigTest {
             Assertions.assertEquals("rest", protocol.getName());
             Assertions.assertEquals(port1, protocol.getPort());
         } finally {
+            DubboBootstrap.getInstance().stop();
         }
     }
 

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/ProviderConfigTest.java

@@ -31,6 +31,7 @@ import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 
 public class ProviderConfigTest {
+
     @Test
     public void testProtocol() throws Exception {
         ProviderConfig provider = new ProviderConfig();

dubbo-config/dubbo-config-api/src/test/java/org/apache/dubbo/config/bootstrap/DubboBootstrapMultiInstanceTest.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.config.bootstrap;
 
+import org.apache.dubbo.common.deploy.ApplicationDeployer;
+import org.apache.dubbo.common.deploy.DeployState;
 import org.apache.dubbo.common.deploy.ModuleDeployer;
 import org.apache.dubbo.common.utils.NetUtils;
 import org.apache.dubbo.common.utils.StringUtils;
@@ -34,6 +36,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.model.FrameworkServiceRepository;
 import org.apache.dubbo.rpc.model.ModuleModel;
+import org.apache.dubbo.test.check.DubboTestChecker;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -41,9 +44,12 @@ import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
-import static org.apache.dubbo.metadata.MetadataConstants.METADATA_PUBLISH_DELAY_KEY;
+import static org.apache.dubbo.remoting.Constants.EVENT_LOOP_BOSS_POOL_NAME;
 
 public class DubboBootstrapMultiInstanceTest {
 
@@ -51,8 +57,12 @@ public class DubboBootstrapMultiInstanceTest {
 
     private static RegistryConfig registryConfig;
 
+    private static DubboTestChecker testChecker;
+    private static String testClassName;
+
     @BeforeAll
     public static void beforeAll() {
+        FrameworkModel.destroyAll();
         registryCenter = new ZookeeperSingleRegistryCenter(NetUtils.getAvailablePort());
         registryCenter.startup();
         RegistryCenter.Instance instance = registryCenter.getRegistryCenterInstance().get(0);
@@ -60,11 +70,45 @@ public class DubboBootstrapMultiInstanceTest {
             instance.getType(),
             instance.getHostname(),
             instance.getPort()));
+
+        // pre-check threads
+        //precheckUnclosedThreads();
     }
 
     @AfterAll
-    public static void afterAll() {
+    public static void afterAll() throws Exception {
         registryCenter.shutdown();
+        FrameworkModel.destroyAll();
+
+        // check threads
+        //checkUnclosedThreads();
+    }
+
+    private static Map<Thread, StackTraceElement[]> precheckUnclosedThreads() throws IOException {
+        // create a special DubboTestChecker
+        if (testChecker == null) {
+            testChecker = new DubboTestChecker();
+            testChecker.init(null);
+            testClassName = DubboBootstrapMultiInstanceTest.class.getName();
+        }
+        return testChecker.checkUnclosedThreads(testClassName, 0);
+    }
+
+    private static void checkUnclosedThreads() {
+        Map<Thread, StackTraceElement[]> unclosedThreadMap = testChecker.checkUnclosedThreads(testClassName, 3000);
+        if (unclosedThreadMap.size() > 0) {
+            String str = getStackTraceString(unclosedThreadMap);
+            Assertions.fail("Found unclosed threads: " + unclosedThreadMap.size()+"\n" + str);
+        }
+    }
+
+    private static String getStackTraceString(Map<Thread, StackTraceElement[]> unclosedThreadMap) {
+        StringBuilder sb = new StringBuilder();
+        for (Thread thread : unclosedThreadMap.keySet()) {
+            sb.append(DubboTestChecker.getFullStacktrace(thread, unclosedThreadMap.get(thread)));
+            sb.append("\n");
+        }
+        return sb.toString();
     }
 
     @BeforeEach
@@ -173,7 +217,7 @@ public class DubboBootstrapMultiInstanceTest {
     @Test
     public void testMultiModuleApplication() throws InterruptedException {
 
-        SysProps.setProperty(METADATA_PUBLISH_DELAY_KEY, "1");
+        //SysProps.setProperty(METADATA_PUBLISH_DELAY_KEY, "100");
         String version1 = "1.0";
         String version2 = "2.0";
         String version3 = "3.0";
@@ -224,7 +268,7 @@ public class DubboBootstrapMultiInstanceTest {
 
             providerBootstrap.start();
 
-            Thread.sleep(100);
+            //Thread.sleep(200);
 
             // consumer app
             consumerBootstrap = DubboBootstrap.newInstance();
@@ -257,7 +301,134 @@ public class DubboBootstrapMultiInstanceTest {
                 consumerBootstrap.destroy();
             }
         }
+    }
 
+    @Test
+    public void testMultiProviderApplicationsStopOneByOne() {
+
+        String version1 = "1.0";
+        String version2 = "2.0";
+
+        DubboBootstrap providerBootstrap1 = null;
+        DubboBootstrap providerBootstrap2 = null;
+        ZookeeperSingleRegistryCenter registryCenter2 = null;
+
+        try {
+
+            // save threads before provider app 1
+            Map<Thread, StackTraceElement[]> stackTraces0 = Thread.getAllStackTraces();
+
+            // start provider app 1
+            ServiceConfig serviceConfig1 = new ServiceConfig();
+            serviceConfig1.setInterface(DemoService.class);
+            serviceConfig1.setRef(new DemoServiceImpl());
+            serviceConfig1.setVersion(version1);
+
+            ProtocolConfig protocolConfig1 = new ProtocolConfig("dubbo", NetUtils.getAvailablePort());
+
+            providerBootstrap1 = DubboBootstrap.getInstance();
+            providerBootstrap1.application("provider1")
+                .registry(registryConfig)
+                .service(serviceConfig1)
+                .protocol(protocolConfig1)
+                .start();
+
+            // save threads of provider app 1
+            Map<Thread, StackTraceElement[]> lastAllThreadStackTraces = Thread.getAllStackTraces();
+            Map<Thread, StackTraceElement[]> stackTraces1 = findNewThreads(lastAllThreadStackTraces, stackTraces0);
+            Assertions.assertTrue(stackTraces1.size() > 0, "Get threads of provider app 1 failed");
+
+            // start zk server 2
+            registryCenter2 = new ZookeeperSingleRegistryCenter(NetUtils.getAvailablePort());
+            registryCenter2.startup();
+            RegistryCenter.Instance instance = registryCenter2.getRegistryCenterInstance().get(0);
+            RegistryConfig registryConfig2 = new RegistryConfig(String.format("%s://%s:%s",
+                instance.getType(),
+                instance.getHostname(),
+                instance.getPort()));
+
+            // start provider app 2 use a difference zk server 2
+            ServiceConfig serviceConfig2 = new ServiceConfig();
+            serviceConfig2.setInterface(DemoService.class);
+            serviceConfig2.setRef(new DemoServiceImpl());
+            serviceConfig2.setVersion(version2);
+
+            ProtocolConfig protocolConfig2 = new ProtocolConfig("dubbo", NetUtils.getAvailablePort());
+
+            providerBootstrap2 = DubboBootstrap.newInstance();
+            providerBootstrap2.application("provider2")
+                .registry(registryConfig2)
+                .service(serviceConfig2)
+                .protocol(protocolConfig2)
+                .start();
+
+            // save threads of provider app 2
+            Map<Thread, StackTraceElement[]> stackTraces2 = findNewThreads(Thread.getAllStackTraces(), stackTraces0);
+            Assertions.assertTrue(stackTraces2.size() > 0, "Get threads of provider app 2 failed");
+
+            // stop provider app 1 and check threads
+            providerBootstrap1.stop();
+
+            // TODO Remove ignore thread prefix of NettyServerBoss if supporting close protocol server only used by one application
+            // see org.apache.dubbo.config.deploy.DefaultApplicationDeployer.postDestroy
+            // NettyServer will close when all applications are shutdown, but not close if any application of the framework is alive, just ignore it currently
+            checkUnclosedThreadsOfApp(stackTraces1, "Found unclosed threads of app 1: ", new String[]{EVENT_LOOP_BOSS_POOL_NAME, "Dubbo-global-shared-handler"});
+
+
+            // stop provider app 2 and check threads
+            providerBootstrap2.stop();
+            // shutdown register center after dubbo application to avoid unregister services blocking
+            registryCenter2.shutdown();
+            checkUnclosedThreadsOfApp(stackTraces2, "Found unclosed threads of app 2: ", null);
+
+        } finally {
+            if (providerBootstrap1 != null) {
+                providerBootstrap1.stop();
+            }
+            if (providerBootstrap2 != null) {
+                providerBootstrap2.stop();
+            }
+            if (registryCenter2 != null) {
+                registryCenter2.shutdown();
+            }
+        }
+    }
+
+    private Map<Thread, StackTraceElement[]> findNewThreads(Map<Thread, StackTraceElement[]> newAllThreadMap, Map<Thread, StackTraceElement[]> prevThreadMap) {
+        Map<Thread, StackTraceElement[]> deltaThreadMap = new HashMap<>(newAllThreadMap);
+        deltaThreadMap.keySet().removeAll(prevThreadMap.keySet());
+        // expect deltaThreadMap not contains any elements of prevThreadMap
+        Assertions.assertFalse(deltaThreadMap.keySet().stream().filter(thread -> prevThreadMap.containsKey(thread)).findAny().isPresent());
+        return deltaThreadMap;
+    }
+
+    private void checkUnclosedThreadsOfApp(Map<Thread, StackTraceElement[]> stackTraces1, String msg, String[] ignoredThreadPrefixes) {
+        int waitTimeMs = 5000;
+        System.out.println("Wait "+waitTimeMs+"ms to check threads of app ...");
+        try {
+            Thread.sleep(waitTimeMs);
+        } catch (InterruptedException e) {
+        }
+        HashMap<Thread, StackTraceElement[]> unclosedThreadMap1 = new HashMap<>(stackTraces1);
+        unclosedThreadMap1.keySet().removeIf(thread -> !thread.isAlive());
+        if (ignoredThreadPrefixes!= null && ignoredThreadPrefixes.length > 0) {
+            unclosedThreadMap1.keySet().removeIf(thread -> isIgnoredThread(thread.getName(), ignoredThreadPrefixes));
+        }
+        if (unclosedThreadMap1.size() > 0) {
+            String str = getStackTraceString(unclosedThreadMap1);
+            Assertions.fail(msg + unclosedThreadMap1.size()+"\n" + str);
+        }
+    }
+
+    private boolean isIgnoredThread(String name, String[] ignoredThreadPrefixes) {
+        if (ignoredThreadPrefixes!= null && ignoredThreadPrefixes.length > 0) {
+            for (String prefix : ignoredThreadPrefixes) {
+                if (name.startsWith(prefix)) {
+                    return true;
+                }
+            }
+        }
+        return false;
     }
 
     @Test
@@ -402,6 +573,142 @@ public class DubboBootstrapMultiInstanceTest {
         }
     }
 
+    @Test
+    public void testBothStartByModuleAndByApplication() throws Exception {
+        String version1 = "1.0";
+        String version2 = "2.0";
+        String version3 = "3.0";
+
+        String serviceKey1 = DemoService.class.getName() + ":" + version1;
+        String serviceKey2 = DemoService.class.getName() + ":" + version2;
+        String serviceKey3 = DemoService.class.getName() + ":" + version3;
+
+        // provider app
+        DubboBootstrap providerBootstrap = null;
+        try {
+            providerBootstrap = DubboBootstrap.newInstance();
+
+            ServiceConfig serviceConfig1 = new ServiceConfig();
+            serviceConfig1.setInterface(DemoService.class);
+            serviceConfig1.setRef(new DemoServiceImpl());
+            serviceConfig1.setVersion(version1);
+
+            //provider module 1
+            providerBootstrap
+                .application("provider-app")
+                .registry(registryConfig)
+                .protocol(new ProtocolConfig("dubbo", -1))
+                .service(builder -> builder
+                    .interfaceClass(Greeting.class)
+                    .ref(new GreetingLocal2()))
+                .newModule()
+                .service(serviceConfig1)
+                .endModule();
+
+            // 1. start module1
+            ModuleDeployer moduleDeployer1 = serviceConfig1.getScopeModel().getDeployer();
+            moduleDeployer1.start().get();
+            Assertions.assertEquals(DeployState.STARTED, moduleDeployer1.getState());
+
+            ApplicationModel applicationModel = providerBootstrap.getApplicationModel();
+            ApplicationDeployer applicationDeployer = applicationModel.getDeployer();
+            Assertions.assertEquals(DeployState.STARTING, applicationDeployer.getState());
+            ModuleModel defaultModule = applicationModel.getDefaultModule();
+            Assertions.assertEquals(DeployState.PENDING, defaultModule.getDeployer().getState());
+
+            // 2. start application after module1 is started
+            providerBootstrap.start();
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(DeployState.STARTED, defaultModule.getDeployer().getState());
+            
+            // 3. add module2 and re-start application
+            ServiceConfig serviceConfig2 = new ServiceConfig();
+            serviceConfig2.setInterface(DemoService.class);
+            serviceConfig2.setRef(new DemoServiceImpl());
+            serviceConfig2.setVersion(version2);
+            ModuleModel moduleModel2 = providerBootstrap.newModule()
+                .service(serviceConfig2)
+                .getModuleModel();
+            providerBootstrap.start();
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(DeployState.STARTED, moduleModel2.getDeployer().getState());
+            
+            // 4. add module3 and start module3
+            ServiceConfig serviceConfig3 = new ServiceConfig();
+            serviceConfig3.setInterface(DemoService.class);
+            serviceConfig3.setRef(new DemoServiceImpl());
+            serviceConfig3.setVersion(version3);
+            ModuleModel moduleModel3 = providerBootstrap.newModule()
+                .service(serviceConfig3)
+                .getModuleModel();
+            moduleModel3.getDeployer().start().get();
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(DeployState.STARTED, moduleModel3.getDeployer().getState());
+
+        } finally {
+            if (providerBootstrap != null) {
+                providerBootstrap.stop();
+            }
+        }
+    }
+
+
+    @Test
+    public void testBothStartByModuleAndByApplication2() throws Exception {
+        String version1 = "1.0";
+        String version2 = "2.0";
+        String version3 = "3.0";
+
+        String serviceKey1 = DemoService.class.getName() + ":" + version1;
+        String serviceKey2 = DemoService.class.getName() + ":" + version2;
+        String serviceKey3 = DemoService.class.getName() + ":" + version3;
+
+        // provider app
+        DubboBootstrap providerBootstrap = null;
+        try {
+            providerBootstrap = DubboBootstrap.newInstance();
+
+            ServiceConfig serviceConfig1 = new ServiceConfig();
+            serviceConfig1.setInterface(DemoService.class);
+            serviceConfig1.setRef(new DemoServiceImpl());
+            serviceConfig1.setVersion(version1);
+
+            //provider module 1
+            providerBootstrap
+                .application("provider-app")
+                .registry(registryConfig)
+                .protocol(new ProtocolConfig("dubbo", -1))
+                .service(builder -> builder
+                    .interfaceClass(Greeting.class)
+                    .ref(new GreetingLocal2()))
+                .newModule()
+                .service(serviceConfig1)
+                .endModule();
+
+            // 1. start module1 but no wait
+            ModuleDeployer moduleDeployer1 = serviceConfig1.getScopeModel().getDeployer();
+            moduleDeployer1.start();
+            Assertions.assertEquals(DeployState.STARTING, moduleDeployer1.getState());
+
+            ApplicationModel applicationModel = providerBootstrap.getApplicationModel();
+            ApplicationDeployer applicationDeployer = applicationModel.getDeployer();
+            Assertions.assertEquals(DeployState.STARTING, applicationDeployer.getState());
+            ModuleModel defaultModule = applicationModel.getDefaultModule();
+            Assertions.assertEquals(DeployState.PENDING, defaultModule.getDeployer().getState());
+
+            // 2. start application after module1 is starting
+            providerBootstrap.start();
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(DeployState.STARTED, moduleDeployer1.getState());
+            Assertions.assertEquals(DeployState.STARTED, defaultModule.getDeployer().getState());
+
+        } finally {
+            if (providerBootstrap != null) {
+                providerBootstrap.stop();
+            }
+        }
+    }
+
 
     private DubboBootstrap configConsumerApp(DubboBootstrap dubboBootstrap) {
         ReferenceConfig<DemoService> referenceConfig = new ReferenceConfig<>();

dubbo-config/dubbo-config-spring/src/main/java/org/apache/dubbo/config/spring/context/DubboDeployApplicationListener.java

@@ -36,7 +36,7 @@ import org.springframework.context.event.ContextClosedEvent;
 import org.springframework.context.event.ContextRefreshedEvent;
 import org.springframework.core.Ordered;
 
-import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 
 /**
  * An ApplicationListener to control Dubbo application.
@@ -105,7 +105,7 @@ public class DubboDeployApplicationListener implements ApplicationListener<Appli
         ModuleDeployer deployer = moduleModel.getDeployer();
         Assert.notNull(deployer, "Module deployer is null");
         // start module
-        CompletableFuture future = deployer.start();
+        Future future = deployer.start();
 
         // if the module does not start in background, await finish
         if (!deployer.isBackground()) {

dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/KeepRunningOnSpringClosedTest.java

@@ -16,7 +16,9 @@
  */
 package org.apache.dubbo.config.spring;
 
+import org.apache.dubbo.common.deploy.ApplicationDeployer;
 import org.apache.dubbo.common.deploy.DeployState;
+import org.apache.dubbo.common.deploy.ModuleDeployer;
 import org.apache.dubbo.config.bootstrap.DubboBootstrap;
 import org.apache.dubbo.config.spring.context.DubboSpringInitCustomizerHolder;
 import org.apache.dubbo.rpc.model.ModuleModel;
@@ -24,46 +26,52 @@ import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
-public class ShutdownHookTest {
+public class KeepRunningOnSpringClosedTest {
 
     @Test
-    public void testDisableShutdownHook() throws InterruptedException {
+    public void test(){
 
         // set KeepRunningOnSpringClosed flag for next spring context
         DubboSpringInitCustomizerHolder.get().addCustomizer(context-> {
             context.setKeepRunningOnSpringClosed(true);
         });
 
+        ClassPathXmlApplicationContext providerContext = null;
         try {
-            ClassPathXmlApplicationContext providerContext;
             String resourcePath = "org/apache/dubbo/config/spring";
             providerContext = new ClassPathXmlApplicationContext(
                 resourcePath + "/demo-provider.xml",
                 resourcePath + "/demo-provider-properties.xml");
             providerContext.start();
 
-            DubboStateListener listener = providerContext.getBean(DubboStateListener.class);
-            for (int i = 0; i < 10; i++) {
-                if (DeployState.STARTED.equals(listener.getState())) {
-                    break;
-                }
-                Thread.sleep(100);
-            }
+            // Expect 1: dubbo application state is STARTED after spring context start finish.
+            // No need check and wait
+
+            DubboStateListener dubboStateListener = providerContext.getBean(DubboStateListener.class);
+            Assertions.assertEquals(DeployState.STARTED, dubboStateListener.getState());
 
             ModuleModel moduleModel = providerContext.getBean(ModuleModel.class);
-            Assertions.assertTrue(moduleModel.getDeployer().isStarted());
-            Assertions.assertEquals(true, DubboBootstrap.getInstance().isStarted());
-            Assertions.assertEquals(false, DubboBootstrap.getInstance().isStopped());
+            ModuleDeployer moduleDeployer = moduleModel.getDeployer();
+            Assertions.assertTrue(moduleDeployer.isStarted());
+
+            ApplicationDeployer applicationDeployer = moduleModel.getApplicationModel().getDeployer();
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(true, applicationDeployer.isStarted());
+            Assertions.assertEquals(false, applicationDeployer.isStopped());
 
             // close spring context
             providerContext.close();
 
-            // expect dubbo bootstrap will not be destroyed after closing spring context
-            Assertions.assertEquals(true, DubboBootstrap.getInstance().isStarted());
-            Assertions.assertEquals(false, DubboBootstrap.getInstance().isStopped());
+            // Expect 2: dubbo application will not be destroyed after closing spring context cause setKeepRunningOnSpringClosed(true)
+            Assertions.assertEquals(DeployState.STARTED, applicationDeployer.getState());
+            Assertions.assertEquals(true, applicationDeployer.isStarted());
+            Assertions.assertEquals(false, applicationDeployer.isStopped());
         } finally {
             DubboBootstrap.getInstance().stop();
             SysProps.clear();
+            if (providerContext != null) {
+                providerContext.close();
+            }
         }
     }
 }

dubbo-config/dubbo-config-spring/src/test/java/org/apache/dubbo/config/spring/reference/javaconfig/JavaConfigReferenceBeanTest.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dubbo.config.spring.reference.javaconfig;
 
+import com.sun.management.HotSpotDiagnosticMXBean;
 import org.apache.dubbo.config.annotation.DubboReference;
 import org.apache.dubbo.config.annotation.DubboService;
 import org.apache.dubbo.config.annotation.Reference;
@@ -42,8 +43,10 @@ import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.PropertySource;
 
+import javax.management.MBeanServer;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.lang.management.ManagementFactory;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -59,8 +62,33 @@ public class JavaConfigReferenceBeanTest {
     }
 
     @AfterAll
-    public static void afterAll() {
+    public static void afterAll() throws Exception {
         multipleRegistryCenter.shutdown();
+        multipleRegistryCenter = null;
+
+        //Thread.sleep(10000);
+        //dumpHeap("/Users/gongdewei/work/dump/dubbo/", true);
+    }
+
+    public static void dumpHeap(String dirpath, boolean live) throws Exception {
+
+        java.lang.management.RuntimeMXBean runtime =
+            java.lang.management.ManagementFactory.getRuntimeMXBean();
+        java.lang.reflect.Field jvm = runtime.getClass().getDeclaredField("jvm");
+        jvm.setAccessible(true);
+        sun.management.VMManagement mgmt =
+            (sun.management.VMManagement) jvm.get(runtime);
+        java.lang.reflect.Method pid_method =
+            mgmt.getClass().getDeclaredMethod("getProcessId");
+        pid_method.setAccessible(true);
+        int pid = (Integer) pid_method.invoke(mgmt);
+
+        MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+        HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(
+            server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class);
+        String filepath = dirpath + pid + ".hprof";
+        mxBean.dumpHeap(filepath, live);
+        System.out.println("Dump heap to file: " + filepath);
     }
 
     @BeforeEach
@@ -78,19 +106,21 @@ public class JavaConfigReferenceBeanTest {
         AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class,
             AnnotationAtFieldConfiguration.class);
 
-        Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
-        Assertions.assertEquals(2, helloServiceMap.size());
-        Assertions.assertNotNull(helloServiceMap.get("helloService"));
-        Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
+        try {
+            Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
+            Assertions.assertEquals(2, helloServiceMap.size());
+            Assertions.assertNotNull(helloServiceMap.get("helloService"));
+            Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
 
-        Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
-        Assertions.assertEquals(1, referenceBeanMap.size());
-        ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
-        Assertions.assertEquals("demo", referenceBean.getGroup());
-        Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
-        Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
-
-        context.close();
+            Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
+            Assertions.assertEquals(1, referenceBeanMap.size());
+            ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
+            Assertions.assertEquals("demo", referenceBean.getGroup());
+            Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
+            Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
+        } finally {
+            context.close();
+        }
     }
 
     @Test
@@ -122,19 +152,21 @@ public class JavaConfigReferenceBeanTest {
         AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class,
                 AnnotationBeanConfiguration.class);
 
-        Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
-        Assertions.assertEquals(2, helloServiceMap.size());
-        Assertions.assertNotNull(helloServiceMap.get("helloService"));
-        Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
+        try {
+            Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
+            Assertions.assertEquals(2, helloServiceMap.size());
+            Assertions.assertNotNull(helloServiceMap.get("helloService"));
+            Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
 
-        Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
-        Assertions.assertEquals(1, referenceBeanMap.size());
-        ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
-        Assertions.assertEquals("demo", referenceBean.getGroup());
-        Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
-        Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
-
-        context.close();
+            Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
+            Assertions.assertEquals(1, referenceBeanMap.size());
+            ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
+            Assertions.assertEquals("demo", referenceBean.getGroup());
+            Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
+            Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
+        } finally {
+            context.close();
+        }
     }
 
     @Test
@@ -142,33 +174,36 @@ public class JavaConfigReferenceBeanTest {
         AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class,
             GenericServiceAnnotationBeanConfiguration.class);
 
-        Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
-        Assertions.assertEquals(1, helloServiceMap.size());
-        Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
+        try {
+            Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
+            Assertions.assertEquals(1, helloServiceMap.size());
+            Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
 
-        Map<String, GenericService> genericServiceMap = context.getBeansOfType(GenericService.class);
-        Assertions.assertEquals(3, genericServiceMap.size());
-        Assertions.assertNotNull(genericServiceMap.get("genericHelloService"));
+            Map<String, GenericService> genericServiceMap = context.getBeansOfType(GenericService.class);
+            Assertions.assertEquals(3, genericServiceMap.size());
+            Assertions.assertNotNull(genericServiceMap.get("genericHelloService"));
 
-        Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
-        Assertions.assertEquals(2, referenceBeanMap.size());
+            Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
+            Assertions.assertEquals(2, referenceBeanMap.size());
 
-        ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService");
-        Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup());
-        Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass());
-        Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface());
+            ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService");
+            Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup());
+            Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass());
+            Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface());
 
-        ReferenceBean genericServiceWithoutInterfaceBean = referenceBeanMap.get("&genericServiceWithoutInterface");
-        Assertions.assertEquals("demo", genericServiceWithoutInterfaceBean.getGroup());
-        Assertions.assertEquals(GenericService.class, genericServiceWithoutInterfaceBean.getInterfaceClass());
-        Assertions.assertEquals("org.apache.dubbo.config.spring.api.LocalMissClass", genericServiceWithoutInterfaceBean.getServiceInterface());
+            ReferenceBean genericServiceWithoutInterfaceBean = referenceBeanMap.get("&genericServiceWithoutInterface");
+            Assertions.assertEquals("demo", genericServiceWithoutInterfaceBean.getGroup());
+            Assertions.assertEquals(GenericService.class, genericServiceWithoutInterfaceBean.getInterfaceClass());
+            Assertions.assertEquals("org.apache.dubbo.config.spring.api.LocalMissClass", genericServiceWithoutInterfaceBean.getServiceInterface());
 
-        GenericService genericServiceWithoutInterface = context.getBean("genericServiceWithoutInterface", GenericService.class);
-        Assertions.assertNotNull(genericServiceWithoutInterface);
-        Object sayHelloResult = genericServiceWithoutInterface.$invoke("sayHello", new String[]{"java.lang.String"}, new Object[]{"Dubbo"});
-        Assertions.assertEquals("Hello Dubbo", sayHelloResult);
+            GenericService genericServiceWithoutInterface = context.getBean("genericServiceWithoutInterface", GenericService.class);
+            Assertions.assertNotNull(genericServiceWithoutInterface);
+            Object sayHelloResult = genericServiceWithoutInterface.$invoke("sayHello", new String[]{"java.lang.String"}, new Object[]{"Dubbo"});
+            Assertions.assertEquals("Hello Dubbo", sayHelloResult);
+        } finally {
+            context.close();
+        }
 
-        context.close();
     }
 
     @Test
@@ -176,22 +211,24 @@ public class JavaConfigReferenceBeanTest {
         AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class,
                 ReferenceBeanConfiguration.class);
 
-        Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
-        Assertions.assertEquals(2, helloServiceMap.size());
-        Assertions.assertNotNull(helloServiceMap.get("helloService"));
-        Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
+        try {
+            Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
+            Assertions.assertEquals(2, helloServiceMap.size());
+            Assertions.assertNotNull(helloServiceMap.get("helloService"));
+            Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
 
-        Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
-        Assertions.assertEquals(2, referenceBeanMap.size());
-        ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
-        Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
-        Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
+            Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
+            Assertions.assertEquals(2, referenceBeanMap.size());
+            ReferenceBean referenceBean = referenceBeanMap.get("&helloService");
+            Assertions.assertEquals(HelloService.class, referenceBean.getInterfaceClass());
+            Assertions.assertEquals(HelloService.class.getName(), referenceBean.getServiceInterface());
 
-        ReferenceBean demoServiceReferenceBean = referenceBeanMap.get("&demoService");
-        Assertions.assertEquals(DemoService.class, demoServiceReferenceBean.getInterfaceClass());
-        Assertions.assertEquals(DemoService.class.getName(), demoServiceReferenceBean.getServiceInterface());
-
-        context.close();
+            ReferenceBean demoServiceReferenceBean = referenceBeanMap.get("&demoService");
+            Assertions.assertEquals(DemoService.class, demoServiceReferenceBean.getInterfaceClass());
+            Assertions.assertEquals(DemoService.class.getName(), demoServiceReferenceBean.getServiceInterface());
+        } finally {
+            context.close();
+        }
     }
 
     @Test
@@ -199,24 +236,26 @@ public class JavaConfigReferenceBeanTest {
         AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(CommonConfig.class,
             GenericServiceReferenceBeanConfiguration.class);
 
-        Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
-        Assertions.assertEquals(1, helloServiceMap.size());
-        Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
+        try {
+            Map<String, HelloService> helloServiceMap = context.getBeansOfType(HelloService.class);
+            Assertions.assertEquals(1, helloServiceMap.size());
+            Assertions.assertNotNull(helloServiceMap.get("helloServiceImpl"));
 
-        Map<String, GenericService> genericServiceMap = context.getBeansOfType(GenericService.class);
-        Assertions.assertEquals(2, genericServiceMap.size());
-        Assertions.assertNotNull(genericServiceMap.get("localMissClassGenericServiceImpl"));
-        Assertions.assertNotNull(genericServiceMap.get("genericHelloService"));
+            Map<String, GenericService> genericServiceMap = context.getBeansOfType(GenericService.class);
+            Assertions.assertEquals(2, genericServiceMap.size());
+            Assertions.assertNotNull(genericServiceMap.get("localMissClassGenericServiceImpl"));
+            Assertions.assertNotNull(genericServiceMap.get("genericHelloService"));
 
-        Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
-        Assertions.assertEquals(1, referenceBeanMap.size());
+            Map<String, ReferenceBean> referenceBeanMap = context.getBeansOfType(ReferenceBean.class);
+            Assertions.assertEquals(1, referenceBeanMap.size());
 
-        ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService");
-        Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup());
-        Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass());
-        Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface());
-
-        context.close();
+            ReferenceBean genericHelloServiceReferenceBean = referenceBeanMap.get("&genericHelloService");
+            Assertions.assertEquals("demo", genericHelloServiceReferenceBean.getGroup());
+            Assertions.assertEquals(GenericService.class, genericHelloServiceReferenceBean.getInterfaceClass());
+            Assertions.assertEquals(HelloService.class.getName(), genericHelloServiceReferenceBean.getServiceInterface());
+        } finally {
+            context.close();
+        }
     }
 
     @Test

dubbo-config/pom.xml

@@ -33,4 +33,13 @@
         <module>dubbo-config-api</module>
         <module>dubbo-config-spring</module>
     </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-configcenter/dubbo-configcenter-apollo/src/main/java/org/apache/dubbo/configcenter/support/apollo/ApolloDynamicConfiguration.java

@@ -16,15 +16,6 @@
  */
 package org.apache.dubbo.configcenter.support.apollo;
 
-import org.apache.dubbo.common.URL;
-import org.apache.dubbo.common.config.configcenter.ConfigChangeType;
-import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent;
-import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
-import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
-import org.apache.dubbo.common.logger.Logger;
-import org.apache.dubbo.common.logger.LoggerFactory;
-import org.apache.dubbo.common.utils.StringUtils;
-
 import com.ctrip.framework.apollo.Config;
 import com.ctrip.framework.apollo.ConfigChangeListener;
 import com.ctrip.framework.apollo.ConfigFile;
@@ -33,6 +24,14 @@ import com.ctrip.framework.apollo.core.enums.ConfigFileFormat;
 import com.ctrip.framework.apollo.enums.ConfigSourceType;
 import com.ctrip.framework.apollo.enums.PropertyChangeType;
 import com.ctrip.framework.apollo.model.ConfigChange;
+import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.config.configcenter.ConfigChangeType;
+import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent;
+import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
+import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.utils.StringUtils;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -113,6 +112,12 @@ public class ApolloDynamicConfiguration implements DynamicConfiguration {
         }
     }
 
+    // TODO release apollo
+//    @Override
+//    public void close() throws Exception {
+//        DynamicConfiguration.super.close();
+//    }
+
     private String getAddressWithProtocolPrefix(URL url) {
         String address = url.getBackupAddress();
         if (StringUtils.isNotEmpty(address)) {

dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosConfigServiceWrapper.java

@@ -65,6 +65,10 @@ public class NacosConfigServiceWrapper {
         return configService.removeConfig(handleInnerSymbol(dataId), handleInnerSymbol(group));
     }
 
+    public void shutdown() throws NacosException {
+        configService.shutDown();
+    }
+
     /**
      * see {@link com.alibaba.nacos.client.config.utils.ParamUtils#isValid(java.lang.String)}
      */

dubbo-configcenter/dubbo-configcenter-nacos/src/main/java/org/apache/dubbo/configcenter/support/nacos/NacosDynamicConfiguration.java

@@ -17,17 +17,6 @@
 
 package org.apache.dubbo.configcenter.support.nacos;
 
-import org.apache.dubbo.common.URL;
-import org.apache.dubbo.common.config.configcenter.ConfigChangeType;
-import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent;
-import org.apache.dubbo.common.config.configcenter.ConfigItem;
-import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
-import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
-import org.apache.dubbo.common.logger.Logger;
-import org.apache.dubbo.common.logger.LoggerFactory;
-import org.apache.dubbo.common.utils.MD5Utils;
-import org.apache.dubbo.common.utils.StringUtils;
-
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -38,6 +27,16 @@ import com.alibaba.nacos.api.config.listener.AbstractSharedListener;
 import com.alibaba.nacos.api.exception.NacosException;
 import com.alibaba.nacos.client.config.http.HttpAgent;
 import com.alibaba.nacos.common.http.HttpRestResult;
+import org.apache.dubbo.common.URL;
+import org.apache.dubbo.common.config.configcenter.ConfigChangeType;
+import org.apache.dubbo.common.config.configcenter.ConfigChangedEvent;
+import org.apache.dubbo.common.config.configcenter.ConfigItem;
+import org.apache.dubbo.common.config.configcenter.ConfigurationListener;
+import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.utils.MD5Utils;
+import org.apache.dubbo.common.utils.StringUtils;
 
 import java.lang.reflect.Field;
 import java.util.HashMap;
@@ -176,6 +175,11 @@ public class NacosDynamicConfiguration implements DynamicConfiguration {
         return configListener;
     }
 
+    @Override
+    public void close() throws Exception {
+        configService.shutdown();
+    }
+
     @Override
     public void addListener(String key, String group, ConfigurationListener listener) {
         String listenerKey = buildListenerKey(key, group);

dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfiguration.java

@@ -25,7 +25,6 @@ import org.apache.dubbo.common.utils.CollectionUtils;
 import org.apache.dubbo.common.utils.NamedThreadFactory;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperClient;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter;
-
 import org.apache.zookeeper.data.Stat;
 
 import java.util.Collection;
@@ -43,7 +42,7 @@ public class ZookeeperDynamicConfiguration extends TreePathDynamicConfiguration
     private Executor executor;
     // The final root path would be: /configRootPath/"config"
     private String rootPath;
-    private final ZookeeperClient zkClient;
+    private ZookeeperClient zkClient;
 
     private CacheListener cacheListener;
     private URL url;
@@ -83,7 +82,11 @@ public class ZookeeperDynamicConfiguration extends TreePathDynamicConfiguration
 
     @Override
     protected void doClose() throws Exception {
-        zkClient.close();
+        // zkClient is shared in framework, should not close it here
+        // zkClient.close();
+        // See: org.apache.dubbo.remoting.zookeeper.AbstractZookeeperTransporter#destroy()
+        // All zk clients is created and destroyed in ZookeeperTransporter.
+        zkClient = null;
     }
 
     @Override

dubbo-configcenter/dubbo-configcenter-zookeeper/src/main/java/org/apache/dubbo/configcenter/support/zookeeper/ZookeeperDynamicConfigurationFactory.java

@@ -21,6 +21,7 @@ import org.apache.dubbo.common.config.configcenter.AbstractDynamicConfigurationF
 import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
 import org.apache.dubbo.common.extension.DisableInject;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter;
+import org.apache.dubbo.rpc.model.ApplicationModel;
 
 /**
  *
@@ -29,8 +30,11 @@ public class ZookeeperDynamicConfigurationFactory extends AbstractDynamicConfigu
 
     private ZookeeperTransporter zookeeperTransporter;
 
-    public ZookeeperDynamicConfigurationFactory() {
-        this.zookeeperTransporter = ZookeeperTransporter.getExtension();
+    private ApplicationModel applicationModel;
+
+    public ZookeeperDynamicConfigurationFactory(ApplicationModel applicationModel) {
+        this.applicationModel = applicationModel;
+        this.zookeeperTransporter = ZookeeperTransporter.getExtension(applicationModel);
     }
 
     @DisableInject

dubbo-configcenter/pom.xml

@@ -35,4 +35,13 @@
         <module>dubbo-configcenter-apollo</module>
         <module>dubbo-configcenter-nacos</module>
     </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-filter/pom.xml

@@ -33,4 +33,13 @@
         <module>dubbo-filter-cache</module>
         <module>dubbo-filter-validation</module>
     </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/MetadataInfo.java

@@ -18,6 +18,8 @@ package org.apache.dubbo.metadata;
 
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.extension.ExtensionLoader;
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
 import org.apache.dubbo.common.url.component.URLParam;
 import org.apache.dubbo.common.utils.ArrayUtils;
 import org.apache.dubbo.common.utils.CollectionUtils;
@@ -51,6 +53,7 @@ import static org.apache.dubbo.rpc.Constants.INTERFACES;
 
 public class MetadataInfo implements Serializable {
     public static final MetadataInfo EMPTY = new MetadataInfo();
+    private static final Logger logger = LoggerFactory.getLogger(MetadataInfo.class);
 
     private String app;
     private String revision;
@@ -115,7 +118,13 @@ public class MetadataInfo implements Serializable {
             for (Map.Entry<String, ServiceInfo> entry : new TreeMap<>(services).entrySet()) {
                 sb.append(entry.getValue().toDescString());
             }
-            this.revision = RevisionResolver.calRevision(sb.toString());
+            String tempRevision = RevisionResolver.calRevision(sb.toString());
+            if (!StringUtils.isEquals(this.revision, tempRevision)) {
+                if (logger.isInfoEnabled()) {
+                    logger.info(String.format("metadata revision changed: %s -> %s, app: %s, services: %d", this.revision, tempRevision, this.app, this.services.size()));
+                }
+                this.revision = tempRevision;
+            }
         }
         return revision;
     }

dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory.java

@@ -27,4 +27,7 @@ public interface MetadataReportFactory {
 
     @Adaptive({"protocol"})
     MetadataReport getMetadataReport(URL url);
+
+    default void destroy() {
+    }
 }

dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReport.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.metadata.report.support;
 
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
@@ -30,9 +32,6 @@ import org.apache.dubbo.metadata.report.identifier.MetadataIdentifier;
 import org.apache.dubbo.metadata.report.identifier.ServiceMetadataIdentifier;
 import org.apache.dubbo.metadata.report.identifier.SubscriberMetadataIdentifier;
 
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -101,6 +100,7 @@ public abstract class AbstractMetadataReport implements MetadataReport {
     File file;
     private AtomicBoolean initialized = new AtomicBoolean(false);
     public MetadataReportRetry metadataReportRetry;
+    private ScheduledExecutorService reportTimerScheduler;
 
     public AbstractMetadataReport(URL reportServerURL) {
         setUrl(reportServerURL);
@@ -129,8 +129,8 @@ public abstract class AbstractMetadataReport implements MetadataReport {
             reportServerURL.getParameter(RETRY_PERIOD_KEY, DEFAULT_METADATA_REPORT_RETRY_PERIOD));
         // cycle report the data switch
         if (reportServerURL.getParameter(CYCLE_REPORT_KEY, DEFAULT_METADATA_REPORT_CYCLE_REPORT)) {
-            ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMetadataReportTimer", true));
-            scheduler.scheduleAtFixedRate(this::publishAll, calculateStartTime(), ONE_DAY_IN_MILLISECONDS, TimeUnit.MILLISECONDS);
+            reportTimerScheduler = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMetadataReportTimer", true));
+            reportTimerScheduler.scheduleAtFixedRate(this::publishAll, calculateStartTime(), ONE_DAY_IN_MILLISECONDS, TimeUnit.MILLISECONDS);
         }
     }
 
@@ -302,8 +302,12 @@ public abstract class AbstractMetadataReport implements MetadataReport {
         if (reportCacheExecutor != null) {
             reportCacheExecutor.shutdown();
         }
+        if (reportTimerScheduler != null) {
+            reportTimerScheduler.shutdown();
+        }
         if (metadataReportRetry != null) {
             metadataReportRetry.destroy();
+            metadataReportRetry = null;
         }
     }
 

dubbo-metadata/dubbo-metadata-api/src/main/java/org/apache/dubbo/metadata/report/support/AbstractMetadataReportFactory.java

@@ -37,14 +37,12 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac
     /**
      * The lock for the acquisition process of the registry
      */
-    private static final ReentrantLock LOCK = new ReentrantLock();
+    private final ReentrantLock lock = new ReentrantLock();
 
     /**
      * Registry Collection Map<metadataAddress, MetadataReport>
      */
-    private static final Map<String, MetadataReport> SERVICE_STORE_MAP = new ConcurrentHashMap<>();
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMetadataReportFactory.class);
+    private final Map<String, MetadataReport> serviceStoreMap = new ConcurrentHashMap<>();
 
     @Override
     public MetadataReport getMetadataReport(URL url) {
@@ -52,15 +50,15 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac
             .removeParameters(EXPORT_KEY, REFER_KEY);
         String key = url.toServiceString();
 
-        MetadataReport metadataReport = SERVICE_STORE_MAP.get(key);
+        MetadataReport metadataReport = serviceStoreMap.get(key);
         if (metadataReport != null) {
             return metadataReport;
         }
 
         // Lock the metadata access process to ensure a single instance of the metadata instance
-        LOCK.lock();
+        lock.lock();
         try {
-            metadataReport = SERVICE_STORE_MAP.get(key);
+            metadataReport = serviceStoreMap.get(key);
             if (metadataReport != null) {
                 return metadataReport;
             }
@@ -69,7 +67,7 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac
                 metadataReport = createMetadataReport(url);
             } catch (Exception e) {
                 if (!check) {
-                    LOGGER.warn("The metadata reporter failed to initialize", e);
+                    logger.warn("The metadata reporter failed to initialize", e);
                 } else {
                     throw e;
                 }
@@ -79,19 +77,20 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac
                 throw new IllegalStateException("Can not create metadata Report " + url);
             }
             if (metadataReport != null) {
-                SERVICE_STORE_MAP.put(key, metadataReport);
+                serviceStoreMap.put(key, metadataReport);
             }
             return metadataReport;
         } finally {
             // Release the lock
-            LOCK.unlock();
+            lock.unlock();
         }
     }
 
-    public static void destroy() {
-        LOCK.lock();
+    @Override
+    public void destroy() {
+        lock.lock();
         try {
-            for (MetadataReport metadataReport : SERVICE_STORE_MAP.values()) {
+            for (MetadataReport metadataReport : serviceStoreMap.values()) {
                 try{
                     metadataReport.destroy();
                 }catch (Throwable ignored){
@@ -99,11 +98,10 @@ public abstract class AbstractMetadataReportFactory implements MetadataReportFac
                     logger.warn(ignored.getMessage(),ignored);
                 }
             }
-            SERVICE_STORE_MAP.clear();
+            serviceStoreMap.clear();
         } finally {
-            LOCK.unlock();
+            lock.unlock();
         }
-
     }
 
     protected abstract MetadataReport createMetadataReport(URL url);

dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReport.java

@@ -16,6 +16,7 @@
  */
 package org.apache.dubbo.metadata.store.zookeeper;
 
+import com.google.gson.Gson;
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.config.configcenter.ConfigItem;
 import org.apache.dubbo.common.logger.Logger;
@@ -34,8 +35,6 @@ import org.apache.dubbo.remoting.zookeeper.DataListener;
 import org.apache.dubbo.remoting.zookeeper.EventType;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperClient;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter;
-
-import com.google.gson.Gson;
 import org.apache.zookeeper.data.Stat;
 
 import java.util.ArrayList;
@@ -59,7 +58,7 @@ public class ZookeeperMetadataReport extends AbstractMetadataReport {
 
     private final String root;
 
-    final ZookeeperClient zkClient;
+    ZookeeperClient zkClient;
 
     private Gson gson = new Gson();
 
@@ -188,6 +187,13 @@ public class ZookeeperMetadataReport extends AbstractMetadataReport {
         }
     }
 
+    @Override
+    public void destroy() {
+        super.destroy();
+        // release zk client reference, but should not close it
+        zkClient = null;
+    }
+
     private String buildPathKey(String group, String serviceKey) {
         return toRootDir() + group + PATH_SEPARATOR + serviceKey;
     }

dubbo-metadata/dubbo-metadata-report-zookeeper/src/main/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportFactory.java

@@ -21,6 +21,7 @@ import org.apache.dubbo.common.extension.DisableInject;
 import org.apache.dubbo.metadata.report.MetadataReport;
 import org.apache.dubbo.metadata.report.support.AbstractMetadataReportFactory;
 import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter;
+import org.apache.dubbo.rpc.model.ApplicationModel;
 
 /**
  * ZookeeperRegistryFactory.
@@ -29,8 +30,11 @@ public class ZookeeperMetadataReportFactory extends AbstractMetadataReportFactor
 
     private ZookeeperTransporter zookeeperTransporter;
 
-    public ZookeeperMetadataReportFactory() {
-        this.zookeeperTransporter = ZookeeperTransporter.getExtension();
+    private ApplicationModel applicationModel;
+
+    public ZookeeperMetadataReportFactory(ApplicationModel applicationModel) {
+        this.applicationModel = applicationModel;
+        this.zookeeperTransporter = ZookeeperTransporter.getExtension(applicationModel);
     }
 
     @DisableInject

dubbo-metadata/dubbo-metadata-report-zookeeper/src/test/java/org/apache/dubbo/metadata/store/zookeeper/ZookeeperMetadataReportTest.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.metadata.store.zookeeper;
 
+import com.google.gson.Gson;
+import org.apache.curator.test.TestingServer;
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.config.configcenter.ConfigItem;
 import org.apache.dubbo.common.utils.NetUtils;
@@ -27,15 +29,18 @@ import org.apache.dubbo.metadata.report.identifier.KeyTypeEnum;
 import org.apache.dubbo.metadata.report.identifier.MetadataIdentifier;
 import org.apache.dubbo.metadata.report.identifier.ServiceMetadataIdentifier;
 import org.apache.dubbo.metadata.report.identifier.SubscriberMetadataIdentifier;
-
-import com.google.gson.Gson;
-import org.apache.curator.test.TestingServer;
+import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-import java.util.*;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 
@@ -58,7 +63,7 @@ public class ZookeeperMetadataReportTest {
         this.zkServer = new TestingServer(zkServerPort, true);
         this.registryUrl = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort);
 
-        zookeeperMetadataReportFactory = new ZookeeperMetadataReportFactory();
+        zookeeperMetadataReportFactory = new ZookeeperMetadataReportFactory(ApplicationModel.defaultModel());
         this.zookeeperMetadataReport = (ZookeeperMetadataReport) zookeeperMetadataReportFactory.getMetadataReport(registryUrl);
     }
 

dubbo-metadata/pom.xml

@@ -36,4 +36,12 @@
         <module>dubbo-metadata-report-nacos</module>
     </modules>
 
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-metrics/pom.xml

@@ -33,4 +33,13 @@
     <properties>
         <skip_maven_deploy>false</skip_maven_deploy>
     </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-monitor/dubbo-monitor-default/src/main/java/org/apache/dubbo/monitor/dubbo/DubboMonitor.java

@@ -20,7 +20,6 @@ import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
 import org.apache.dubbo.common.utils.ExecutorUtil;
-import org.apache.dubbo.common.utils.NamedThreadFactory;
 import org.apache.dubbo.monitor.Monitor;
 import org.apache.dubbo.monitor.MonitorService;
 import org.apache.dubbo.rpc.Invoker;
@@ -29,7 +28,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
@@ -52,7 +50,7 @@ public class DubboMonitor implements Monitor {
     /**
      * The timer for sending statistics
      */
-    private final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(3, new NamedThreadFactory("DubboMonitorSendTimer", true));
+    private final ScheduledExecutorService scheduledExecutorService;
 
     /**
      * The future that can cancel the <b>scheduledExecutorService</b>
@@ -68,6 +66,7 @@ public class DubboMonitor implements Monitor {
     public DubboMonitor(Invoker<MonitorService> monitorInvoker, MonitorService monitorService) {
         this.monitorInvoker = monitorInvoker;
         this.monitorService = monitorService;
+        scheduledExecutorService = monitorInvoker.getUrl().getOrDefaultApplicationModel().getApplicationExecutorRepository().getSharedScheduledExecutor();
         // The time interval for timer <b>scheduledExecutorService</b> to send data
         final long monitorInterval = monitorInvoker.getUrl().getPositiveParameter("interval", 60000);
         // collect timer for collecting statistics data

dubbo-monitor/dubbo-monitor-default/src/test/resources/log4j.properties

@@ -0,0 +1,7 @@
+###set log levels###
+log4j.rootLogger=info, stdout
+###output to the console###
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d{dd/MM/yy HH:mm:ss:SSS z}] %t %5p %c{2}: %m%n

dubbo-monitor/pom.xml

@@ -33,4 +33,14 @@
         <module>dubbo-monitor-api</module>
         <module>dubbo-monitor-default</module>
     </modules>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
 </project>

dubbo-native/pom.xml

@@ -64,7 +64,12 @@
             <version>${project.parent.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
-
 </project>

dubbo-native/src/main/java/org/apache/dubbo/common/serialize/Serialization$Adaptive.java

@@ -21,14 +21,8 @@ public class Serialization$Adaptive implements org.apache.dubbo.common.serialize
 public byte getContentTypeId()  {
 throw new UnsupportedOperationException("The method public abstract byte org.apache.dubbo.common.serialize.Serialization.getContentTypeId() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!");
 }
-public org.apache.dubbo.common.serialize.ObjectOutput serialize(org.apache.dubbo.common.URL arg0, java.io.OutputStream arg1) throws java.io.IOException {
-if (arg0 == null) throw new IllegalArgumentException("url == null");
-org.apache.dubbo.common.URL url = arg0;
-String extName = url.getParameter("serialization", "hessian2");
-if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.common.serialize.Serialization) name from url (" + url.toString() + ") use keys([serialization])");
-ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.common.serialize.Serialization.class);
-org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName);
-return extension.serialize(arg0, arg1);
+public java.lang.String getContentType()  {
+throw new UnsupportedOperationException("The method public abstract java.lang.String org.apache.dubbo.common.serialize.Serialization.getContentType() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!");
 }
 public org.apache.dubbo.common.serialize.ObjectInput deserialize(org.apache.dubbo.common.URL arg0, java.io.InputStream arg1) throws java.io.IOException {
 if (arg0 == null) throw new IllegalArgumentException("url == null");
@@ -39,7 +33,13 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa
 org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName);
 return extension.deserialize(arg0, arg1);
 }
-public java.lang.String getContentType()  {
-throw new UnsupportedOperationException("The method public abstract java.lang.String org.apache.dubbo.common.serialize.Serialization.getContentType() of interface org.apache.dubbo.common.serialize.Serialization is not adaptive method!");
+public org.apache.dubbo.common.serialize.ObjectOutput serialize(org.apache.dubbo.common.URL arg0, java.io.OutputStream arg1) throws java.io.IOException {
+if (arg0 == null) throw new IllegalArgumentException("url == null");
+org.apache.dubbo.common.URL url = arg0;
+String extName = url.getParameter("serialization", "hessian2");
+if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.common.serialize.Serialization) name from url (" + url.toString() + ") use keys([serialization])");
+ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.common.serialize.Serialization.class);
+org.apache.dubbo.common.serialize.Serialization extension = (org.apache.dubbo.common.serialize.Serialization)scopeModel.getExtensionLoader(org.apache.dubbo.common.serialize.Serialization.class).getExtension(extName);
+return extension.serialize(arg0, arg1);
 }
 }

dubbo-native/src/main/java/org/apache/dubbo/metadata/report/MetadataReportFactory$Adaptive.java

@@ -27,4 +27,7 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa
 org.apache.dubbo.metadata.report.MetadataReportFactory extension = (org.apache.dubbo.metadata.report.MetadataReportFactory)scopeModel.getExtensionLoader(org.apache.dubbo.metadata.report.MetadataReportFactory.class).getExtension(extName);
 return extension.getMetadataReport(arg0);
 }
+public void destroy()  {
+throw new UnsupportedOperationException("The method public abstract void org.apache.dubbo.metadata.report.MetadataReportFactory.destroy() of interface org.apache.dubbo.metadata.report.MetadataReportFactory is not adaptive method!");
+}
 }

dubbo-native/src/main/java/org/apache/dubbo/remoting/zookeeper/ZookeeperTransporter$Adaptive.java

@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dubbo.remoting.zookeeper;
-import org.apache.dubbo.common.extension.ExtensionLoader;
-public class ZookeeperTransporter$Adaptive implements org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter {
-public org.apache.dubbo.remoting.zookeeper.ZookeeperClient connect(org.apache.dubbo.common.URL arg0)  {
-if (arg0 == null) throw new IllegalArgumentException("url == null");
-org.apache.dubbo.common.URL url = arg0;
-String extName = url.getParameter("client", url.getParameter("transporter", "curator"));
-if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter) name from url (" + url.toString() + ") use keys([client, transporter])");
-org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter extension = (org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter)ExtensionLoader.getExtensionLoader(org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter.class).getExtension(extName);
-return extension.connect(arg0);
-}
-}

dubbo-native/src/main/java/org/apache/dubbo/rpc/Protocol$Adaptive.java

@@ -18,6 +18,9 @@ package org.apache.dubbo.rpc;
 import org.apache.dubbo.rpc.model.ScopeModel;
 import org.apache.dubbo.rpc.model.ScopeModelUtil;
 public class Protocol$Adaptive implements org.apache.dubbo.rpc.Protocol {
+public int getDefaultPort()  {
+throw new UnsupportedOperationException("The method public abstract int org.apache.dubbo.rpc.Protocol.getDefaultPort() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!");
+}
 public org.apache.dubbo.rpc.Exporter export(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException {
 if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null");
 if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null");
@@ -37,13 +40,10 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa
 org.apache.dubbo.rpc.Protocol extension = (org.apache.dubbo.rpc.Protocol)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.Protocol.class).getExtension(extName);
 return extension.refer(arg0, arg1);
 }
-public java.util.List getServers()  {
-throw new UnsupportedOperationException("The method public default java.util.List org.apache.dubbo.rpc.Protocol.getServers() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!");
-}
 public void destroy()  {
 throw new UnsupportedOperationException("The method public abstract void org.apache.dubbo.rpc.Protocol.destroy() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!");
 }
-public int getDefaultPort()  {
-throw new UnsupportedOperationException("The method public abstract int org.apache.dubbo.rpc.Protocol.getDefaultPort() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!");
+public java.util.List getServers()  {
+throw new UnsupportedOperationException("The method public default java.util.List org.apache.dubbo.rpc.Protocol.getServers() of interface org.apache.dubbo.rpc.Protocol is not adaptive method!");
 }
 }

dubbo-native/src/main/java/org/apache/dubbo/rpc/ProxyFactory$Adaptive.java

@@ -18,14 +18,15 @@ package org.apache.dubbo.rpc;
 import org.apache.dubbo.rpc.model.ScopeModel;
 import org.apache.dubbo.rpc.model.ScopeModelUtil;
 public class ProxyFactory$Adaptive implements org.apache.dubbo.rpc.ProxyFactory {
-public org.apache.dubbo.rpc.Invoker getInvoker(java.lang.Object arg0, java.lang.Class arg1, org.apache.dubbo.common.URL arg2) throws org.apache.dubbo.rpc.RpcException {
-if (arg2 == null) throw new IllegalArgumentException("url == null");
-org.apache.dubbo.common.URL url = arg2;
+public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException {
+if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null");
+if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null");
+org.apache.dubbo.common.URL url = arg0.getUrl();
 String extName = url.getParameter("proxy", "javassist");
 if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.rpc.ProxyFactory) name from url (" + url.toString() + ") use keys([proxy])");
 ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.rpc.ProxyFactory.class);
 org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName);
-return extension.getInvoker(arg0, arg1, arg2);
+return extension.getProxy(arg0);
 }
 public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0, boolean arg1) throws org.apache.dubbo.rpc.RpcException {
 if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null");
@@ -37,14 +38,13 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa
 org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName);
 return extension.getProxy(arg0, arg1);
 }
-public java.lang.Object getProxy(org.apache.dubbo.rpc.Invoker arg0) throws org.apache.dubbo.rpc.RpcException {
-if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument == null");
-if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.Invoker argument getUrl() == null");
-org.apache.dubbo.common.URL url = arg0.getUrl();
+public org.apache.dubbo.rpc.Invoker getInvoker(java.lang.Object arg0, java.lang.Class arg1, org.apache.dubbo.common.URL arg2) throws org.apache.dubbo.rpc.RpcException {
+if (arg2 == null) throw new IllegalArgumentException("url == null");
+org.apache.dubbo.common.URL url = arg2;
 String extName = url.getParameter("proxy", "javassist");
 if(extName == null) throw new IllegalStateException("Failed to get extension (org.apache.dubbo.rpc.ProxyFactory) name from url (" + url.toString() + ") use keys([proxy])");
 ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apache.dubbo.rpc.ProxyFactory.class);
 org.apache.dubbo.rpc.ProxyFactory extension = (org.apache.dubbo.rpc.ProxyFactory)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.ProxyFactory.class).getExtension(extName);
-return extension.getProxy(arg0);
+return extension.getInvoker(arg0, arg1, arg2);
 }
 }

dubbo-native/src/main/java/org/apache/dubbo/rpc/cluster/Cluster$Adaptive.java

@@ -18,12 +18,6 @@ package org.apache.dubbo.rpc.cluster;
 import org.apache.dubbo.rpc.model.ScopeModel;
 import org.apache.dubbo.rpc.model.ScopeModelUtil;
 public class Cluster$Adaptive implements org.apache.dubbo.rpc.cluster.Cluster {
-public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1)  {
-throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!");
-}
-public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1, boolean arg2)  {
-throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String,boolean) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!");
-}
 public org.apache.dubbo.rpc.Invoker join(org.apache.dubbo.rpc.cluster.Directory arg0, boolean arg1) throws org.apache.dubbo.rpc.RpcException {
 if (arg0 == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.cluster.Directory argument == null");
 if (arg0.getUrl() == null) throw new IllegalArgumentException("org.apache.dubbo.rpc.cluster.Directory argument getUrl() == null");
@@ -34,4 +28,10 @@ ScopeModel scopeModel = ScopeModelUtil.getOrDefault(url.getScopeModel(), org.apa
 org.apache.dubbo.rpc.cluster.Cluster extension = (org.apache.dubbo.rpc.cluster.Cluster)scopeModel.getExtensionLoader(org.apache.dubbo.rpc.cluster.Cluster.class).getExtension(extName);
 return extension.join(arg0, arg1);
 }
+public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1)  {
+throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!");
+}
+public org.apache.dubbo.rpc.cluster.Cluster getCluster(org.apache.dubbo.rpc.model.ScopeModel arg0, java.lang.String arg1, boolean arg2)  {
+throw new UnsupportedOperationException("The method public static org.apache.dubbo.rpc.cluster.Cluster org.apache.dubbo.rpc.cluster.Cluster.getCluster(org.apache.dubbo.rpc.model.ScopeModel,java.lang.String,boolean) of interface org.apache.dubbo.rpc.cluster.Cluster is not adaptive method!");
+}
 }

dubbo-native/src/main/java/org/apache/dubbo/utils/CodeGenerator.java

@@ -17,7 +17,6 @@
 package org.apache.dubbo.utils;
 
 import org.apache.commons.io.FileUtils;
-
 import org.apache.dubbo.common.extension.Adaptive;
 import org.apache.dubbo.common.extension.AdaptiveClassCodeGenerator;
 import org.apache.dubbo.common.extension.SPI;
@@ -69,7 +68,7 @@ public class CodeGenerator {
                 value = "adaptive";
             }
             AdaptiveClassCodeGenerator codeGenerator = new AdaptiveClassCodeGenerator(it, value);
-            String code = codeGenerator.generate();
+            String code = codeGenerator.generate(true);
             System.out.println(code);
             System.out.println("-----:" + it.getPackage());
             try {

dubbo-plugin/dubbo-qos/src/main/java/org/apache/dubbo/qos/command/impl/ShutdownTelnet.java

@@ -17,7 +17,6 @@
 package org.apache.dubbo.qos.command.impl;
 
 import org.apache.dubbo.common.utils.StringUtils;
-import org.apache.dubbo.config.deploy.DefaultApplicationDeployer;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
 import org.apache.dubbo.qos.command.annotation.Cmd;
@@ -60,8 +59,7 @@ public class ShutdownTelnet implements BaseCommand {
         StringBuilder buf = new StringBuilder();
         List<ApplicationModel> applicationModels = frameworkModel.getApplicationModels();
         for (ApplicationModel applicationModel : new ArrayList<>(applicationModels)) {
-            DefaultApplicationDeployer deployer = applicationModel.getBeanFactory().getBean(DefaultApplicationDeployer.class);
-            deployer.destroy();
+            applicationModel.destroy();
         }
         // TODO change to ApplicationDeployer.destroy() or ApplicationModel.destroy()
 //        DubboShutdownHook.getDubboShutdownHook().unregister();

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ChangeTelnetTest.java

@@ -22,7 +22,6 @@ import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.qos.legacy.service.DemoService;
 import org.apache.dubbo.rpc.Invoker;
 import org.apache.dubbo.rpc.Protocol;
@@ -30,6 +29,7 @@ import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -40,7 +40,7 @@ import static org.mockito.Mockito.reset;
 
 public class ChangeTelnetTest {
     private final DefaultAttributeMap defaultAttributeMap = new DefaultAttributeMap();
-    private static final BaseCommand change = new ChangeTelnet(FrameworkModel.defaultModel());
+    private BaseCommand change;
 
     private Channel mockChannel;
     private CommandContext mockCommandContext;
@@ -48,12 +48,19 @@ public class ChangeTelnetTest {
 
     @AfterAll
     public static void tearDown() {
+        FrameworkModel.destroyAll();
+    }
 
+    @BeforeAll
+    public static void setUp() {
+        FrameworkModel.destroyAll();
     }
 
     @SuppressWarnings("unchecked")
     @BeforeEach
-    public void setUp() {
+    public void beforeEach() {
+        change = new ChangeTelnet(FrameworkModel.defaultModel());
+
         mockCommandContext = mock(CommandContext.class);
         mockChannel = mock(Channel.class);
         mockInvoker = mock(Invoker.class);
@@ -66,8 +73,8 @@ public class ChangeTelnetTest {
     }
 
     @AfterEach
-    public void after() {
-        ProtocolUtils.closeAll();
+    public void afterEach() {
+        FrameworkModel.destroyAll();
         reset(mockCommandContext, mockChannel, mockInvoker);
     }
 

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/CountTelnetTest.java

@@ -21,7 +21,6 @@ import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
 import org.apache.dubbo.qos.command.impl.channel.MockNettyChannel;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.qos.legacy.service.DemoService;
 import org.apache.dubbo.remoting.telnet.support.TelnetUtils;
 import org.apache.dubbo.rpc.Invoker;
@@ -44,7 +43,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 
 public class CountTelnetTest {
-    private static final BaseCommand count = new CountTelnet(FrameworkModel.defaultModel());
+    private BaseCommand count;
 
     private MockNettyChannel mockChannel;
     private Invoker<DemoService> mockInvoker;
@@ -56,6 +55,7 @@ public class CountTelnetTest {
 
     @BeforeEach
     public void setUp() {
+        count = new CountTelnet(FrameworkModel.defaultModel());
         latch = new CountDownLatch(2);
         mockInvoker = mock(Invoker.class);
         mockCommandContext = mock(CommandContext.class);
@@ -68,7 +68,7 @@ public class CountTelnetTest {
 
     @AfterEach
     public void tearDown() {
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
         mockChannel.close();
         reset(mockInvoker, mockCommandContext);
     }

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/InvokeTelnetTest.java

@@ -21,7 +21,6 @@ import io.netty.util.DefaultAttributeMap;
 import org.apache.dubbo.config.bootstrap.DubboBootstrap;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.qos.legacy.service.DemoService;
 import org.apache.dubbo.qos.legacy.service.DemoServiceImpl;
 import org.apache.dubbo.remoting.RemotingException;
@@ -64,7 +63,6 @@ public class InvokeTelnetTest {
 
     @AfterEach
     public void after() {
-        ProtocolUtils.closeAll();
         frameworkModel.destroy();
         reset(mockChannel, mockCommandContext);
     }

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PortTelnetTest.java

@@ -17,11 +17,9 @@
 package org.apache.dubbo.qos.command.impl;
 
 import org.apache.dubbo.common.URL;
-import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.common.utils.NetUtils;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.qos.legacy.service.DemoService;
 import org.apache.dubbo.remoting.RemotingException;
 import org.apache.dubbo.remoting.exchange.ExchangeClient;
@@ -41,7 +39,7 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.reset;
 
 public class PortTelnetTest {
-    private static final BaseCommand port = new PortTelnet(FrameworkModel.defaultModel());
+    private BaseCommand port;
 
     private Invoker<DemoService> mockInvoker;
     private CommandContext mockCommandContext;
@@ -51,17 +49,19 @@ public class PortTelnetTest {
     @SuppressWarnings("unchecked")
     @BeforeEach
     public void before() {
+        FrameworkModel frameworkModel = FrameworkModel.defaultModel();
+        port = new PortTelnet(frameworkModel);
         mockCommandContext = mock(CommandContext.class);
         mockInvoker = mock(Invoker.class);
         given(mockInvoker.getInterface()).willReturn(DemoService.class);
         given(mockInvoker.getUrl()).willReturn(URL.valueOf("dubbo://127.0.0.1:" + availablePort + "/demo"));
 
-        ExtensionLoader.getExtensionLoader(Protocol.class).getExtension(DubboProtocol.NAME).export(mockInvoker);
+        frameworkModel.getExtensionLoader(Protocol.class).getExtension(DubboProtocol.NAME).export(mockInvoker);
     }
 
     @AfterEach
-    public void after() {
-        ProtocolUtils.closeAll();
+    public void afterEach() {
+        FrameworkModel.destroyAll();
         reset(mockInvoker, mockCommandContext);
     }
 

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/PwdTelnetTest.java

@@ -16,13 +16,12 @@
  */
 package org.apache.dubbo.qos.command.impl;
 
-import org.apache.dubbo.qos.command.BaseCommand;
-import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
-import org.apache.dubbo.remoting.RemotingException;
-
 import io.netty.channel.Channel;
 import io.netty.util.DefaultAttributeMap;
+import org.apache.dubbo.qos.command.BaseCommand;
+import org.apache.dubbo.qos.command.CommandContext;
+import org.apache.dubbo.remoting.RemotingException;
+import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -49,7 +48,7 @@ public class PwdTelnetTest {
 
     @AfterEach
     public void tearDown() {
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
         mockChannel.close();
         reset(mockChannel, mockCommandContext);
     }

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/SelectTelnetTest.java

@@ -21,7 +21,6 @@ import io.netty.util.DefaultAttributeMap;
 import org.apache.dubbo.config.bootstrap.DubboBootstrap;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.qos.legacy.service.DemoService;
 import org.apache.dubbo.qos.legacy.service.DemoServiceImpl;
 import org.apache.dubbo.remoting.RemotingException;
@@ -44,17 +43,19 @@ import static org.mockito.Mockito.reset;
 
 public class SelectTelnetTest {
 
-    private static BaseCommand select = new SelectTelnet(FrameworkModel.defaultModel());
+    private BaseCommand select;
 
     private Channel mockChannel;
     private CommandContext mockCommandContext;
 
-    private final ModuleServiceRepository repository = ApplicationModel.defaultModel().getDefaultModule().getServiceRepository();
+    private ModuleServiceRepository repository;
     private final DefaultAttributeMap defaultAttributeMap = new DefaultAttributeMap();
     private List<Method> methods;
 
     @BeforeEach
     public void setup() {
+        repository = ApplicationModel.defaultModel().getDefaultModule().getServiceRepository();
+        select = new SelectTelnet(FrameworkModel.defaultModel());
         String methodName = "getPerson";
         methods = new ArrayList<>();
         for (Method method : DemoService.class.getMethods()) {
@@ -71,7 +72,7 @@ public class SelectTelnetTest {
 
     @AfterEach
     public void after() {
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
         reset(mockChannel, mockCommandContext);
     }
 

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/impl/ShutdownTelnetTest.java

@@ -19,7 +19,6 @@ package org.apache.dubbo.qos.command.impl;
 import io.netty.channel.Channel;
 import org.apache.dubbo.qos.command.BaseCommand;
 import org.apache.dubbo.qos.command.CommandContext;
-import org.apache.dubbo.qos.legacy.ProtocolUtils;
 import org.apache.dubbo.remoting.RemotingException;
 import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.junit.jupiter.api.AfterEach;
@@ -33,12 +32,13 @@ import static org.mockito.Mockito.reset;
 
 public class ShutdownTelnetTest {
 
-    private static final BaseCommand shutdown = new ShutdownTelnet(FrameworkModel.defaultModel());
+    private BaseCommand shutdown;
     private Channel mockChannel;
     private CommandContext mockCommandContext;
 
     @BeforeEach
     public void setUp() {
+        shutdown = new ShutdownTelnet(FrameworkModel.defaultModel());
         mockCommandContext = mock(CommandContext.class);
         mockChannel = mock(Channel.class);
         given(mockCommandContext.getRemote()).willReturn(mockChannel);
@@ -46,7 +46,7 @@ public class ShutdownTelnetTest {
 
     @AfterEach
     public void after() {
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
         reset(mockChannel, mockCommandContext);
     }
 

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/command/util/CommandHelperTest.java

@@ -53,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class CommandHelperTest {
     private CommandHelper commandHelper = new CommandHelper(FrameworkModel.defaultModel());
+
     @Test
     public void testHasCommand() throws Exception {
         assertTrue(commandHelper.hasCommand("greeting"));

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ChangeTelnetHandlerTest.java

@@ -24,8 +24,8 @@ import org.apache.dubbo.remoting.RemotingException;
 import org.apache.dubbo.remoting.telnet.TelnetHandler;
 import org.apache.dubbo.rpc.Invoker;
 import org.apache.dubbo.rpc.Protocol;
+import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol;
-
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -74,7 +74,7 @@ public class ChangeTelnetHandlerTest {
 
     @AfterEach
     public void after() {
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
         reset(mockChannel, mockInvoker);
     }
 

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/ProtocolUtils.java

@@ -20,25 +20,22 @@ import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.extension.ExtensionLoader;
 import org.apache.dubbo.rpc.Exporter;
 import org.apache.dubbo.rpc.Protocol;
-import org.apache.dubbo.rpc.ProtocolServer;
 import org.apache.dubbo.rpc.ProxyFactory;
+import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol;
 
-import java.util.Collection;
-
 /**
  * TODO Comment of ProtocolUtils
  */
 public class ProtocolUtils {
 
-    private static Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension();
-    private static ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension();
-
     public static <T> T refer(Class<T> type, String url) {
         return refer(type, URL.valueOf(url));
     }
 
     public static <T> T refer(Class<T> type, URL url) {
+        Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension();
+        ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension();
         return proxy.getProxy(protocol.refer(type, url));
     }
 
@@ -47,14 +44,14 @@ public class ProtocolUtils {
     }
 
     public static <T> Exporter<T> export(T instance, Class<T> type, URL url) {
+        Protocol protocol = ExtensionLoader.getExtensionLoader(Protocol.class).getAdaptiveExtension();
+        ProxyFactory proxy = ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension();
         return protocol.export(proxy.getInvoker(instance, type, url));
     }
 
     public static void closeAll() {
         DubboProtocol.getDubboProtocol().destroy();
-        Collection<ProtocolServer> servers = DubboProtocol.getDubboProtocol().getServers();
-        for (ProtocolServer server : servers) {
-            server.close();
-        }
+        ExtensionLoader.getExtensionLoader(Protocol.class).destroy();
+        FrameworkModel.destroyAll();
     }
 }

dubbo-plugin/dubbo-qos/src/test/java/org/apache/dubbo/qos/legacy/TraceTelnetHandlerTest.java

@@ -23,9 +23,9 @@ import org.apache.dubbo.remoting.Channel;
 import org.apache.dubbo.remoting.telnet.TelnetHandler;
 import org.apache.dubbo.rpc.Invoker;
 import org.apache.dubbo.rpc.Protocol;
+import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.apache.dubbo.rpc.protocol.dubbo.DubboProtocol;
 import org.apache.dubbo.rpc.protocol.dubbo.filter.TraceFilter;
-
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -58,7 +58,7 @@ public class TraceTelnetHandlerTest {
     @AfterEach
     public void tearDown() {
         reset(mockChannel, mockInvoker);
-        ProtocolUtils.closeAll();
+        FrameworkModel.destroyAll();
     }
 
     @Test

dubbo-plugin/pom.xml

@@ -36,4 +36,12 @@
         <skip_maven_deploy>false</skip_maven_deploy>
     </properties>
 
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.dubbo</groupId>
+            <artifactId>dubbo-test-check</artifactId>
+            <version>${project.parent.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
 </project>

dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/metadata/store/InMemoryWritableMetadataService.java

@@ -37,6 +37,7 @@ import org.apache.dubbo.rpc.model.ApplicationModel;
 import org.apache.dubbo.rpc.model.ScopeModelAware;
 import org.apache.dubbo.rpc.support.ProtocolUtils;
 
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -134,7 +135,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService,
     @Override
     public void setApplicationModel(ApplicationModel applicationModel) {
         this.applicationModel = applicationModel;
-        this.metadataPublishDelayTime = ConfigurationUtils.get(applicationModel, METADATA_PUBLISH_DELAY_KEY, DEFAULT_METADATA_PUBLISH_DELAY) * 100L;
+        this.metadataPublishDelayTime = ConfigurationUtils.get(applicationModel, METADATA_PUBLISH_DELAY_KEY, DEFAULT_METADATA_PUBLISH_DELAY);
     }
 
     @Override
@@ -196,6 +197,15 @@ public class InMemoryWritableMetadataService implements WritableMetadataService,
         }
     }
 
+    public void addMetadataInfo(String key, MetadataInfo metadataInfo) {
+        updateLock.readLock().lock();
+        try {
+            metadataInfos.put(key, metadataInfo);
+        } finally {
+            updateLock.readLock().unlock();
+        }
+    }
+
     @Override
     public boolean unexportURL(URL url) {
         if (MetadataService.class.getName().equals(url.getServiceInterface())) {
@@ -286,6 +296,9 @@ public class InMemoryWritableMetadataService implements WritableMetadataService,
                 return metadataInfo;
             }
         }
+        if (logger.isInfoEnabled()) {
+            logger.info("metadata not found for revision: " + revision);
+        }
         return null;
     }
 
@@ -324,7 +337,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService,
             metadataSemaphore.drainPermits();
             updateLock.writeLock().lock();
         } catch (InterruptedException e) {
-            if (!applicationModel.isStopping()) {
+            if (!applicationModel.isDestroyed()) {
                 logger.warn("metadata refresh thread has been interrupted unexpectedly while waiting for update.", e);
             }
         }
@@ -335,7 +348,7 @@ public class InMemoryWritableMetadataService implements WritableMetadataService,
     }
 
     public Map<String, MetadataInfo> getMetadataInfos() {
-        return metadataInfos;
+        return Collections.unmodifiableMap(metadataInfos);
     }
 
     void addMetaServiceURL(URL url) {

dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/client/migration/MigrationRuleListener.java

@@ -110,7 +110,7 @@ public class MigrationRuleListener implements RegistryProtocolListener, Configur
 
         String localRawRule = moduleModel.getModelEnvironment().getLocalMigrationRule();
         if (!StringUtils.isEmpty(localRawRule)) {
-            localRuleMigrationFuture = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("DubboMigrationRuleDelayWorker", true))
+            localRuleMigrationFuture = moduleModel.getApplicationModel().getApplicationExecutorRepository().getSharedScheduledExecutor()
                 .schedule(() -> {
                     if (this.rawRule.equals(INIT)) {
                         this.process(new ConfigChangedEvent(null, null, localRawRule));

dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/integration/RegistryProtocol.java

@@ -21,6 +21,7 @@ import org.apache.dubbo.common.config.ConfigurationUtils;
 import org.apache.dubbo.common.config.configcenter.DynamicConfiguration;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.threadpool.manager.ExecutorRepository;
 import org.apache.dubbo.common.timer.HashedWheelTimer;
 import org.apache.dubbo.common.url.component.ServiceConfigURL;
 import org.apache.dubbo.common.utils.CollectionUtils;
@@ -64,10 +65,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
-import static java.util.concurrent.Executors.newSingleThreadExecutor;
 import static org.apache.dubbo.common.constants.CommonConstants.APPLICATION_KEY;
 import static org.apache.dubbo.common.constants.CommonConstants.CLUSTER_KEY;
 import static org.apache.dubbo.common.constants.CommonConstants.COMMA_SPLIT_PATTERN;
@@ -592,6 +592,7 @@ public class RegistryProtocol implements Protocol, ScopeModelAware {
 
     @Override
     public void destroy() {
+        // FIXME all application models in framework are removed at this moment
         for (ApplicationModel applicationModel : frameworkModel.getApplicationModels()) {
             for (ModuleModel moduleModel : applicationModel.getModuleModels()) {
                 List<RegistryProtocolListener> listeners = moduleModel.getExtensionLoader(RegistryProtocolListener.class)
@@ -849,7 +850,7 @@ public class RegistryProtocol implements Protocol, ScopeModelAware {
      */
     private class ExporterChangeableWrapper<T> implements Exporter<T> {
 
-        private final ExecutorService executor = newSingleThreadExecutor(new NamedThreadFactory("Exporter-Unexport", true));
+        private final ScheduledExecutorService executor;
 
         private final Invoker<T> originInvoker;
         private Exporter<T> exporter;
@@ -859,6 +860,9 @@ public class RegistryProtocol implements Protocol, ScopeModelAware {
         public ExporterChangeableWrapper(Exporter<T> exporter, Invoker<T> originInvoker) {
             this.exporter = exporter;
             this.originInvoker = originInvoker;
+            ExecutorRepository executorRepository = originInvoker.getUrl().getOrDefaultApplicationModel()
+                .getDefaultExtension(ExecutorRepository.class);
+            this.executor = executorRepository.getSharedScheduledExecutor();
         }
 
         public Invoker<T> getOriginInvoker() {
@@ -907,19 +911,15 @@ public class RegistryProtocol implements Protocol, ScopeModelAware {
                 logger.warn(t.getMessage(), t);
             }
 
-            executor.submit(() -> {
+            //TODO wait for shutdown timeout is a bit strange
+            int timeout = ConfigurationUtils.getServerShutdownTimeout(subscribeUrl.getScopeModel());
+            executor.schedule(() -> {
                 try {
-                    int timeout = ConfigurationUtils.getServerShutdownTimeout(subscribeUrl.getScopeModel());
-                    if (timeout > 0) {
-                        logger.info("Waiting " + timeout + "ms for registry to notify all consumers before unexport. " +
-                            "Usually, this is called when you use dubbo API");
-                        Thread.sleep(timeout);
-                    }
                     exporter.unexport();
                 } catch (Throwable t) {
                     logger.warn(t.getMessage(), t);
                 }
-            });
+            }, timeout, TimeUnit.MILLISECONDS);
         }
 
         public void setSubscribeUrl(URL subscribeUrl) {

dubbo-registry/dubbo-registry-api/src/main/java/org/apache/dubbo/registry/support/AbstractRegistry.java

@@ -19,10 +19,10 @@ package org.apache.dubbo.registry.support;
 import org.apache.dubbo.common.URL;
 import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
+import org.apache.dubbo.common.threadpool.manager.ExecutorRepository;
 import org.apache.dubbo.common.utils.CollectionUtils;
 import org.apache.dubbo.common.utils.ConcurrentHashSet;
 import org.apache.dubbo.common.utils.ConfigUtils;
-import org.apache.dubbo.common.utils.NamedThreadFactory;
 import org.apache.dubbo.common.utils.StringUtils;
 import org.apache.dubbo.common.utils.UrlUtils;
 import org.apache.dubbo.registry.NotifyListener;
@@ -48,7 +48,6 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -82,7 +81,7 @@ public abstract class AbstractRegistry implements Registry {
     // Local disk cache, where the special key value.registries records the list of registry centers, and the others are the list of notified service providers
     private final Properties properties = new Properties();
     // File cache timing writing
-    private final ExecutorService registryCacheExecutor = Executors.newFixedThreadPool(1, new NamedThreadFactory("DubboSaveRegistryCache", true));
+    private final ExecutorService registryCacheExecutor;
     private final AtomicLong lastCacheChanged = new AtomicLong();
     private final AtomicInteger savePropertiesRetryTimes = new AtomicInteger();
     private final Set<URL> registered = new ConcurrentHashSet<>();
@@ -100,6 +99,7 @@ public abstract class AbstractRegistry implements Registry {
         setUrl(url);
         registryManager = url.getOrDefaultApplicationModel().getBeanFactory().getBean(RegistryManager.class);
         localCacheEnabled = url.getParameter(REGISTRY_LOCAL_FILE_CACHE_ENABLED, true);
+        registryCacheExecutor = url.getOrDefaultApplicationModel().getDefaultExtension(ExecutorRepository.class).getSharedExecutor();
         if (localCacheEnabled) {
             // Start file save timer
             syncSaveFile = url.getParameter(REGISTRY_FILESAVE_SYNC_KEY, false);

dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/metadata/ServiceInstanceMetadataUtilsTest.java

@@ -26,6 +26,7 @@ import org.apache.dubbo.registry.client.DefaultServiceInstance;
 import org.apache.dubbo.registry.client.InMemoryServiceDiscovery;
 import org.apache.dubbo.registry.client.ServiceDiscovery;
 import org.apache.dubbo.registry.client.ServiceDiscoveryRegistry;
+import org.apache.dubbo.registry.client.metadata.store.InMemoryWritableMetadataService;
 import org.apache.dubbo.registry.support.RegistryManager;
 import org.apache.dubbo.rpc.model.ApplicationModel;
 
@@ -172,17 +173,16 @@ public class ServiceInstanceMetadataUtilsTest {
 
         ServiceDiscovery serviceDiscovery = Mockito.mock(ServiceDiscovery.class);
 
-        WritableMetadataService writableMetadataService = WritableMetadataService.getDefaultExtension(serviceInstance.getApplicationModel());
-        Map<String, MetadataInfo> metadataInfoMap = writableMetadataService.getMetadataInfos();
+        InMemoryWritableMetadataService writableMetadataService = (InMemoryWritableMetadataService) WritableMetadataService.getDefaultExtension(serviceInstance.getApplicationModel());
         MetadataInfo metadataInfo = new MetadataInfo("demo");
         metadataInfo.addService(new MetadataInfo.ServiceInfo(url1));
-        metadataInfoMap.put(DEFAULT_KEY, metadataInfo);
+        writableMetadataService.addMetadataInfo(DEFAULT_KEY, metadataInfo);
 
         ServiceInstanceMetadataUtils.calInstanceRevision(serviceDiscovery, serviceInstance);
         Assertions.assertEquals(metadataInfo.calAndGetRevision(), serviceInstance.getMetadata().get(EXPORTED_SERVICES_REVISION_PROPERTY_NAME));
         Assertions.assertNull(serviceInstance.getExtendParams().get(INSTANCE_REVISION_UPDATED_KEY));
 
-        metadataInfoMap.get(DEFAULT_KEY).addService(new MetadataInfo.ServiceInfo(url2));
+        writableMetadataService.getMetadataInfos().get(DEFAULT_KEY).addService(new MetadataInfo.ServiceInfo(url2));
         ServiceInstanceMetadataUtils.calInstanceRevision(serviceDiscovery, serviceInstance);
         Assertions.assertEquals(metadataInfo.calAndGetRevision(), serviceInstance.getMetadata().get(EXPORTED_SERVICES_REVISION_PROPERTY_NAME));
         Assertions.assertEquals(serviceInstance.getExtendParams().get(INSTANCE_REVISION_UPDATED_KEY), "true");

dubbo-registry/dubbo-registry-api/src/test/java/org/apache/dubbo/registry/client/migration/MigrationInvokerTest.java

@@ -20,12 +20,13 @@ import org.apache.dubbo.common.URL;
 import org.apache.dubbo.config.ApplicationConfig;
 import org.apache.dubbo.registry.client.migration.model.MigrationRule;
 import org.apache.dubbo.registry.client.migration.model.MigrationStep;
+import org.apache.dubbo.registry.integration.DemoService;
 import org.apache.dubbo.registry.integration.DynamicDirectory;
 import org.apache.dubbo.registry.integration.RegistryProtocol;
 import org.apache.dubbo.rpc.Invoker;
 import org.apache.dubbo.rpc.cluster.ClusterInvoker;
 import org.apache.dubbo.rpc.model.ApplicationModel;
-
+import org.apache.dubbo.rpc.model.FrameworkModel;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -39,7 +40,7 @@ import java.util.List;
 public class MigrationInvokerTest {
     @BeforeEach
     public void before() {
-        ApplicationModel.reset();
+        FrameworkModel.destroyAll();
         ApplicationConfig applicationConfig = new ApplicationConfig();
         applicationConfig.setName("Test");
         ApplicationModel.defaultModel().getApplicationConfigManager().setApplication(applicationConfig);
@@ -47,7 +48,7 @@ public class MigrationInvokerTest {
 
     @AfterEach
     public void after() {
-        ApplicationModel.reset();
+        FrameworkModel.destroyAll();
     }
 
     @Test
@@ -88,7 +89,7 @@ public class MigrationInvokerTest {
         Mockito.when(invoker.getUrl()).thenReturn(consumerURL);
         Mockito.when(serviceDiscoveryInvoker.getUrl()).thenReturn(consumerURL);
 
-        MigrationInvoker migrationInvoker = new MigrationInvoker(registryProtocol, null, null, null, null, consumerURL);
+        MigrationInvoker migrationInvoker = new MigrationInvoker(registryProtocol, null, null, DemoService.class, null, consumerURL);
 
         MigrationRule migrationRule = Mockito.mock(MigrationRule.class);
         Mockito.when(migrationRule.getForce(Mockito.any())).thenReturn(true);

dubbo-registry/dubbo-registry-api/src/test/resources/log4j.xml

@@ -23,7 +23,7 @@
         </layout>
     </appender>
     <root>
-        <level value="WARN"/>
+        <level value="INFO"/>
         <appender-ref ref="CONSOLE"/>
     </root>
-</log4j:configuration>
+</log4j:configuration>

dubbo-registry/dubbo-registry-zookeeper/src/main/java/org/apache/dubbo/registry/zookeeper/ZookeeperRegistry.java

@@ -67,7 +67,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
 
     private final ConcurrentMap<URL, ConcurrentMap<NotifyListener, ChildListener>> zkListeners = new ConcurrentHashMap<>();
 
-    private final ZookeeperClient zkClient;
+    private ZookeeperClient zkClient;
 
     public ZookeeperRegistry(URL url, ZookeeperTransporter zookeeperTransporter) {
         super(url);
@@ -106,22 +106,27 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
 
     @Override
     public boolean isAvailable() {
-        return zkClient.isConnected();
+        return zkClient != null && zkClient.isConnected();
     }
 
     @Override
     public void destroy() {
         super.destroy();
-        try {
-            zkClient.close();
-        } catch (Exception e) {
-            logger.warn("Failed to close zookeeper client " + getUrl() + ", cause: " + e.getMessage(), e);
+        // Just release zkClient reference, but can not close zk client here for zk client is shared somewhere else.
+        // See org.apache.dubbo.remoting.zookeeper.AbstractZookeeperTransporter#destroy()
+        zkClient = null;
+    }
+
+    private void checkDestroyed() {
+        if (zkClient == null) {
+            throw new IllegalStateException("registry is destroyed");
         }
     }
 
     @Override
     public void doRegister(URL url) {
         try {
+            checkDestroyed();
             zkClient.create(toUrlPath(url), url.getParameter(DYNAMIC_KEY, true));
         } catch (Throwable e) {
             throw new RpcException("Failed to register " + url + " to zookeeper " + getUrl() + ", cause: " + e.getMessage(), e);
@@ -131,6 +136,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
     @Override
     public void doUnregister(URL url) {
         try {
+            checkDestroyed();
             zkClient.delete(toUrlPath(url));
         } catch (Throwable e) {
             throw new RpcException("Failed to unregister " + url + " to zookeeper " + getUrl() + ", cause: " + e.getMessage(), e);
@@ -140,6 +146,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
     @Override
     public void doSubscribe(final URL url, final NotifyListener listener) {
         try {
+            checkDestroyed();
             if (ANY_VALUE.equals(url.getServiceInterface())) {
                 String root = toRootPath();
                 boolean check = url.getParameter(CHECK_KEY, false);
@@ -193,6 +200,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
 
     @Override
     public void doUnsubscribe(URL url, NotifyListener listener) {
+        checkDestroyed();
         ConcurrentMap<NotifyListener, ChildListener> listeners = zkListeners.get(url);
         if (listeners != null) {
             ChildListener zkListener = listeners.remove(listener);
@@ -219,6 +227,7 @@ public class ZookeeperRegistry extends CacheableFailbackRegistry {
             throw new IllegalArgumentException("lookup url == null");
         }
         try {
+            checkDestroyed();
             List<String> providers = new ArrayList<>();
             for (String path : toCategoriesPath(url)) {
                 List<String> children = zkClient.getChildren(path);