fix: Fix the bug of downloading zk binary file (#9274)

* fix: Fix the bug of downloading zk binary file

1. Avoid the concurrence when donwloading zookeeper binary file
2. Destroy the process after stopped zookeeper instance in Unix OS
3. Support to provide zookeeper binary file by use self and avoid to download zookeeper binary file

see issue: https://github.com/apache/dubbo/issues/9227

* fix: Fix the bug on start and stop global zk

* fix: Fix the bug on download zk binary file.

* fix: Add more log for troubleshooting

* fix: Fix the bug on move file

* fix: Fix the bug on create target directory

* fix: Fix the exception on AtomicMoveNotSupportedException

* fix: Fix the bug on start global zk in Windows OS

* perf: Rename the directory's name from test to .tmp

* perf: Optimize the code and set timeout when download zk binary archive

* perf: Use AsyncHttpClient to download the zookeeper binary archive

.gitignore

@@ -45,3 +45,6 @@ compiler/.gradle/*
 # protobuf
 dubbo-serialization/dubbo-serialization-protobuf/build/*
 dubbo-demo/dubbo-demo-triple/build/*
+
+# global registry center
+.tmp

dubbo-test/dubbo-test-check/pom.xml

@@ -38,6 +38,7 @@
         <commons.compress.version>1.20</commons.compress.version>
         <junit.platform.launcher.version>1.6.2</junit.platform.launcher.version>
         <commons.exec.version>1.3</commons.exec.version>
+        <async.http.client.version>2.12.1</async.http.client.version>
     </properties>
 
     <dependencies>
@@ -76,5 +77,11 @@
             <artifactId>commons-exec</artifactId>
             <version>${commons.exec.version}</version>
         </dependency>
+        <!-- async-http-client -->
+        <dependency>
+            <groupId>org.asynchttpclient</groupId>
+            <artifactId>async-http-client</artifactId>
+            <version>${async.http.client.version}</version>
+        </dependency>
     </dependencies>
 </project>

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/ZookeeperRegistryCenter.java

@@ -16,6 +16,8 @@
  */
 package org.apache.dubbo.test.check.registrycenter;
 
+import org.apache.dubbo.common.logger.Logger;
+import org.apache.dubbo.common.logger.LoggerFactory;
 import org.apache.dubbo.test.check.exception.DubboTestException;
 import org.apache.dubbo.test.check.registrycenter.context.ZookeeperContext;
 import org.apache.dubbo.test.check.registrycenter.context.ZookeeperWindowsContext;
@@ -29,6 +31,9 @@ import org.apache.dubbo.test.check.registrycenter.processor.ResetZookeeperProces
 import org.apache.dubbo.test.check.registrycenter.processor.StopZookeeperUnixProcessor;
 import org.apache.dubbo.test.check.registrycenter.processor.StopZookeeperWindowsProcessor;
 
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Objects;
@@ -65,8 +70,14 @@ class ZookeeperRegistryCenter implements RegistryCenter {
         } else {
             this.context = new ZookeeperWindowsContext();
         }
+
+        // initialize the context
+        this.context.setUnpackedDirectory(UNPACKED_DIRECTORY);
+        this.context.setSourceFile(TARGET_FILE_PATH);
     }
 
+    private static final Logger logger = LoggerFactory.getLogger(ZookeeperRegistryCenter.class);
+
     /**
      * The OS type.
      */
@@ -87,10 +98,47 @@ class ZookeeperRegistryCenter implements RegistryCenter {
      */
     private Map<OS, Map<Command, Processor>> processors = new HashMap<>();
 
+    /**
+     * The default unpacked directory.
+     */
+    private static final String UNPACKED_DIRECTORY = "apache-zookeeper-bin";
+
+    /**
+     * The target name of zookeeper binary file.
+     */
+    private static final String TARGET_ZOOKEEPER_FILE_NAME = UNPACKED_DIRECTORY + ".tar.gz";
+
+    /**
+     * The target directory.
+     * The zookeeper binary file named {@link #TARGET_ZOOKEEPER_FILE_NAME} will be saved in
+     * {@link #TARGET_DIRECTORY} if it downloaded successfully.
+     */
+    private static final String TARGET_DIRECTORY = ".tmp" + File.separator + "zookeeper";
+
+    /**
+     * The path of target zookeeper binary file.
+     */
+    private static final Path TARGET_FILE_PATH = getTargetFilePath();
+
     /**
      * The {@link #INITIALIZED} for flagging the {@link #startup()} method is called or not.
      */
-    private final AtomicBoolean INITIALIZED = new AtomicBoolean(false);
+    private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false);
+
+    /**
+     * Returns the target file path.
+     */
+    private static Path getTargetFilePath() {
+        String currentWorkDirectory = System.getProperty("user.dir");
+        logger.info("Current work directory: " + currentWorkDirectory);
+        int index = currentWorkDirectory.lastIndexOf(File.separator + "dubbo" + File.separator);
+        Path targetFilePath = Paths.get(currentWorkDirectory.substring(0, index),
+            "dubbo",
+            TARGET_DIRECTORY,
+            TARGET_ZOOKEEPER_FILE_NAME);
+        logger.info("Target file's absolute directory: " + targetFilePath.toString());
+        return targetFilePath;
+    }
 
     /**
      * Returns the Operating System.
@@ -140,12 +188,17 @@ class ZookeeperRegistryCenter implements RegistryCenter {
      */
     @Override
     public void startup() throws DubboTestException {
-        if (!this.INITIALIZED.get()) {
-            if (!this.INITIALIZED.compareAndSet(false, true)) {
-                return;
-            }
-            for (Initializer initializer : this.initializers) {
-                initializer.initialize(this.context);
+        if (!INITIALIZED.get()) {
+            // global look, make sure only one thread can initialize the zookeeper instances.
+            synchronized (ZookeeperRegistryCenter.class) {
+                if (!INITIALIZED.get()) {
+                    for (Initializer initializer : this.initializers) {
+                        initializer.initialize(this.context);
+                    }
+                    // add shutdown hook
+                    Runtime.getRuntime().addShutdownHook(new Thread(() -> shutdown()));
+                    INITIALIZED.set(true);
+                }
             }
         }
         this.get(os, Command.Start).process(this.context);

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/context/ZookeeperContext.java

@@ -36,6 +36,11 @@ public class ZookeeperContext implements Context {
      */
     private Path sourceFile;
 
+    /**
+     * The directory after unpacked zookeeper archive binary file.
+     */
+    private String unpackedDirectory;
+
     /**
      * Sets the source file path of downloaded zookeeper binary archive.
      */
@@ -50,6 +55,20 @@ public class ZookeeperContext implements Context {
         return this.sourceFile;
     }
 
+    /**
+     * Returns the directory after unpacked zookeeper archive binary file.
+     */
+    public String getUnpackedDirectory() {
+        return unpackedDirectory;
+    }
+
+    /**
+     * Sets the directory after unpacked zookeeper archive binary file.
+     */
+    public void setUnpackedDirectory(String unpackedDirectory) {
+        this.unpackedDirectory = unpackedDirectory;
+    }
+
     /**
      * Returns the zookeeper's version.
      */

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/initializer/ConfigZookeeperInitializer.java

@@ -47,7 +47,7 @@ public class ConfigZookeeperInitializer extends ZookeeperInitializer {
     private void updateConfig(ZookeeperContext context, int clientPort, int adminServerPort) throws DubboTestException {
         Path zookeeperConf = Paths.get(context.getSourceFile().getParent().toString(),
             String.valueOf(clientPort),
-            String.format("apache-zookeeper-%s-bin", context.getVersion()),
+            context.getUnpackedDirectory(),
             "conf");
         File zooSample = Paths.get(zookeeperConf.toString(), "zoo_sample.cfg").toFile();
 

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/initializer/DownloadZookeeperInitializer.java

@@ -20,13 +20,21 @@ import org.apache.dubbo.common.logger.Logger;
 import org.apache.dubbo.common.logger.LoggerFactory;
 import org.apache.dubbo.test.check.exception.DubboTestException;
 import org.apache.dubbo.test.check.registrycenter.context.ZookeeperContext;
+import org.asynchttpclient.Response;
+import org.asynchttpclient.AsyncHttpClient;
+import org.asynchttpclient.AsyncCompletionHandler;
+import org.asynchttpclient.DefaultAsyncHttpClientConfig;
+import org.asynchttpclient.DefaultAsyncHttpClient;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
  * Download zookeeper binary archive.
@@ -46,47 +54,124 @@ public class DownloadZookeeperInitializer extends ZookeeperInitializer {
     private static final String ZOOKEEPER_BINARY_URL_FORMAT = "https://archive.apache.org/dist/zookeeper/zookeeper-%s/" + ZOOKEEPER_FILE_NAME_FORMAT;
 
     /**
-     * The temporary directory name
+     * The temporary directory.
      */
-    private static final String TEMPORARY_DIRECTORY_NAME = "dubbo-mocked-zookeeper";
+    private static final String TEMPORARY_DIRECTORY = "zookeeper";
+
+    /**
+     * The timeout when download zookeeper binary archive file.
+     */
+    private static final int REQUEST_TIMEOUT = 30 * 1000;
+
+    /**
+     * The timeout when connect the download url.
+     */
+    private static final int CONNECT_TIMEOUT = 10 * 1000;
+
+    /**
+     * Returns {@code true} if the file exists with the given file path, otherwise {@code false}.
+     *
+     * @param filePath the file path to check.
+     */
+    private boolean checkFile(Path filePath) {
+        return Files.exists(filePath) && filePath.toFile().isFile();
+    }
 
     @Override
     protected void doInitialize(ZookeeperContext context) throws DubboTestException {
-        String zookeeperFileName = String.format(ZOOKEEPER_FILE_NAME_FORMAT, context.getVersion());
-        try {
-            context.setSourceFile(Paths.get(Files.createTempDirectory("").getParent().toString(),
-                TEMPORARY_DIRECTORY_NAME,
-                zookeeperFileName));
-        } catch (IOException e) {
-            throw new RuntimeException(String.format("Cannot create the temporary directory, related directory:%s/%s",
-                TEMPORARY_DIRECTORY_NAME, zookeeperFileName), e);
-        }
-        // check if the zookeeper binary file exists
-        if (context.getSourceFile() != null && context.getSourceFile().toFile().isFile()) {
+        // checks the zookeeper binary file exists or not
+        if (checkFile(context.getSourceFile())) {
             return;
         }
-        // create the temporary directory path.
-        if (!Files.exists(context.getSourceFile())) {
-            try {
-                Files.createDirectories(context.getSourceFile());
-            } catch (IOException e) {
-                throw new RuntimeException(String.format("Failed to create the temporary directory to save zookeeper binary file, file path:%s", context.getSourceFile()), e);
-            }
+        String zookeeperFileName = String.format(ZOOKEEPER_FILE_NAME_FORMAT, context.getVersion());
+        Path temporaryFilePath;
+        try {
+            temporaryFilePath = Paths.get(Files.createTempDirectory("").getParent().toString(),
+                TEMPORARY_DIRECTORY,
+                zookeeperFileName);
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Cannot create the temporary directory, file path: %s", TEMPORARY_DIRECTORY), e);
         }
-        // download zookeeper binary file
+
+        // create the temporary directory path.
+        try {
+            Files.createDirectories(temporaryFilePath.getParent());
+        } catch (IOException e) {
+            throw new RuntimeException(String.format("Failed to create the temporary directory to save zookeeper binary file, file path:%s", temporaryFilePath.getParent()), e);
+        }
+
+        // download zookeeper binary file in temporary directory.
         String zookeeperBinaryUrl = String.format(ZOOKEEPER_BINARY_URL_FORMAT, context.getVersion(), context.getVersion());
         try {
-            logger.info("It is beginning to download the zookeeper binary archive, it will take several minutes...");
-            URL zookeeperBinaryURL = new URL(zookeeperBinaryUrl);
-            InputStream inputStream = zookeeperBinaryURL.openStream();
-            Files.copy(inputStream, context.getSourceFile(), StandardCopyOption.REPLACE_EXISTING);
+            logger.info("It is beginning to download the zookeeper binary archive, it will take several minutes..." +
+                "\nThe zookeeper binary archive file will be download from " + zookeeperBinaryUrl + "," +
+                "\nwhich will be saved in " + temporaryFilePath.toString() + "," +
+                "\nalso it will be renamed to 'apache-zookeeper-bin.tar.gz' and moved into " + context.getSourceFile() + ".\n");
+            this.download(zookeeperBinaryUrl, temporaryFilePath);
         } catch (Exception e) {
-            throw new RuntimeException(String.format("Download zookeeper binary archive failed, download url:%s, file path:%s",
-                zookeeperBinaryUrl, context.getSourceFile()), e);
+            throw new RuntimeException(String.format("Download zookeeper binary archive failed, download url:%s, file path:%s." +
+                    "\nOr you can do something to avoid this problem as below:" +
+                    "\n1. Download zookeeper binary archive manually regardless of the version" +
+                    "\n2. Rename the downloaded file named 'apache-zookeeper-{version}-bin.tar.gz' to 'apache-zookeeper-bin.tar.gz'" +
+                    "\n3. Put the renamed file in %s, you maybe need to create the directory if necessary.\n",
+                zookeeperBinaryUrl, temporaryFilePath, context.getSourceFile()), e);
         }
-        // check if the zookeeper binary file exists again.
-        if (context.getSourceFile() == null || !context.getSourceFile().toFile().isFile()) {
+
+        // check downloaded zookeeper binary file in temporary directory.
+        if (!checkFile(temporaryFilePath)) {
+            throw new IllegalArgumentException(String.format("There are some unknown problem occurred when downloaded the zookeeper binary archive file, file path:%s", temporaryFilePath));
+        }
+
+        // create target directory if necessary
+        if (!Files.exists(context.getSourceFile())) {
+            try {
+                Files.createDirectories(context.getSourceFile().getParent());
+            } catch (IOException e) {
+                throw new IllegalArgumentException(String.format("Failed to create target directory, the directory path: %s", context.getSourceFile().getParent()), e);
+            }
+        }
+
+        // copy the downloaded zookeeper binary file into the target file path
+        try {
+            Files.copy(temporaryFilePath, context.getSourceFile(), StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new IllegalArgumentException(String.format("Failed to copy file, the source file path: %s, the target file path: %s", temporaryFilePath, context.getSourceFile()), e);
+        }
+
+        // checks the zookeeper binary file exists or not again
+        if (!checkFile(context.getSourceFile())) {
             throw new IllegalArgumentException(String.format("The zookeeper binary archive file doesn't exist, file path:%s", context.getSourceFile()));
         }
     }
+
+    /**
+     * Download the file with the given url.
+     *
+     * @param url        the url to download.
+     * @param targetPath the target path to save the downloaded file.
+     */
+    private void download(String url, Path targetPath) throws ExecutionException, InterruptedException, IOException, TimeoutException {
+        AsyncHttpClient asyncHttpClient = new DefaultAsyncHttpClient(
+            new DefaultAsyncHttpClientConfig.Builder()
+                .setConnectTimeout(CONNECT_TIMEOUT)
+                .setRequestTimeout(REQUEST_TIMEOUT)
+                .setMaxRequestRetry(1)
+                .build());
+        Future<Response> responseFuture = asyncHttpClient.prepareGet(url).execute(new AsyncCompletionHandler<Response>() {
+            @Override
+            public Response onCompleted(Response response) {
+                logger.info("Download zookeeper binary archive file successfully! download url: " + url);
+                return response;
+            }
+
+            @Override
+            public void onThrowable(Throwable t) {
+                logger.warn("Failed to download the file, download url: " + url);
+                super.onThrowable(t);
+            }
+        });
+        // Future timeout should 2 times as equal as REQUEST_TIMEOUT, because it will retry 1 time.
+        Response response = responseFuture.get(REQUEST_TIMEOUT * 2, TimeUnit.MILLISECONDS);
+        Files.copy(response.getResponseBodyAsStream(), targetPath, StandardCopyOption.REPLACE_EXISTING);
+    }
 }

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/initializer/UnpackZookeeperInitializer.java

@@ -29,6 +29,7 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
@@ -82,11 +83,27 @@ public class UnpackZookeeperInitializer extends ZookeeperInitializer {
     protected void doInitialize(ZookeeperContext context) throws DubboTestException {
         for (int clientPort : context.getClientPorts()) {
             this.unpack(context, clientPort);
+            // get the file name, just like apache-zookeeper-{version}-bin
+            // the version we maybe unknown if the zookeeper archive binary file is copied by user self.
+            Path parentPath = Paths.get(context.getSourceFile().getParent().toString(),
+                String.valueOf(clientPort));
+            if (!Files.exists(parentPath) ||
+                !parentPath.toFile().isDirectory() ||
+                parentPath.toFile().listFiles().length != 1) {
+                throw new IllegalStateException("There is something wrong in unpacked file!");
+            }
+            // rename directory
+            File sourceFile = parentPath.toFile().listFiles()[0];
+            File targetFile = Paths.get(parentPath.toString(), context.getUnpackedDirectory()).toFile();
+            sourceFile.renameTo(targetFile);
+            if (!Files.exists(targetFile.toPath()) || !targetFile.isDirectory()) {
+                throw new IllegalStateException(String.format("Failed to rename the directory. source directory: %s, target directory: %s",
+                    sourceFile.toPath().toString(),
+                    targetFile.toPath().toString()));
+            }
+            // get the bin path
+            Path zookeeperBin = Paths.get(targetFile.toString(), "bin");
             // update file permission
-            Path zookeeperBin = Paths.get(context.getSourceFile().getParent().toString(),
-                String.valueOf(clientPort),
-                String.format("apache-zookeeper-%s-bin", context.getVersion()),
-                "bin");
             for (File file : zookeeperBin.toFile().listFiles()) {
                 file.setExecutable(true, false);
                 file.setReadable(true, false);

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/processor/StartZookeeperUnixProcessor.java

@@ -41,7 +41,7 @@ public class StartZookeeperUnixProcessor extends ZookeeperUnixProcessor {
         List<String> commands = new ArrayList<>();
         Path zookeeperBin = Paths.get(context.getSourceFile().getParent().toString(),
             String.valueOf(clientPort),
-            String.format("apache-zookeeper-%s-bin", context.getVersion()),
+            context.getUnpackedDirectory(),
             "bin");
         commands.add(Paths.get(zookeeperBin.toString(), "zkServer.sh")
             .toAbsolutePath().toString());

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/processor/StartZookeeperWindowsProcessor.java

@@ -56,7 +56,7 @@ public class StartZookeeperWindowsProcessor extends ZookeeperWindowsProcessor {
             logger.info(String.format("The zookeeper-%d is starting...", clientPort));
             Path zookeeperBin = Paths.get(context.getSourceFile().getParent().toString(),
                 String.valueOf(clientPort),
-                String.format("apache-zookeeper-%s-bin", context.getVersion()),
+                context.getUnpackedDirectory(),
                 "bin");
             Executor executor = new DefaultExecutor();
             executor.setExitValues(null);

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/processor/StopZookeeperUnixProcessor.java

@@ -41,7 +41,7 @@ public class StopZookeeperUnixProcessor extends ZookeeperUnixProcessor {
         List<String> commands = new ArrayList<>();
         Path zookeeperBin = Paths.get(context.getSourceFile().getParent().toString(),
             String.valueOf(clientPort),
-            String.format("apache-zookeeper-%s-bin", context.getVersion()),
+            context.getUnpackedDirectory(),
             "bin");
         commands.add(Paths.get(zookeeperBin.toString(), "zkServer.sh")
             .toAbsolutePath().toString());

dubbo-test/dubbo-test-check/src/main/java/org/apache/dubbo/test/check/registrycenter/processor/ZookeeperUnixProcessor.java

@@ -43,6 +43,12 @@ public abstract class ZookeeperUnixProcessor implements Processor {
             Process process = this.doProcess(zookeeperContext, clientPort);
             this.logErrorStream(process.getErrorStream());
             this.awaitProcessReady(process.getInputStream());
+            // kill the process
+            try {
+                process.destroy();
+            } catch (Throwable cause) {
+                logger.warn(String.format("Failed to kill the process, with client port %s !", clientPort), cause);
+            }
         }
     }