[DSIP-25][Remote Logging] Split remote logging configuration (#15826)

Co-authored-by: Rick Cheng <rickchengx@gmail.com>

dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml

@@ -52,6 +52,7 @@
             <directory>${basedir}/../../dolphinscheduler-common/src/main/resources</directory>
             <includes>
                 <include>**/*.properties</include>
+                <include>**/*.yaml</include>
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>

dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml

@@ -53,6 +53,7 @@
             <directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
             <includes>
                 <include>**/*.properties</include>
+                <include>**/*.yaml</include>
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java

@@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j;
  * This class will get the property by the priority of the following: env > jvm > properties.
  */
 @Slf4j
-public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate {
+public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate {
 
     private static final Map<String, Optional<ConfigValue<String>>> configValueMap = new ConcurrentHashMap<>();
 
-    public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) {
+    private ImmutablePropertyDelegate immutablePropertyDelegate;
-        super(propertyAbsolutePath);
+
+    private ImmutableYamlDelegate immutableYamlDelegate;
+
+    public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate,
+                                             ImmutableYamlDelegate immutableYamlDelegate) {
+        this.immutablePropertyDelegate = immutablePropertyDelegate;
+        this.immutableYamlDelegate = immutableYamlDelegate;
     }
 
     @Override
@@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
                 return value;
             }
             value = getConfigValueFromProperties(key);
+            if (value.isPresent()) {
+                log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
+                        k, value.get().getActualKey(), value.get().getValue());
+                return value;
+            }
+            value = getConfigValueFromYaml(key);
             value.ifPresent(
-                    stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}",
+                    stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}",
                             k, stringConfigValue.getActualKey(), stringConfigValue.getValue()));
             return value;
         });
@@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
     @Override
     public Set<String> getPropertyKeys() {
         Set<String> propertyKeys = new HashSet<>();
-        propertyKeys.addAll(super.getPropertyKeys());
+        propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys());
+        propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys());
         propertyKeys.addAll(System.getProperties().stringPropertyNames());
         propertyKeys.addAll(System.getenv().keySet());
         return propertyKeys;
@@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate
     }
 
     private Optional<ConfigValue<String>> getConfigValueFromProperties(String key) {
-        String value = super.get(key);
+        String value = this.immutablePropertyDelegate.get(key);
+        if (value != null) {
+            return Optional.of(ConfigValue.fromProperties(key, value));
+        }
+        return Optional.empty();
+    }
+
+    private Optional<ConfigValue<String>> getConfigValueFromYaml(String key) {
+        String value = this.immutableYamlDelegate.get(key);
         if (value != null) {
             return Optional.of(ConfigValue.fromProperties(key, value));
         }

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java

@@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate {
             } catch (IOException e) {
                 log.error("Load property: {} error, please check if the file exist under classpath",
                         propertyAbsolutePath, e);
-                System.exit(1);
+                throw new RuntimeException(e);
             }
         }
         printProperties();

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java

@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.common.config;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.Set;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
+import org.springframework.core.io.InputStreamResource;
+
+@Slf4j
+public class ImmutableYamlDelegate implements IPropertyDelegate {
+
+    private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml";
+
+    private final Properties properties;
+
+    public ImmutableYamlDelegate() {
+        this(REMOTE_LOGGING_YAML_NAME);
+    }
+
+    public ImmutableYamlDelegate(String... yamlAbsolutePath) {
+        properties = new Properties();
+        // read from classpath
+        for (String fileName : yamlAbsolutePath) {
+            try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) {
+                YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
+                factory.setResources(new InputStreamResource(fis));
+                factory.afterPropertiesSet();
+                Properties subProperties = factory.getObject();
+                properties.putAll(subProperties);
+            } catch (IOException e) {
+                log.error("Load property: {} error, please check if the file exist under classpath",
+                        yamlAbsolutePath, e);
+                throw new RuntimeException(e);
+            }
+        }
+        printProperties();
+    }
+
+    public ImmutableYamlDelegate(Properties properties) {
+        this.properties = properties;
+    }
+
+    @Override
+    public String get(String key) {
+        return properties.getProperty(key);
+    }
+
+    @Override
+    public String get(String key, String defaultValue) {
+        return properties.getProperty(key, defaultValue);
+    }
+
+    @Override
+    public Set<String> getPropertyKeys() {
+        return properties.stringPropertyNames();
+    }
+
+    private void printProperties() {
+        properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v));
+    }
+}

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java

@@ -35,6 +35,8 @@ public final class Constants {
      */
     public static final String COMMON_PROPERTIES_PATH = "/common.properties";
 
+    public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml";
+
     public static final String FORMAT_SS = "%s%s";
     public static final String FORMAT_S_S = "%s/%s";
     public static final String FORMAT_S_S_COLON = "%s:%s";
@@ -683,9 +685,6 @@ public final class Constants {
     public static final Integer QUERY_ALL_ON_WORKFLOW = 2;
     public static final Integer QUERY_ALL_ON_TASK = 3;
 
-    /**
-     * remote logging
-     */
     public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable";
 
     public static final String REMOTE_LOGGING_TARGET = "remote.logging.target";

dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java

@@ -18,9 +18,11 @@
 package org.apache.dolphinscheduler.common.utils;
 
 import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
+import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
 
-import org.apache.dolphinscheduler.common.config.IPropertyDelegate;
 import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate;
+import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate;
+import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -37,8 +39,10 @@ import com.google.common.base.Strings;
 public class PropertyUtils {
 
     // todo: add another implementation for zookeeper/etcd/consul/xx
-    private static final IPropertyDelegate propertyDelegate =
+    private final ImmutablePriorityPropertyDelegate propertyDelegate =
-            new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
+            new ImmutablePriorityPropertyDelegate(
+                    new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
+                    new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
 
     public static String getString(String key) {
         return propertyDelegate.get(key.trim());

dolphinscheduler-common/src/main/resources/remote-logging.yaml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+remote-logging:
+  # Whether to enable remote logging
+  enable: false
+  # if remote-logging.enable = true, set the target of remote logging
+  target: OSS
+  # if remote-logging.enable = true, set the log base directory
+  base.dir: logs
+  # if remote-logging.enable = true, set the number of threads to send logs to remote storage
+  thread.pool.size: 10
+  # required if you set remote-logging.target=OSS
+  oss:
+    # oss access key id, required if you set remote-logging.target=OSS
+    access.key.id: <access.key.id>
+    # oss access key secret, required if you set remote-logging.target=OSS
+    access.key.secret: <access.key.secret>
+    # oss bucket name, required if you set remote-logging.target=OSS
+    bucket.name: <bucket.name>
+    # oss endpoint, required if you set remote-logging.target=OSS
+    endpoint: <endpoint>
+  # required if you set remote-logging.target=S3
+  s3:
+    # s3 access key id, required if you set remote-logging.target=S3
+    access.key.id: <access.key.id>
+    # s3 access key secret, required if you set remote-logging.target=S3
+    access.key.secret: <access.key.secret>
+      # s3 bucket name, required if you set remote-logging.target=S3
+    bucket.name: <bucket.name>
+      # s3 endpoint, required if you set remote-logging.target=S3
+    endpoint: <endpoint>
+      # s3 region, required if you set remote-logging.target=S3
+    region: <region>
+  google.cloud.storage:
+    # the location of the google cloud credential, required if you set remote-logging.target=GCS
+    credential: /path/to/credential
+      # gcs bucket name, required if you set remote-logging.target=GCS
+    bucket.name: <your-bucket>
+  abs:
+    # abs account name, required if you set resource.storage.type=ABS
+    account.name: <your-account-name>
+      # abs account key, required if you set resource.storage.type=ABS
+    account.key: <your-account-key>
+      # abs container name, required if you set resource.storage.type=ABS
+    container.name: <your-container-name>
+

dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java

@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config;
 
 import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
 import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH;
+import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
@@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test;
 class ImmutablePriorityPropertyDelegateTest {
 
     private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate =
-            new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH);
+            new ImmutablePriorityPropertyDelegate(
+                    new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH),
+                    new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH));
 
     @Test
     void getOverrideFromEnv() throws Exception {

dolphinscheduler-common/src/test/resources/remote-logging.yaml

@@ -0,0 +1,61 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+remote-logging:
+  # Whether to enable remote logging
+  enable: false
+  # if remote-logging.enable = true, set the target of remote logging
+  target: OSS
+  # if remote-logging.enable = true, set the log base directory
+  base.dir: logs
+  # if remote-logging.enable = true, set the number of threads to send logs to remote storage
+  thread.pool.size: 10
+  # required if you set remote-logging.target=OSS
+  oss:
+    # oss access key id, required if you set remote-logging.target=OSS
+    access.key.id: <access.key.id>
+    # oss access key secret, required if you set remote-logging.target=OSS
+    access.key.secret: <access.key.secret>
+    # oss bucket name, required if you set remote-logging.target=OSS
+    bucket.name: <bucket.name>
+    # oss endpoint, required if you set remote-logging.target=OSS
+    endpoint: <endpoint>
+  # required if you set remote-logging.target=S3
+  s3:
+    # s3 access key id, required if you set remote-logging.target=S3
+    access.key.id: <access.key.id>
+    # s3 access key secret, required if you set remote-logging.target=S3
+    access.key.secret: <access.key.secret>
+    # s3 bucket name, required if you set remote-logging.target=S3
+    bucket.name: <bucket.name>
+    # s3 endpoint, required if you set remote-logging.target=S3
+    endpoint: <endpoint>
+    # s3 region, required if you set remote-logging.target=S3
+    region: <region>
+  google.cloud.storage:
+    # the location of the google cloud credential, required if you set remote-logging.target=GCS
+    credential: /path/to/credential
+    # gcs bucket name, required if you set remote-logging.target=GCS
+    bucket.name: <your-bucket>
+  abs:
+    # abs account name, required if you set resource.storage.type=ABS
+    account.name: <your-account-name>
+    # abs account key, required if you set resource.storage.type=ABS
+    account.key: <your-account-key>
+    # abs container name, required if you set resource.storage.type=ABS
+    container.name: <your-container-name>
+

dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml

@@ -52,6 +52,7 @@
             <directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
             <includes>
                 <include>**/*.properties</include>
+                <include>**/*.yaml</include>
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>

dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml

@@ -53,6 +53,7 @@
             <directory>${basedir}/../dolphinscheduler-common/src/main/resources</directory>
             <includes>
                 <include>**/*.properties</include>
+                <include>**/*.yaml</include>
             </includes>
             <outputDirectory>conf</outputDirectory>
         </fileSet>