diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml index 6f39096221..bf28193b3f 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/assembly/dolphinscheduler-alert-server.xml @@ -52,6 +52,7 @@ ${basedir}/../../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml index 77b2f54c64..c9fdab4d51 100644 --- a/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml +++ b/dolphinscheduler-api/src/main/assembly/dolphinscheduler-api-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java index 742e745fe4..620f74ef95 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegate.java @@ -31,12 +31,18 @@ import lombok.extern.slf4j.Slf4j; * This class will get the property by the priority of the following: env > jvm > properties. */ @Slf4j -public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate { +public class ImmutablePriorityPropertyDelegate implements IPropertyDelegate { private static final Map>> configValueMap = new ConcurrentHashMap<>(); - public ImmutablePriorityPropertyDelegate(String propertyAbsolutePath) { - super(propertyAbsolutePath); + private ImmutablePropertyDelegate immutablePropertyDelegate; + + private ImmutableYamlDelegate immutableYamlDelegate; + + public ImmutablePriorityPropertyDelegate(ImmutablePropertyDelegate immutablePropertyDelegate, + ImmutableYamlDelegate immutableYamlDelegate) { + this.immutablePropertyDelegate = immutablePropertyDelegate; + this.immutableYamlDelegate = immutableYamlDelegate; } @Override @@ -56,8 +62,14 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate return value; } value = getConfigValueFromProperties(key); + if (value.isPresent()) { + log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + k, value.get().getActualKey(), value.get().getValue()); + return value; + } + value = getConfigValueFromYaml(key); value.ifPresent( - stringConfigValue -> log.debug("Get config value from properties, key: {} actualKey: {}, value: {}", + stringConfigValue -> log.debug("Get config value from yaml, key: {} actualKey: {}, value: {}", k, stringConfigValue.getActualKey(), stringConfigValue.getValue())); return value; }); @@ -76,7 +88,8 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate @Override public Set getPropertyKeys() { Set propertyKeys = new HashSet<>(); - propertyKeys.addAll(super.getPropertyKeys()); + propertyKeys.addAll(this.immutablePropertyDelegate.getPropertyKeys()); + propertyKeys.addAll(this.immutableYamlDelegate.getPropertyKeys()); propertyKeys.addAll(System.getProperties().stringPropertyNames()); propertyKeys.addAll(System.getenv().keySet()); return propertyKeys; @@ -104,7 +117,15 @@ public class ImmutablePriorityPropertyDelegate extends ImmutablePropertyDelegate } private Optional> getConfigValueFromProperties(String key) { - String value = super.get(key); + String value = this.immutablePropertyDelegate.get(key); + if (value != null) { + return Optional.of(ConfigValue.fromProperties(key, value)); + } + return Optional.empty(); + } + + private Optional> getConfigValueFromYaml(String key) { + String value = this.immutableYamlDelegate.get(key); if (value != null) { return Optional.of(ConfigValue.fromProperties(key, value)); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java index b58735afb0..4a0c192210 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutablePropertyDelegate.java @@ -49,7 +49,7 @@ public class ImmutablePropertyDelegate implements IPropertyDelegate { } catch (IOException e) { log.error("Load property: {} error, please check if the file exist under classpath", propertyAbsolutePath, e); - System.exit(1); + throw new RuntimeException(e); } } printProperties(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java new file mode 100644 index 0000000000..5806a20fd7 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common.config; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.io.InputStreamResource; + +@Slf4j +public class ImmutableYamlDelegate implements IPropertyDelegate { + + private static final String REMOTE_LOGGING_YAML_NAME = "/remote-logging.yaml"; + + private final Properties properties; + + public ImmutableYamlDelegate() { + this(REMOTE_LOGGING_YAML_NAME); + } + + public ImmutableYamlDelegate(String... yamlAbsolutePath) { + properties = new Properties(); + // read from classpath + for (String fileName : yamlAbsolutePath) { + try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { + YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); + factory.setResources(new InputStreamResource(fis)); + factory.afterPropertiesSet(); + Properties subProperties = factory.getObject(); + properties.putAll(subProperties); + } catch (IOException e) { + log.error("Load property: {} error, please check if the file exist under classpath", + yamlAbsolutePath, e); + throw new RuntimeException(e); + } + } + printProperties(); + } + + public ImmutableYamlDelegate(Properties properties) { + this.properties = properties; + } + + @Override + public String get(String key) { + return properties.getProperty(key); + } + + @Override + public String get(String key, String defaultValue) { + return properties.getProperty(key, defaultValue); + } + + @Override + public Set getPropertyKeys() { + return properties.stringPropertyNames(); + } + + private void printProperties() { + properties.forEach((k, v) -> log.debug("Get property {} -> {}", k, v)); + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index ebf668a312..cc07accc9b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -35,6 +35,8 @@ public final class Constants { */ public static final String COMMON_PROPERTIES_PATH = "/common.properties"; + public static final String REMOTE_LOGGING_YAML_PATH = "/remote-logging.yaml"; + public static final String FORMAT_SS = "%s%s"; public static final String FORMAT_S_S = "%s/%s"; public static final String FORMAT_S_S_COLON = "%s:%s"; @@ -683,9 +685,6 @@ public final class Constants { public static final Integer QUERY_ALL_ON_WORKFLOW = 2; public static final Integer QUERY_ALL_ON_TASK = 3; - /** - * remote logging - */ public static final String REMOTE_LOGGING_ENABLE = "remote.logging.enable"; public static final String REMOTE_LOGGING_TARGET = "remote.logging.target"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index 8289b14479..82d4de9599 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -18,9 +18,11 @@ package org.apache.dolphinscheduler.common.utils; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; -import org.apache.dolphinscheduler.common.config.IPropertyDelegate; import org.apache.dolphinscheduler.common.config.ImmutablePriorityPropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutablePropertyDelegate; +import org.apache.dolphinscheduler.common.config.ImmutableYamlDelegate; import java.util.HashMap; import java.util.Map; @@ -37,8 +39,10 @@ import com.google.common.base.Strings; public class PropertyUtils { // todo: add another implementation for zookeeper/etcd/consul/xx - private static final IPropertyDelegate propertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + private final ImmutablePriorityPropertyDelegate propertyDelegate = + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); public static String getString(String key) { return propertyDelegate.get(key.trim()); diff --git a/dolphinscheduler-common/src/main/resources/remote-logging.yaml b/dolphinscheduler-common/src/main/resources/remote-logging.yaml new file mode 100644 index 0000000000..2cb48750a4 --- /dev/null +++ b/dolphinscheduler-common/src/main/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java index efba923a5a..6333250492 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/config/ImmutablePriorityPropertyDelegateTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.common.config; import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable; import static org.apache.dolphinscheduler.common.constants.Constants.COMMON_PROPERTIES_PATH; +import static org.apache.dolphinscheduler.common.constants.Constants.REMOTE_LOGGING_YAML_PATH; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -26,7 +27,9 @@ import org.junit.jupiter.api.Test; class ImmutablePriorityPropertyDelegateTest { private final ImmutablePriorityPropertyDelegate immutablePriorityPropertyDelegate = - new ImmutablePriorityPropertyDelegate(COMMON_PROPERTIES_PATH); + new ImmutablePriorityPropertyDelegate( + new ImmutablePropertyDelegate(COMMON_PROPERTIES_PATH), + new ImmutableYamlDelegate(REMOTE_LOGGING_YAML_PATH)); @Test void getOverrideFromEnv() throws Exception { diff --git a/dolphinscheduler-common/src/test/resources/remote-logging.yaml b/dolphinscheduler-common/src/test/resources/remote-logging.yaml new file mode 100644 index 0000000000..cb149a77fe --- /dev/null +++ b/dolphinscheduler-common/src/test/resources/remote-logging.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +remote-logging: + # Whether to enable remote logging + enable: false + # if remote-logging.enable = true, set the target of remote logging + target: OSS + # if remote-logging.enable = true, set the log base directory + base.dir: logs + # if remote-logging.enable = true, set the number of threads to send logs to remote storage + thread.pool.size: 10 + # required if you set remote-logging.target=OSS + oss: + # oss access key id, required if you set remote-logging.target=OSS + access.key.id: + # oss access key secret, required if you set remote-logging.target=OSS + access.key.secret: + # oss bucket name, required if you set remote-logging.target=OSS + bucket.name: + # oss endpoint, required if you set remote-logging.target=OSS + endpoint: + # required if you set remote-logging.target=S3 + s3: + # s3 access key id, required if you set remote-logging.target=S3 + access.key.id: + # s3 access key secret, required if you set remote-logging.target=S3 + access.key.secret: + # s3 bucket name, required if you set remote-logging.target=S3 + bucket.name: + # s3 endpoint, required if you set remote-logging.target=S3 + endpoint: + # s3 region, required if you set remote-logging.target=S3 + region: + google.cloud.storage: + # the location of the google cloud credential, required if you set remote-logging.target=GCS + credential: /path/to/credential + # gcs bucket name, required if you set remote-logging.target=GCS + bucket.name: + abs: + # abs account name, required if you set resource.storage.type=ABS + account.name: + # abs account key, required if you set resource.storage.type=ABS + account.key: + # abs container name, required if you set resource.storage.type=ABS + container.name: + diff --git a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml index 9fc3a3b679..d521e53bc2 100644 --- a/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml +++ b/dolphinscheduler-master/src/main/assembly/dolphinscheduler-master-server.xml @@ -52,6 +52,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf diff --git a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml index 70622942f0..5ac9b6350d 100644 --- a/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml +++ b/dolphinscheduler-worker/src/main/assembly/dolphinscheduler-worker-server.xml @@ -53,6 +53,7 @@ ${basedir}/../dolphinscheduler-common/src/main/resources **/*.properties + **/*.yaml conf