natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Resource relate path invalid when tenant change (#15581) 

* fix can not get correct resource related path when
  user run workflow with differnet tenant of resource
  created
* also fix can not get correct related path when we
  use `resource.storage.type=LOCAL`
This commit is contained in:
Jay Chung 2024-02-06 23:56:03 +08:00 committed by GitHub
parent 89da67ddeb
commit bd83631955
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 53 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java
View File

@ -17,6 +17,8 @@
package org.apache.dolphinscheduler.plugin.storage.api;
import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@ -72,7 +74,16 @@ public interface StorageOperate {
*/
default String getResourceFileName(String tenantCode, String fullName) {
String resDir = getResDir(tenantCode);
return fullName.replaceFirst(resDir, "");
String filenameReplaceResDir = fullName.replaceFirst(resDir, "");
if (!filenameReplaceResDir.equals(fullName)) {
return filenameReplaceResDir;
}
// Replace resource dir not effective in case of run workflow with different tenant from resource file's.
// this is backup solution to get related path, by split with RESOURCE_TYPE_FILE
return filenameReplaceResDir.contains(RESOURCE_TYPE_FILE)
? filenameReplaceResDir.split(String.format("%s/", RESOURCE_TYPE_FILE))[1]
: filenameReplaceResDir;
}
/**

2
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java
View File

@ -71,7 +71,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
@Slf4j
public class HdfsStorageOperator implements Closeable, StorageOperate {
private static HdfsStorageProperties hdfsProperties = new HdfsStorageProperties();
protected static HdfsStorageProperties hdfsProperties = new HdfsStorageProperties();
private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY";
private volatile boolean yarnEnabled = false;

39
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.storage.hdfs;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class LocalStorageOperator extends HdfsStorageOperator {
public LocalStorageOperator() {
super(new HdfsStorageProperties());
}
public LocalStorageOperator(HdfsStorageProperties hdfsStorageProperties) {
super(hdfsStorageProperties);
}
@Override
public String getResourceFileName(String tenantCode, String fullName) {
// prefix schema `file:/` should be remove in local file mode
String fullNameRemoveSchema = fullName.replaceFirst(hdfsProperties.getDefaultFS(), "");
return super.getResourceFileName(tenantCode, fullNameRemoveSchema);
}
}

2
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java
View File

@ -32,7 +32,7 @@ public class LocalStorageOperatorFactory implements StorageOperateFactory {
public StorageOperate createStorageOperate() {
HdfsStorageProperties hdfsStorageProperties = new HdfsStorageProperties();
hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS);
return new HdfsStorageOperator(hdfsStorageProperties);
return new LocalStorageOperator(hdfsStorageProperties);
}
@Override