natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix createFile with permission will not work (#15556)

This commit is contained in:
Wenjun Ruan 2024-02-03 18:00:33 +08:00 committed by GitHub
parent 1b42d45fcd
commit 4d15932ede
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 78 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

151
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
View File

@ -25,8 +25,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VI
import static org.apache.dolphinscheduler.common.constants.Constants.UTF_8;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
import org.apache.dolphinscheduler.common.exception.FileOperateException;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SystemUtils;
@ -38,9 +36,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.util.Set;
@ -48,11 +44,10 @@ import java.util.zip.CRC32;
import java.util.zip.CheckedInputStream;
import lombok.NonNull;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
/**
* file utils
*/
@UtilityClass
@Slf4j
public class FileUtils {
@ -62,14 +57,7 @@ public class FileUtils {
public static final String KUBE_CONFIG_FILE = "config";
private static final String RWXR_XR_X = "rwxr-xr-x";
private static final FileAttribute<Set<PosixFilePermission>> PERMISSION_755 =
PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString(RWXR_XR_X));
private FileUtils() {
throw new UnsupportedOperationException("Construct FileUtils");
}
private static final Set<PosixFilePermission> PERMISSION_755 = PosixFilePermissions.fromString("rwxr-xr-x");
/**
* get download file absolute path and name
@ -161,34 +149,6 @@ public class FileUtils {
return PropertyUtils.getString(RESOURCE_VIEW_SUFFIXES, RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE);
}
/**
* create directory if absent
*
* @param execLocalPath execute local path
* @throws IOException errors
*/
public static void createWorkDirIfAbsent(String execLocalPath) throws IOException {
// if work dir exists, first delete
File execLocalPathFile = new File(execLocalPath);
if (execLocalPathFile.exists()) {
try {
org.apache.commons.io.FileUtils.forceDelete(execLocalPathFile);
} catch (Exception ex) {
if (ex instanceof NoSuchFileException || ex.getCause() instanceof NoSuchFileException) {
// this file is already be deleted.
} else {
throw ex;
}
}
}
// create work dir
org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
String mkdirLog = "create dir success " + execLocalPath;
log.info(mkdirLog);
}
/**
* write content to file ,if parent path not exists, it will do one's utmost to mkdir
*
@ -231,25 +191,6 @@ public class FileUtils {
org.apache.commons.io.FileUtils.deleteQuietly(new File(filename));
}
/**
* Gets all the parent subdirectories of the parentDir directory
*
* @param parentDir parent dir
* @return all dirs
*/
public static File[] getAllDir(String parentDir) {
if (parentDir == null || "".equals(parentDir)) {
throw new RuntimeException("parentDir can not be empty");
}
File file = new File(parentDir);
if (!file.exists() || !file.isDirectory()) {
throw new RuntimeException("parentDir not exist, or is not a directory:" + parentDir);
}
return file.listFiles(File::isDirectory);
}
/**
* Get Content
*
@ -325,59 +266,47 @@ public class FileUtils {
return crcString;
}
public static void setFileOwner(Path filePath, String fileOwner) throws FileOperateException {
try {
// We use linux command to set the file owner, since jdk api will not use sudo.
String command = String.format("sudo chown %s %s", fileOwner, filePath.toString());
Runtime.getRuntime().exec(command);
Process process = Runtime.getRuntime().exec(command);
int exitCode = process.waitFor();
if (0 != exitCode) {
throw new FileOperateException(
"Set file: " + filePath + " to owner: " + fileOwner + " failed, existCode(" + exitCode + ")");
}
} catch (FileOperateException ex) {
throw ex;
} catch (Exception ex) {
throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
}
}
public static void setDirectoryOwner(Path filePath, String fileOwner) throws FileOperateException {
try {
// We use linux command to set the file owner, since jdk api will not use sudo.
String command = String.format("sudo chown -R %s %s", fileOwner, filePath.toString());
Runtime.getRuntime().exec(command);
Process process = Runtime.getRuntime().exec(command);
int exitCode = process.waitFor();
if (0 != exitCode) {
throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner
+ " failed, existCode(" + exitCode + ")");
}
} catch (FileOperateException ex) {
throw ex;
} catch (Exception ex) {
throw new FileOperateException("Set directory: " + filePath + " to owner: " + fileOwner + " failed");
}
}
public static void createDirectoryIfNotPresent(Path path) throws IOException {
if (Files.exists(path)) {
return;
}
Files.createDirectories(path);
}
/**
* Create a file with '755'.
*/
public static void createFileWith755(@NonNull Path path) throws IOException {
if (SystemUtils.IS_OS_WINDOWS) {
Files.createFile(path);
} else {
Files.createFile(path, PERMISSION_755);
Files.createFile(path);
Files.setPosixFilePermissions(path, PERMISSION_755);
}
}
public static void createDirectoryWith755(@NonNull Path path) throws IOException {
if (path.toFile().exists()) {
return;
}
if (OSUtils.isWindows()) {
Files.createDirectories(path);
} else {
Path parent = path.getParent();
if (parent != null && !parent.toFile().exists()) {
createDirectoryWith755(parent);
}
Files.createDirectory(path);
Files.setPosixFilePermissions(path, PERMISSION_755);
}
}
public static void setFileTo755(File file) throws IOException {
if (OSUtils.isWindows()) {
return;
}
if (file.isFile()) {
Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
return;
}
Files.setPosixFilePermissions(file.toPath(), PERMISSION_755);
File[] files = file.listFiles();
if (files != null) {
for (File f : files) {
setFileTo755(f);
}
}
}

24
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
View File

@ -19,8 +19,13 @@ package org.apache.dolphinscheduler.common.utils;
import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@ -55,12 +60,23 @@ public class FileUtilsTest {
}
@Test
public void testCreateWorkDirIfAbsent() {
public void createDirectoryWith755() throws IOException {
Path path = Paths.get("/tmp/createWorkDirAndUserIfAbsent");
try {
FileUtils.createWorkDirIfAbsent("/tmp/createWorkDirAndUserIfAbsent");
Assertions.assertTrue(true);
FileUtils.createDirectoryWith755(path);
File file = path.toFile();
Assertions.assertTrue(file.exists());
Assertions.assertTrue(file.isDirectory());
Assertions.assertTrue(file.canExecute());
Assertions.assertTrue(file.canRead());
Assertions.assertTrue(file.canWrite());
FileUtils.createDirectoryWith755(Paths.get("/"));
} catch (Exception e) {
Assertions.fail();
e.printStackTrace();
Assertions.fail(e.getMessage());
} finally {
Files.deleteIfExists(path);
}
}

3
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/utils/ProcessUtils.java
View File

@ -30,6 +30,7 @@ import org.apache.commons.lang3.SystemUtils;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -176,7 +177,7 @@ public class ProcessUtils {
taskExecutionContext.getProcessInstanceId(),
taskExecutionContext.getTaskInstanceId()));
}
FileUtils.createWorkDirIfAbsent(taskExecutionContext.getExecutePath());
FileUtils.createDirectoryWith755(Paths.get(taskExecutionContext.getExecutePath()));
org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils.cancelApplication(taskExecutionContext);
return appIds;
} else {

3
dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java
View File

@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@ -170,7 +171,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
BlobClient blobClient = blobContainerClient.getBlobClient(srcFilePath);

3
dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java
View File

@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@ -143,7 +144,7 @@ public class GcsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
Blob blob = gcsStorage.get(BlobId.of(bucketName, srcFilePath));

3
dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java
View File

@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@ -187,7 +188,7 @@ public class ObsStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
ObsObject obsObject = obsClient.getObject(bucketName, srcFilePath);
try (

3
dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java
View File

@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.factory.OssClientFactory;
import org.apache.dolphinscheduler.common.model.OssConnection;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@ -213,7 +214,7 @@ public class OssStorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
OSSObject ossObject = ossClient.getObject(bucketName, srcFilePath);
try (

3
dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java
View File

@ -25,6 +25,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TY
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate;
@ -200,7 +201,7 @@ public class S3StorageOperator implements Closeable, StorageOperate {
if (dstFile.isDirectory()) {
Files.delete(dstFile.toPath());
} else {
Files.createDirectories(dstFile.getParentFile().toPath());
FileUtils.createDirectoryWith755(dstFile.getParentFile().toPath());
}
S3Object o = s3Client.getObject(bucketName, srcFilePath);
try (

8
dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.task.api.shell;
import org.apache.dolphinscheduler.common.constants.TenantConstants;
import org.apache.dolphinscheduler.common.exception.FileOperateException;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.utils.AbstractCommandExecutorConstants;
@ -71,12 +69,8 @@ public abstract class BaseLinuxShellInterceptorBuilder<T extends BaseLinuxShellI
"****************************** Script Content *****************************************************************");
}
protected List<String> generateBootstrapCommand() throws FileOperateException {
protected List<String> generateBootstrapCommand() {
if (sudoEnable) {
if (!TenantConstants.BOOTSTRAPT_SYSTEM_USER.equals(runUser)) {
// Set the tenant owner as the working directory
FileUtils.setDirectoryOwner(Paths.get(shellDirectory), runUser);
}
return bootstrapCommandInSudoMode();
}
return bootstrapCommandInNormalMode();

3
dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java
View File

@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.enums.Direct.IN;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAR;
import static org.apache.dolphinscheduler.plugin.task.java.JavaConstants.RUN_TYPE_JAVA;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
@ -155,7 +156,7 @@ public class JavaTaskTest {
try {
Path path = Paths.get(fileName);
if (!Files.exists(path)) {
Files.createDirectories(path);
FileUtils.createDirectoryWith755(path);
}
javaTask.createJavaSourceFileIfNotExists(sourceCode, fileName);
} finally {

3
dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java
View File

@ -97,7 +97,7 @@ public class TaskExecutionContextUtils {
log.warn("The TaskInstance WorkingDirectory: {} is exist, will recreate again",
taskInstanceWorkingDirectory);
}
Files.createDirectories(Paths.get(taskInstanceWorkingDirectory));
FileUtils.createDirectoryWith755(Paths.get(taskInstanceWorkingDirectory));
taskExecutionContext.setExecutePath(taskInstanceWorkingDirectory);
taskExecutionContext.setAppInfoPath(FileUtils.getAppInfoPath(taskInstanceWorkingDirectory));
@ -137,6 +137,7 @@ public class TaskExecutionContextUtils {
storageOperate.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true);
log.debug("Download resource file {} under: {} successfully", resourceAbsolutePathInStorage,
resourceAbsolutePathInLocal);
FileUtils.setFileTo755(file);
WorkerServerMetrics
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime);
WorkerServerMetrics

2
dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtilsTest.java
View File

@ -49,7 +49,7 @@ class TaskExecutionContextUtilsTest {
try {
// Test if the working directory is exist
// will delete it and recreate
Files.createDirectories(Paths.get(taskWorkingDirectory));
FileUtils.createDirectoryWith755(Paths.get(taskWorkingDirectory));
Files.createFile(Paths.get(taskWorkingDirectory, "text.txt"));
Assertions.assertTrue(Files.exists(Paths.get(taskWorkingDirectory, "text.txt")));