jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!879 Delete empty staging directory and subdirectories through a single api call 

Merge pull request !879 from Pradeep Agrawal/deleteEmptyStagingDir
This commit is contained in:
i-robot 2021-05-23 13:45:48 +08:00 committed by Gitee
parent f86dd04869 23645d9b85
commit c5af94b1d1
1 changed files with 23 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/SemiTransactionalHiveMetastore.java
View File

@ -1667,7 +1667,7 @@ public class SemiTransactionalHiveMetastore
if (pathExists(hdfsContext, hdfsEnvironment, currentPath)) {
if (!targetPath.equals(currentPath)) {
renameNewPartitionDirectory(
renamePartitionDirectory(
hdfsContext,
hdfsEnvironment,
currentPath,
@ -2264,6 +2264,22 @@ public class SemiTransactionalHiveMetastore
}
}
private static void renamePartitionDirectory(HdfsContext context,
HdfsEnvironment hdfsEnvironment,
Path source,
Path target,
List<DirectoryCleanUpTask> cleanUpTasksForAbort)
{
try {
if (hdfsEnvironment.getFileSystem(context, source).rename(source, target)) {
cleanUpTasksForAbort.add(new DirectoryCleanUpTask(context, target, true));
}
}
catch (IOException e) {
renameNewPartitionDirectory(context, hdfsEnvironment, source, target, cleanUpTasksForAbort);
}
}
private static void renameDirectory(HdfsEnvironment.HdfsContext context, HdfsEnvironment hdfsEnvironment, Path source, Path target, Runnable runWhenRenameSuccess)
{
if (pathExists(context, hdfsEnvironment, target)) {
@ -2312,6 +2328,12 @@ public class SemiTransactionalHiveMetastore
return new RecursiveDeleteResult(false, ImmutableList.of());
}
if (directory.getName().startsWith(".staging")) {
if (deleteIfExists(fileSystem, directory, true)) {
return new RecursiveDeleteResult(true, ImmutableList.of());
}
}
FileStatus[] allFiles;
try {
allFiles = fileSystem.listStatus(directory);