natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

split sqoop import hive database and table (#4141)

This commit is contained in:
Yelli 2020-12-02 15:50:50 +08:00 committed by GitHub
parent 3ceb612467
commit aa0974fd1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopConstants.java
View File

@ -57,6 +57,7 @@ public final class SqoopConstants {
//sqoop hive
public static final String HIVE_IMPORT = "--hive-import";
public static final String HIVE_DATABASE = "--hive-database";
public static final String HIVE_TABLE = "--hive-table";
public static final String CREATE_HIVE_TABLE = "--create-hive-table";
public static final String HIVE_DROP_IMPORT_DELIMS = "--hive-drop-import-delims";

7
dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sqoop/generator/targets/HiveTargetGenerator.java
View File

@ -49,9 +49,10 @@ public class HiveTargetGenerator implements ITargetGenerator {
if (StringUtils.isNotEmpty(targetHiveParameter.getHiveDatabase())
&& StringUtils.isNotEmpty(targetHiveParameter.getHiveTable())) {
hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE)
.append(Constants.SPACE).append(String.format("%s.%s", targetHiveParameter.getHiveDatabase(),
targetHiveParameter.getHiveTable()));
hiveTargetSb.append(Constants.SPACE).append(SqoopConstants.HIVE_DATABASE)
.append(Constants.SPACE).append(targetHiveParameter.getHiveDatabase())
.append(Constants.SPACE).append(SqoopConstants.HIVE_TABLE)
.append(Constants.SPACE).append(targetHiveParameter.getHiveTable());
}
if (targetHiveParameter.isCreateHiveTable()) {

2
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/task/sqoop/SqoopTaskTest.java
View File

@ -144,7 +144,7 @@ public class SqoopTaskTest {
String mysqlToHiveScript = generator.generateSqoopJob(mysqlToHiveParams, mysqlTaskExecutionContext);
String mysqlToHiveExpected =
"sqoop import -D mapred.job.name=sqoop_import -m 1 --connect \"jdbc:mysql://192.168.0.111:3306/test\" --username kylo --password \"123456\" "
+ "--query \"SELECT * FROM person_2 WHERE \\$CONDITIONS\" --map-column-java id=Integer --hive-import --hive-table stg.person_internal_2 "
+ "--query \"SELECT * FROM person_2 WHERE \\$CONDITIONS\" --map-column-java id=Integer --hive-import --hive-database stg --hive-table person_internal_2 "
+ "--create-hive-table --hive-overwrite --delete-target-dir --hive-partition-key date --hive-partition-value 2020-02-16";
Assert.assertEquals(mysqlToHiveExpected, mysqlToHiveScript);