jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix sql-migration-tool bugs on hive and impala. include, syntax incorrection and ui bugs.

This commit is contained in:
Jiwen 2020-09-05 18:13:36 +08:00
parent c38bae13ff
commit bf8f5136de
13 changed files with 156 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hetu-docs/en/migration/sql-migration-tool.md
View File

@ -52,6 +52,7 @@ This tool also can take parameters and running in batch mode. It has five parame
| `--type` or `-t` | The type of input SQL statement, such as `hive`, `impala`. It\'s optional parameter and the default value is `hive`. |
| `--output` or `-o` | the directory to save the converted SQL results. The result file's naming convention will be the input file's name + timestamp + .html suffix. |
| `--config` or `-c` | the config file of SQL Migration Tool. |
| `--debug` or `-d` | if set value to 'true', then print the debug information in console. |
*Tip:*
@ -237,6 +238,7 @@ Below Impala statements are partially supported, which mean some keywords or att
| ADD COMMENTS | Adding comments to databases or columns is not supported | [COMMENT](../sql/comment.md) |
| SET SESSION | Only support "SET" and "SET ALL" | [SET SESSION](../sql/set-session.md) |
| ADD COLUMNS | ADD multiple columns within single statement is not supported, kudu properties are not supported. | [ALTER TABLE](../sql/alter-table.md) |
| SHOW FUNCTIONS | Only support show all functions. | [SHOW FUNCTIONS](../sql/show-functions.md) |
Below Impala statements are not supported, because of feature differences:
@ -258,7 +260,6 @@ Below Impala statements are not supported, because of feature differences:
| REFRESH FUNCTION |
| UPDATE TABLE |
| UPSERT |
| SHOW FUNCTIONS |
| SHOW TABLE/COLUMN STATS |
| SHOW PARTITIONS |
| SHOW FILES |

3
hetu-docs/zh/migration/sql-migration-tool.md
View File

@ -49,6 +49,7 @@ INSERT INTO table1
| `--type` or `-t`| 输入SQL语句的类型，如`hive`、`impala`。可选参数，默认值为`hive`。|
| `--output` or `-o`| 转换后SQL结果的存放目录。|
| `--config` or `-c`| SQL迁移工具的配置文件。|
| `--debug` or `-d`| 需要打印debug到控制界面时，设置值为true。|
*提示：*
@ -233,6 +234,7 @@ SELECT (DECIMAL '2.0' * 3)
| ADD COMMENTS | 不支持给数据库和列添加评论 | [COMMENT](../sql/comment.md) |
| SET SESSION | 仅支持 "SET" 和 "SET ALL" | [SET SESSION](../sql/set-session.md) |
| ADD COLUMNS | 不支持在一条语句添加多列，也不支持设置kudu属性 | [ALTER TABLE](../sql/alter-table.md) |
| SHOW FUNCTIONS | 仅支持显示全部的函数 | [SHOW FUNCTIONS](../sql/show-functions.md) |
由于特性差异，如下Impala语句暂不支持：
@ -254,7 +256,6 @@ SELECT (DECIMAL '2.0' * 3)
| REFRESH FUNCTION |
| UPDATE TABLE |
| UPSERT |
| SHOW FUNCTIONS |
| SHOW TABLE/COLUMN STATS |
| SHOW PARTITIONS |
| SHOW FILES |

62
hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/hive/HiveSql.g4
View File

@ -78,9 +78,9 @@ statement
| DELETE FROM qualifiedName (WHERE booleanExpression)? #delete
| ALTER TABLE from=qualifiedName RENAME TO to=qualifiedName #renameTable
| ALTER TABLE qualifiedName SET TBLPROPERTIES properties #commentTable
| ALTER TABLE tableName=qualifiedName (PARTITION partition=properties)?
| ALTER TABLE tableName=qualifiedName (PARTITION partition=partitionSpec)?
(ADD | REPLACE) COLUMNS '('tableElement (',' tableElement)*')' (CASCADE | RESTRICT)? #addReplaceColumn
| ALTER TABLE qualifiedName (PARTITION properties)? (SET SERDE string (WITH SERDEPROPERTIES properties)? | SET SERDEPROPERTIES properties) #alterTableSerde
| ALTER TABLE qualifiedName (PARTITION partitionSpec)? (SET SERDE string (WITH SERDEPROPERTIES properties)? | SET SERDEPROPERTIES properties) #alterTableSerde
| ALTER TABLE qualifiedName CLUSTERED BY columnAliases (SORTED BY columnAliases)? INTO expression BUCKETS #alterTableStorage
| ALTER TABLE qualifiedName SKEWED BY columnAliases ON expression (',' expression)* (STORED_AS DIRECTORIES)? #alterTableSkewed
| ALTER TABLE qualifiedName NOT SKEWED #alterTableNotSkewed
@ -88,25 +88,25 @@ statement
| ALTER TABLE qualifiedName SET SKEWED LOCATION properties #alterTableSetSkewedLocation
| ALTER TABLE qualifiedName ADD CONSTRAINT identifier
(PRIMARY KEY columnAliases DISABLE NOVALIDATE
| FOREIGN KEY columnAliases REFERENCES identifier columnAliases DISABLE NOVALIDATE RELY
| FOREIGN KEY columnAliases REFERENCES qualifiedName columnAliases DISABLE NOVALIDATE RELY
| UNIQUE columnAliases DISABLE NOVALIDATE) #alterTableAddConstraint
| ALTER TABLE qualifiedName CHANGE COLUMN identifier identifier type
CONSTRAINT identifier (NOT NULL ENABLE | DEFAULT defaultValue ENABLE | CHECK expression ENABLE) #alterTableChangeConstraint
| ALTER TABLE qualifiedName DROP CONSTRAINT identifier #alterTableDropConstraint
| ALTER TABLE qualifiedName ADD (IF NOT EXISTS)? PARTITION properties (LOCATION string) (PARTITION properties (LOCATION string)?)* #alterTableAddPartition
| ALTER TABLE qualifiedName PARTITION properties RENAME TO PARTITION properties #alterTableRenamePartition
| ALTER TABLE qualifiedName EXCHANGE PARTITION properties WITH TABLE qualifiedName #alterTableExchangePartition
| ALTER TABLE qualifiedName ADD (IF NOT EXISTS)? PARTITION partitionSpec (LOCATION string)? (PARTITION partitionSpec (LOCATION string)?)* #alterTableAddPartition
| ALTER TABLE qualifiedName PARTITION partitionSpec RENAME TO PARTITION partitionSpec #alterTableRenamePartition
| ALTER TABLE qualifiedName EXCHANGE PARTITION partitionSpec WITH TABLE qualifiedName #alterTableExchangePartition
| ALTER TABLE qualifiedName RECOVER PARTITIONS #alterTableRecoverPartitions
| ALTER TABLE qualifiedName DROP (IF EXISTS)? PARTITION properties (',' PARTITION properties)? (IGNORE PROTECTION)? PURGE? #alterTableDropPartition
| ALTER TABLE qualifiedName (ARCHIVE | UNARCHIVE) PARTITION properties #alterTableArchivePartition
| ALTER TABLE qualifiedName PARTITION properties SET FILEFORMAT identifier #alterTablePartitionFileFormat
| ALTER TABLE qualifiedName PARTITION properties SET LOCATION string #alterTablePartitionLocation
| ALTER TABLE qualifiedName TOUCH PARTITION properties #alterTablePartitionTouch
| ALTER TABLE qualifiedName PARTITION properties (ENABLE | DISABLE) (NO_DROP CASCADE? | OFFLINE) #alterTablePartitionProtections
| ALTER TABLE qualifiedName PARTITION properties COMPACT string (AND WAIT)? WITH OVERWRITE TBLPROPERTIES properties #alterTablePartitionCompact
| ALTER TABLE qualifiedName PARTITION properties CONCATENATE #alterTablePartitionConcatenate
| ALTER TABLE qualifiedName PARTITION properties UPDATE COLUMNS #alterTablePartitionUpdateColumns
| ALTER TABLE qualifiedName (PARTITION properties)? CHANGE COLUMN? oldName=identifier newName=identifier type
| ALTER TABLE qualifiedName DROP (IF EXISTS)? PARTITION partitionSpec (',' PARTITION partitionSpec)? (IGNORE PROTECTION)? PURGE? #alterTableDropPartition
| ALTER TABLE qualifiedName (ARCHIVE | UNARCHIVE) PARTITION partitionSpec #alterTableArchivePartition
| ALTER TABLE qualifiedName (PARTITION partitionSpec)? SET FILEFORMAT identifier #alterTablePartitionFileFormat
| ALTER TABLE qualifiedName (PARTITION partitionSpec) SET LOCATION string #alterTablePartitionLocation
| ALTER TABLE qualifiedName TOUCH (PARTITION partitionSpec)? #alterTablePartitionTouch
| ALTER TABLE qualifiedName PARTITION partitionSpec (ENABLE | DISABLE) (NO_DROP CASCADE? | OFFLINE) #alterTablePartitionProtections
| ALTER TABLE qualifiedName PARTITION partitionSpec COMPACT string (AND WAIT)? WITH OVERWRITE TBLPROPERTIES properties #alterTablePartitionCompact
| ALTER TABLE qualifiedName PARTITION partitionSpec CONCATENATE #alterTablePartitionConcatenate
| ALTER TABLE qualifiedName PARTITION partitionSpec UPDATE COLUMNS #alterTablePartitionUpdateColumns
| ALTER TABLE qualifiedName (PARTITION partitionSpec)? CHANGE COLUMN? oldName=identifier newName=identifier type
(COMMENT string)? (FIRST | AFTER columnName=identifier)? (CASCADE | RESTRICT)? #alterTableChangeColumn
| CREATE VIEW (IF NOT EXISTS)? qualifiedName viewColumns?
(COMMENT string)?
@ -164,18 +164,18 @@ statement
| RELOAD (FUNCTIONS | FUNCTION) #reloadFunctions
| SHOW FUNCTIONS
(LIKE pattern=string)? #showFunctions
| SET property? #setSession
| SET setProperty? #setSession
| RESET #resetSession
| CREATE MATERIALIZED VIEW (IF NOT EXISTS)? qualifiedName
createMaterializedViewOption* AS query #createMaterializedView
| DROP MATERIALIZED VIEW qualifiedName #dropMaterializedView
| ALTER MATERIALIZED VIEW qualifiedName (ENABLE | DISABLE) REWRITE #alterMaterializedView
| SHOW MATERIALIZED VIEWS ((IN | FROM) qualifiedName)? (LIKE pattern=string)? #showMaterializedViews
| CREATE INDEX identifier ON TABLE? identifier columnAliases
AS identifier createIndexOptions* #createIndex
| DROP INDEX (IF EXISTS)? identifier ON identifier #dropIndex
| ALTER INDEX identifier ON identifier (PARTITION properties)? REBUILD #alterIndex
| SHOW (FORMATTED)? (INDEX | INDEXES) ON identifier ((FROM | IN) qualifiedName)? #showIndex
| CREATE INDEX identifier ON TABLE? qualifiedName columnAliases
AS identifier createIndexOptions* #createIndex
| DROP INDEX (IF EXISTS)? identifier ON qualifiedName #dropIndex
| ALTER INDEX identifier ON qualifiedName (PARTITION properties)? REBUILD #alterIndex
| SHOW (FORMATTED)? (INDEX | INDEXES) ON qualifiedName ((FROM | IN) qualifiedName)? #showIndex
| SHOW PARTITIONS qualifiedName (PARTITION properties)?
(WHERE where=booleanExpression)?
(ORDER BY sortItem (',' sortItem)*)?
@ -188,19 +188,19 @@ statement
| (DESCRIBE | DESC) FUNCTION EXTENDED? functionName=expression #describeFunction
| CREATE TEMPORARY MACRO identifier '(' (tableElement (',' tableElement)*)? ')' expression #createMacro
| DROP TEMPORARY MACRO (IF EXISTS)? identifier #dropMacro
| SHOW LOCKS (DATABASE | SCHEMA)? identifier (PARTITION properties)? EXTENDED? #showLocks
| SHOW LOCKS (DATABASE | SCHEMA)? qualifiedName (PARTITION properties)? EXTENDED? #showLocks
| SHOW CONF identifier #showConf
| SHOW TRANSACTIONS #showTransactions
| SHOW COMPACTIONS #showCompactions
| ABORT TRANSACTIONS INTEGER_VALUE (INTEGER_VALUE)* #abortTransactions
| LOAD DATA LOCAL? INPATH identifier OVERWRITE? INTO TABLE identifier
| LOAD DATA LOCAL? INPATH identifier OVERWRITE? INTO TABLE qualifiedName
(PARTITION properties)? (INPUTFORMAT identifier SERDE identifier)? #loadData
| MERGE INTO qualifiedName AS T USING (qualifiedName | query) AS S ON booleanExpression
(WHEN MATCHED (AND booleanExpression)? THEN UPDATE SET property (',' property)*)?
(WHEN MATCHED (AND booleanExpression)? THEN DELETE)?
(WHEN NOT MATCHED (AND booleanExpression)? THEN INSERT VALUES expression (',' expression)* )? #merge
| EXPORT TABLE identifier (PARTITION properties)? TO string (FOR REPLICATION '(' identifier ')')? #exportData
| IMPORT (EXTERNAL? TABLE identifier (PARTITION properties)?)? FROM string (LOCATION string)? #importData
(WHEN NOT MATCHED (AND booleanExpression)? THEN INSERT VALUES expression (',' expression)* )? #merge
| EXPORT TABLE qualifiedName (PARTITION properties)? TO string (FOR REPLICATION '(' identifier ')')? #exportData
| IMPORT (EXTERNAL? TABLE qualifiedName (PARTITION properties)?)? FROM string (LOCATION string)? #importData
;
assignmentList
@ -224,6 +224,10 @@ createFunctionOption
: USING (JAR | FILE | ARCHIVE) expression (',' (JAR | FILE | ARCHIVE) expression)?
;
setProperty
: identifier ('.' identifier)* EQ expression
;
createMaterializedViewOption
: DISABLE REWRITE
| COMMENT string
@ -272,6 +276,10 @@ properties
: '(' property (',' property)* ')'
;
partitionSpec
: '(' identifier (EQ expression)? (',' identifier (EQ expression)?)* ')'
;
clusteredBy
: expression (',' expression)*
;

8
hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/impala/ImpalaSql.g4
View File

@ -141,7 +141,7 @@ statement
| COMMENT ON (DATABASE|TABLE|COLUMN) qualifiedName IS (string | NULL) #addComments
| EXPLAIN statement #explain
| SET (ALL | identifier EQ expression)? #setSession
| ':'SHUTDOWN '(' (string)? (':' expression)? (',' expression )? ')' #shutdown
| ':'SHUTDOWN '(' ('\\')? (expression)? (':' expression)? (',' expression )? ')' #shutdown
| INVALIDATE METADATA qualifiedName #invalidateMeta
| LOAD DATA INPATH STRING (OVERWRITE)? INTO TABLE qualifiedName
(PARTITION '('expression (',' expression)?')')? #loadData
@ -844,8 +844,8 @@ PERCENT: '%';
CONCAT: '||';
STRING
: '\'' ( ~'\'' | '\'\'' )* '\''
| '"' ( ~'"' | '""' )* '"'
: '\'' (~'\'' | '\\' | ('\\' '\''))* '\''
| '"' (~'"' | '\\' | '\\"')* '"'
;
UNICODE_STRING
@ -874,7 +874,7 @@ DOUBLE_VALUE
;
IDENTIFIER
: (LETTER | '_') (LETTER | DIGIT | '_' | '@' | ':')*
: (LETTER | '_') (LETTER | DIGIT | '_')*
;
DIGIT_IDENTIFIER

6
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/HiveAstBuilder.java
View File

@ -1436,9 +1436,9 @@ public class HiveAstBuilder
@Override
public Node visitSetSession(HiveSqlParser.SetSessionContext context)
{
if (context.property() != null) {
addDiif(DiffType.UNSUPPORTED, context.property().getText(), "[SET PROPERTY] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported to set session property", context.property());
if (context.setProperty() != null) {
addDiif(DiffType.UNSUPPORTED, context.setProperty().getText(), "[SET PROPERTY] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported to set session property", context.setProperty());
}
return new ShowSession(getLocation(context));

41
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/ImpalaAstBuilder.java
View File

@ -115,6 +115,7 @@ import io.prestosql.sql.tree.Select;
import io.prestosql.sql.tree.SelectItem;
import io.prestosql.sql.tree.ShowColumns;
import io.prestosql.sql.tree.ShowCreate;
import io.prestosql.sql.tree.ShowFunctions;
import io.prestosql.sql.tree.ShowRoles;
import io.prestosql.sql.tree.ShowSchemas;
import io.prestosql.sql.tree.ShowSession;
@ -320,9 +321,6 @@ public class ImpalaAstBuilder
Optional<String> comment = Optional.empty();
if (context.COMMENT() != null) {
comment = Optional.of(((StringLiteral) visit(context.comment)).getValue());
addDiif(DiffType.DELETED, context.COMMENT().getText(), null, null);
addDiif(DiffType.DELETED, comment.get(), null, format("[COMMENT] is omitted: %s", comment));
}
List<Property> properties = new ArrayList<>();
@ -466,9 +464,6 @@ public class ImpalaAstBuilder
Optional<String> comment = Optional.empty();
if (context.COMMENT() != null) {
comment = Optional.of(((StringLiteral) visit(context.comment)).getValue());
addDiif(DiffType.DELETED, context.COMMENT().getText(), null, null);
addDiif(DiffType.DELETED, comment.get(), null, format("[COMMENT] is omitted: %s", comment));
}
// like clause
@ -1059,8 +1054,24 @@ public class ImpalaAstBuilder
@Override
public Node visitShowFunctions(ImpalaSqlParser.ShowFunctionsContext context)
{
addDiif(DiffType.UNSUPPORTED, context.FUNCTIONS().getText(), "[SHOW FUNCTIONS] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "SHOW FUNCTIONS", context);
if (context.AGGREGATE() != null) {
addDiif(DiffType.UNSUPPORTED, context.AGGREGATE().getText(), "[AGGREGATE] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "AGGREGATE is not supported", context);
}
if (context.ANALYTIC() != null) {
addDiif(DiffType.UNSUPPORTED, context.ANALYTIC().getText(), "[ANALYTIC] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "ANALYTIC is not supported", context);
}
if (context.IN() != null) {
addDiif(DiffType.UNSUPPORTED, context.IN().getText(), "[IN] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "IN is not supported", context);
}
if (context.LIKE() != null) {
addDiif(DiffType.UNSUPPORTED, context.LIKE().getText(), "[LIKE] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "LIKE is not supported", context);
}
return new ShowFunctions(getLocation(context));
}
@Override
@ -1302,9 +1313,9 @@ public class ImpalaAstBuilder
}
@Override
public Node visitProperty(ImpalaSqlParser.PropertyContext ctx)
public Node visitProperty(ImpalaSqlParser.PropertyContext context)
{
return super.visitProperty(ctx);
return new Property(getLocation(context), (Identifier) visit(context.identifier()), (Expression) visit(context.expression()));
}
@Override
@ -1502,6 +1513,11 @@ public class ImpalaAstBuilder
Relation left = (Relation) visit(context.left);
Relation right;
if (context.CROSS() != null) {
right = (Relation) visit(context.right);
return new Join(getLocation(context), Join.Type.CROSS, left, right, Optional.empty());
}
if (context.joinType().SEMI() != null) {
addDiif(DiffType.UNSUPPORTED, context.joinType().SEMI().getText(), "[SEMI] is not supported");
throw unsupportedError(ErrorType.UNSUPPORTED_KEYWORDS, "SEMI", context);
@ -1518,11 +1534,6 @@ public class ImpalaAstBuilder
throw unsupportedError(ErrorType.UNSUPPORTED_KEYWORDS, "LEFT INNER || RIGHT INNER", context);
}
if (context.CROSS() != null) {
right = (Relation) visit(context.right);
return new Join(getLocation(context), Join.Type.CROSS, left, right, Optional.empty());
}
JoinCriteria criteria;
right = (Relation) visit(context.rightRelation);
if (context.joinCriteria().ON() != null) {

3
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/CliOptions.java
View File

@ -33,4 +33,7 @@ public class CliOptions
@Option(name = {"-c", "--config"}, title = "config", description = "Config file path")
public String configFile;
@Option(name = {"-d", "--debug"}, title = "debug", description = "Print Debug info")
public String debug;
}

6
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/Console.java
View File

@ -167,7 +167,8 @@ public class Console
}
}
}
session.setConsolePrintEnable(cliOptions.execute != null);
session.setDebugEnable(cliOptions.debug != null && cliOptions.debug.equalsIgnoreCase("true"));
session.setConsolePrintEnable(cliOptions.execute != null || session.isDebugEnable());
return executeCommand(query, outputFile, session);
}
@ -354,6 +355,9 @@ public class Console
StatementSplitter splitter = new StatementSplitter(query);
SqlSyntaxConverter sqlConverter = SqlConverterFactory.getSqlConverter(session);
for (StatementSplitter.Statement split : splitter.getCompleteStatements()) {
if (session.isDebugEnable()) {
log.info(String.format("Processing sql: %s", split.toString()));
}
JSONObject result = sqlConverter.convert(split.statement());
output.put(result);
if (session.isConsolePrintEnable()) {

11
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/SessionProperties.java
View File

@ -26,6 +26,7 @@ public class SessionProperties
private ParsingOptions parsingOptions;
private MigrationConfig migrationConfig;
private boolean isConsolePrintEnable;
private boolean isDebugEnable;
public SqlSyntaxType getSourceType()
{
@ -71,4 +72,14 @@ public class SessionProperties
{
this.migrationConfig = migrationConfig;
}
public boolean isDebugEnable()
{
return isDebugEnable;
}
public void setDebugEnable(boolean debugEnable)
{
isDebugEnable = debugEnable;
}
}

7
hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/util/SqlResultHandleUtils.java
View File

@ -63,12 +63,7 @@ public class SqlResultHandleUtils
htmlEscape(convertedSqls);
String htmlFileText = htmlTextTemplate.replaceAll("\\$\\{conversionResultData}\\$", convertedSqls.toString());
if (!"html".equalsIgnoreCase(outputFile.substring(outputFile.lastIndexOf(".")))) {
// if the output file specified by user is not end with html, then append the html suffix
outputFile += ".html";
}
OutputStream out = new FileOutputStream(outputFile);
OutputStream out = new FileOutputStream(outputFile + ".html");
writer = new BufferedWriter(new OutputStreamWriter(out, UTF_8), BUFFER_SIZE);
writer.write(htmlFileText);
}

18
hetu-sql-migration-tool/src/main/resources/htmlResultTemplate.html
View File

@ -552,38 +552,40 @@ function formatSqlDiffs(entry) {
for (let diff of entry.diffs) {
if (diff.message && diff.message.length > 0) {
msgText += '<li>' + diff.message + '</li>';
let formattedMsg = diff.message.replace(/</g, '&lt;');
formattedMsg = diff.message.replace(/>/g, '&gt;');
msgText += '<li>' + formattedMsg + '</li>';
}
switch (diff.diffType) {
case 'deleted':
if (diff.source && diff.source.length > 0) {
originalSql = originalSql.replace(diff.source, '<span style="color:red"><s>' + diff.source + '</s></span>')
originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '<span style="color:red"><s>' + diff.source + '</s></span>')
}
break;
case 'modified':
if (diff.source && diff.source.length > 0) {
originalSql = originalSql.replace(diff.source, '<span style="color:blue">' + diff.source + '</span>')
originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '<span style="color:blue">' + diff.source + '</span>')
}
if (diff.target && diff.target.length > 0) {
convertedSql = convertedSql.replace(diff.target, '<span style="color:blue">' + diff.target + '</span>')
convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '<span style="color:blue">' + diff.target + '</span>')
}
break;
case 'inserted':
if (diff.target && diff.target.length > 0) {
convertedSql = convertedSql.replace(diff.target, '<span style="color:green">' + diff.target + '</span>')
convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '<span style="color:green">' + diff.target + '</span>')
}
break;
case 'warning':
if (diff.source && diff.source.length > 0) {
originalSql = originalSql.replace(diff.source, '<span style="color:#ffcc00">' + diff.source + '</span>')
originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '<span style="color:#ffcc00">' + diff.source + '</span>')
}
if (diff.target && diff.target.length > 0) {
convertedSql = convertedSql.replace(diff.target, '<span style="color:#ffcc00">' + diff.target + '</span>')
convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '<span style="color:#ffcc00">' + diff.target + '</span>')
}
break;
case 'unsupported':
if (diff.source && diff.source.length > 0) {
originalSql = originalSql.replace(diff.source, '<span style="color:#993333">' + diff.source + '</span>')
originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '<span style="color:#993333">' + diff.source + '</span>')
}
break;
default:

9
hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestHiveSqlMigrate.java
View File

@ -314,7 +314,7 @@ public class TestHiveSqlMigrate
String sql9 = "ALTER TABLE T1 ADD CONSTRAINT TEST PRIMARY KEY (ID, NAME) DISABLE NOVALIDATE";
assertUnsupported(sql9, Optional.of("ADD CONSTRAINT"));
String sql10 = "ALTER TABLE T1 CHANGE COLUMN ID1 ID2 INT CONSTRAINT TEST NOT NULL ENABLE";
String sql10 = "ALTER TABLE FOO PARTITION (DS='2008-04-08', HR) CHANGE COLUMN DEC_COLUMN_NAME DEC_COLUMN_NAME DECIMAL(38,18)";
assertUnsupported(sql10, Optional.of("CHANGE COLUMN"));
String sql11 = "ALTER TABLE T1 DROP CONSTRAINT TEST";
@ -730,6 +730,13 @@ public class TestHiveSqlMigrate
assertUnsupported(sql, Optional.of("LOAD DATA"));
}
@Test
public void testSetSession()
{
String sql = "set hive.support.concurrency=true";
assertUnsupported(sql, Optional.of("SET PROPERTY"));
}
@Test
public void testResetSession()
{

52
hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestImpalaSqlMigrate.java
View File

@ -135,7 +135,10 @@ public class TestImpalaSqlMigrate
" format = 'Parquet',\n" +
" location = '/user/tmp'\n" +
")";
assertWarning(sql2, expectedSql2);
assertSuccess(sql2, expectedSql2);
String sql3 = "create table impala10(impala1 tinyint, impala2 smallint, impala3 int) TBLPROPERTIES('id'='11','cnt'=1)";
assertUnsupported(sql3, Optional.of("TBLPROPERTIES"));
// Test negative sql - row format is not supported
String negativeSql1 = "CREATE TABLE tbl_row_format (id INT, name STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY 'char'";
@ -180,7 +183,7 @@ public class TestImpalaSqlMigrate
")";
assertSuccess(sql2, expectedSql2);
String negativeSql3 = "CREATE TABLE tb3 (c1 STRUCT <employer : STRING, id : BIGINT, address : STRING>)";
String negativeSql3 = "CREATE TABLE tb3 (c1 STRUCT <employer:STRING, id : BIGINT, address : STRING>)";
assertUnsupported(negativeSql3, Optional.of("STRUCT"));
}
@ -208,7 +211,7 @@ public class TestImpalaSqlMigrate
" location = '/USER/TEST',\n" +
" format = 'Avro'\n" +
")";
assertWarning(sql, expectedSql);
assertSuccess(sql, expectedSql);
String sql2 = "CREATE EXTERNAL TABLE T2 LIKE PARQUET '/user/test/impala' COMMENT 'HETU' STORED AS AVRO LOCATION '/USER/TEST'";
assertUnsupported(sql2, Optional.of("PARQUET"));
@ -286,6 +289,39 @@ public class TestImpalaSqlMigrate
"HAVING (\"max\"(salary) > 110)\n" +
"LIMIT 5\n";
assertSuccess(sql4, expectedSql4);
// teet join
String sql5 = "SELECT * FROM t1 JOIN t2 ON t1.id = t2.id";
String expectedSql5 = "SELECT *\n" +
"FROM\n" +
" (t1\n" +
"INNER JOIN t2 ON (t1.id = t2.id))\n";
assertSuccess(sql5, expectedSql5);
String sql6 = "SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id";
assertSuccess(sql6, expectedSql5);
String sql7 = "SELECT * FROM t1 FULL OUTER JOIN t2 ON t1.id = t2.id";
String expectedSql7 = "SELECT *\n" +
"FROM\n" +
" (t1\n" +
"FULL JOIN t2 ON (t1.id = t2.id))\n";
assertSuccess(sql7, expectedSql7);
String sql8 = "SELECT * FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.id";
String expectedSql8 = "SELECT *\n" +
"FROM\n" +
" (t1\n" +
"LEFT JOIN t2 ON (t1.id = t2.id))\n";
assertSuccess(sql8, expectedSql8);
String sql9 = "SELECT * FROM t1 CROSS JOIN t2 WHERE t1.id > t2.id";
String expectedSql9 = "SELECT *\n" +
"FROM\n" +
" (t1\n" +
"CROSS JOIN t2)\n" +
"WHERE (t1.id > t2.id)\n";
assertSuccess(sql9, expectedSql9);
}
@Test
@ -642,7 +678,9 @@ public class TestImpalaSqlMigrate
public void testShutDown()
{
String sql = ":SHUTDOWN('LOCALHOST' : 8090, 0)";
String sql2 = ":SHUTDOWN(\\\"hostname:1234\\\")";
assertUnsupported(sql, Optional.of("SHUTDOWN"));
assertUnsupported(sql2, Optional.of("SHUTDOWN"));
}
@Test
@ -794,9 +832,11 @@ public class TestImpalaSqlMigrate
String sql = "SHOW FUNCTIONS IN DB1";
String sql2 = "SHOW AGGREGATE FUNCTIONS IN DB1";
String sql3 = "SHOW ANALYTIC FUNCTIONS IN DB1";
assertUnsupported(sql, Optional.of("FUNCTIONS"));
assertUnsupported(sql2, Optional.of("FUNCTIONS"));
assertUnsupported(sql3, Optional.of("FUNCTIONS"));
String sql4 = "SHOW FUNCTIONS";
assertUnsupported(sql, Optional.of("IN"));
assertUnsupported(sql2, Optional.of("AGGREGATE"));
assertUnsupported(sql3, Optional.of("ANALYTIC"));
assertSuccess(sql4, "SHOW FUNCTIONS");
}
@Test