From bf8f5136decf6e5f515d18af2de4d6df18faec7a Mon Sep 17 00:00:00 2001 From: Jiwen Date: Sat, 5 Sep 2020 18:13:36 +0800 Subject: [PATCH] Fix sql-migration-tool bugs on hive and impala. include, syntax incorrection and ui bugs. --- hetu-docs/en/migration/sql-migration-tool.md | 3 +- hetu-docs/zh/migration/sql-migration-tool.md | 3 +- .../core/migration/source/hive/HiveSql.g4 | 62 +++++++++++-------- .../core/migration/source/impala/ImpalaSql.g4 | 8 +-- .../sql/migration/parser/HiveAstBuilder.java | 6 +- .../migration/parser/ImpalaAstBuilder.java | 41 +++++++----- .../core/sql/migration/tool/CliOptions.java | 3 + .../hetu/core/sql/migration/tool/Console.java | 6 +- .../sql/migration/tool/SessionProperties.java | 11 ++++ .../core/sql/util/SqlResultHandleUtils.java | 7 +-- .../main/resources/htmlResultTemplate.html | 18 +++--- .../migration/tool/TestHiveSqlMigrate.java | 9 ++- .../migration/tool/TestImpalaSqlMigrate.java | 52 ++++++++++++++-- 13 files changed, 156 insertions(+), 73 deletions(-) diff --git a/hetu-docs/en/migration/sql-migration-tool.md b/hetu-docs/en/migration/sql-migration-tool.md index 4b4cde6b0..60c00d430 100644 --- a/hetu-docs/en/migration/sql-migration-tool.md +++ b/hetu-docs/en/migration/sql-migration-tool.md @@ -52,6 +52,7 @@ This tool also can take parameters and running in batch mode. It has five parame | `--type` or `-t` | The type of input SQL statement, such as `hive`, `impala`. It\'s optional parameter and the default value is `hive`. | | `--output` or `-o` | the directory to save the converted SQL results. The result file's naming convention will be the input file's name + timestamp + .html suffix. | | `--config` or `-c` | the config file of SQL Migration Tool. | +| `--debug` or `-d` | if set value to 'true', then print the debug information in console. | *Tip:* @@ -237,6 +238,7 @@ Below Impala statements are partially supported, which mean some keywords or att | ADD COMMENTS | Adding comments to databases or columns is not supported | [COMMENT](../sql/comment.md) | | SET SESSION | Only support "SET" and "SET ALL" | [SET SESSION](../sql/set-session.md) | | ADD COLUMNS | ADD multiple columns within single statement is not supported, kudu properties are not supported. | [ALTER TABLE](../sql/alter-table.md) | +| SHOW FUNCTIONS | Only support show all functions. | [SHOW FUNCTIONS](../sql/show-functions.md) | Below Impala statements are not supported, because of feature differences: @@ -258,7 +260,6 @@ Below Impala statements are not supported, because of feature differences: | REFRESH FUNCTION | | UPDATE TABLE | | UPSERT | -| SHOW FUNCTIONS | | SHOW TABLE/COLUMN STATS | | SHOW PARTITIONS | | SHOW FILES | diff --git a/hetu-docs/zh/migration/sql-migration-tool.md b/hetu-docs/zh/migration/sql-migration-tool.md index 8ea5086f7..73df5fcf2 100644 --- a/hetu-docs/zh/migration/sql-migration-tool.md +++ b/hetu-docs/zh/migration/sql-migration-tool.md @@ -49,6 +49,7 @@ INSERT INTO table1 | `--type` or `-t`| 输入SQL语句的类型，如`hive`、`impala`。可选参数，默认值为`hive`。| | `--output` or `-o`| 转换后SQL结果的存放目录。| | `--config` or `-c`| SQL迁移工具的配置文件。| +| `--debug` or `-d`| 需要打印debug到控制界面时，设置值为true。| *提示：* @@ -233,6 +234,7 @@ SELECT (DECIMAL '2.0' * 3) | ADD COMMENTS | 不支持给数据库和列添加评论 | [COMMENT](../sql/comment.md) | | SET SESSION | 仅支持 "SET" 和 "SET ALL" | [SET SESSION](../sql/set-session.md) | | ADD COLUMNS | 不支持在一条语句添加多列，也不支持设置kudu属性 | [ALTER TABLE](../sql/alter-table.md) | +| SHOW FUNCTIONS | 仅支持显示全部的函数 | [SHOW FUNCTIONS](../sql/show-functions.md) | 由于特性差异，如下Impala语句暂不支持： @@ -254,7 +256,6 @@ SELECT (DECIMAL '2.0' * 3) | REFRESH FUNCTION | | UPDATE TABLE | | UPSERT | -| SHOW FUNCTIONS | | SHOW TABLE/COLUMN STATS | | SHOW PARTITIONS | | SHOW FILES | diff --git a/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/hive/HiveSql.g4 b/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/hive/HiveSql.g4 index 3950e9f8f..f171e4c14 100644 --- a/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/hive/HiveSql.g4 +++ b/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/hive/HiveSql.g4 @@ -78,9 +78,9 @@ statement | DELETE FROM qualifiedName (WHERE booleanExpression)? #delete | ALTER TABLE from=qualifiedName RENAME TO to=qualifiedName #renameTable | ALTER TABLE qualifiedName SET TBLPROPERTIES properties #commentTable - | ALTER TABLE tableName=qualifiedName (PARTITION partition=properties)? + | ALTER TABLE tableName=qualifiedName (PARTITION partition=partitionSpec)? (ADD | REPLACE) COLUMNS '('tableElement (',' tableElement)*')' (CASCADE | RESTRICT)? #addReplaceColumn - | ALTER TABLE qualifiedName (PARTITION properties)? (SET SERDE string (WITH SERDEPROPERTIES properties)? | SET SERDEPROPERTIES properties) #alterTableSerde + | ALTER TABLE qualifiedName (PARTITION partitionSpec)? (SET SERDE string (WITH SERDEPROPERTIES properties)? | SET SERDEPROPERTIES properties) #alterTableSerde | ALTER TABLE qualifiedName CLUSTERED BY columnAliases (SORTED BY columnAliases)? INTO expression BUCKETS #alterTableStorage | ALTER TABLE qualifiedName SKEWED BY columnAliases ON expression (',' expression)* (STORED_AS DIRECTORIES)? #alterTableSkewed | ALTER TABLE qualifiedName NOT SKEWED #alterTableNotSkewed @@ -88,25 +88,25 @@ statement | ALTER TABLE qualifiedName SET SKEWED LOCATION properties #alterTableSetSkewedLocation | ALTER TABLE qualifiedName ADD CONSTRAINT identifier (PRIMARY KEY columnAliases DISABLE NOVALIDATE - | FOREIGN KEY columnAliases REFERENCES identifier columnAliases DISABLE NOVALIDATE RELY + | FOREIGN KEY columnAliases REFERENCES qualifiedName columnAliases DISABLE NOVALIDATE RELY | UNIQUE columnAliases DISABLE NOVALIDATE) #alterTableAddConstraint | ALTER TABLE qualifiedName CHANGE COLUMN identifier identifier type CONSTRAINT identifier (NOT NULL ENABLE | DEFAULT defaultValue ENABLE | CHECK expression ENABLE) #alterTableChangeConstraint | ALTER TABLE qualifiedName DROP CONSTRAINT identifier #alterTableDropConstraint - | ALTER TABLE qualifiedName ADD (IF NOT EXISTS)? PARTITION properties (LOCATION string) (PARTITION properties (LOCATION string)?)* #alterTableAddPartition - | ALTER TABLE qualifiedName PARTITION properties RENAME TO PARTITION properties #alterTableRenamePartition - | ALTER TABLE qualifiedName EXCHANGE PARTITION properties WITH TABLE qualifiedName #alterTableExchangePartition + | ALTER TABLE qualifiedName ADD (IF NOT EXISTS)? PARTITION partitionSpec (LOCATION string)? (PARTITION partitionSpec (LOCATION string)?)* #alterTableAddPartition + | ALTER TABLE qualifiedName PARTITION partitionSpec RENAME TO PARTITION partitionSpec #alterTableRenamePartition + | ALTER TABLE qualifiedName EXCHANGE PARTITION partitionSpec WITH TABLE qualifiedName #alterTableExchangePartition | ALTER TABLE qualifiedName RECOVER PARTITIONS #alterTableRecoverPartitions - | ALTER TABLE qualifiedName DROP (IF EXISTS)? PARTITION properties (',' PARTITION properties)? (IGNORE PROTECTION)? PURGE? #alterTableDropPartition - | ALTER TABLE qualifiedName (ARCHIVE | UNARCHIVE) PARTITION properties #alterTableArchivePartition - | ALTER TABLE qualifiedName PARTITION properties SET FILEFORMAT identifier #alterTablePartitionFileFormat - | ALTER TABLE qualifiedName PARTITION properties SET LOCATION string #alterTablePartitionLocation - | ALTER TABLE qualifiedName TOUCH PARTITION properties #alterTablePartitionTouch - | ALTER TABLE qualifiedName PARTITION properties (ENABLE | DISABLE) (NO_DROP CASCADE? | OFFLINE) #alterTablePartitionProtections - | ALTER TABLE qualifiedName PARTITION properties COMPACT string (AND WAIT)? WITH OVERWRITE TBLPROPERTIES properties #alterTablePartitionCompact - | ALTER TABLE qualifiedName PARTITION properties CONCATENATE #alterTablePartitionConcatenate - | ALTER TABLE qualifiedName PARTITION properties UPDATE COLUMNS #alterTablePartitionUpdateColumns - | ALTER TABLE qualifiedName (PARTITION properties)? CHANGE COLUMN? oldName=identifier newName=identifier type + | ALTER TABLE qualifiedName DROP (IF EXISTS)? PARTITION partitionSpec (',' PARTITION partitionSpec)? (IGNORE PROTECTION)? PURGE? #alterTableDropPartition + | ALTER TABLE qualifiedName (ARCHIVE | UNARCHIVE) PARTITION partitionSpec #alterTableArchivePartition + | ALTER TABLE qualifiedName (PARTITION partitionSpec)? SET FILEFORMAT identifier #alterTablePartitionFileFormat + | ALTER TABLE qualifiedName (PARTITION partitionSpec) SET LOCATION string #alterTablePartitionLocation + | ALTER TABLE qualifiedName TOUCH (PARTITION partitionSpec)? #alterTablePartitionTouch + | ALTER TABLE qualifiedName PARTITION partitionSpec (ENABLE | DISABLE) (NO_DROP CASCADE? | OFFLINE) #alterTablePartitionProtections + | ALTER TABLE qualifiedName PARTITION partitionSpec COMPACT string (AND WAIT)? WITH OVERWRITE TBLPROPERTIES properties #alterTablePartitionCompact + | ALTER TABLE qualifiedName PARTITION partitionSpec CONCATENATE #alterTablePartitionConcatenate + | ALTER TABLE qualifiedName PARTITION partitionSpec UPDATE COLUMNS #alterTablePartitionUpdateColumns + | ALTER TABLE qualifiedName (PARTITION partitionSpec)? CHANGE COLUMN? oldName=identifier newName=identifier type (COMMENT string)? (FIRST | AFTER columnName=identifier)? (CASCADE | RESTRICT)? #alterTableChangeColumn | CREATE VIEW (IF NOT EXISTS)? qualifiedName viewColumns? (COMMENT string)? @@ -164,18 +164,18 @@ statement | RELOAD (FUNCTIONS | FUNCTION) #reloadFunctions | SHOW FUNCTIONS (LIKE pattern=string)? #showFunctions - | SET property? #setSession + | SET setProperty? #setSession | RESET #resetSession | CREATE MATERIALIZED VIEW (IF NOT EXISTS)? qualifiedName createMaterializedViewOption* AS query #createMaterializedView | DROP MATERIALIZED VIEW qualifiedName #dropMaterializedView | ALTER MATERIALIZED VIEW qualifiedName (ENABLE | DISABLE) REWRITE #alterMaterializedView | SHOW MATERIALIZED VIEWS ((IN | FROM) qualifiedName)? (LIKE pattern=string)? #showMaterializedViews - | CREATE INDEX identifier ON TABLE? identifier columnAliases - AS identifier createIndexOptions* #createIndex - | DROP INDEX (IF EXISTS)? identifier ON identifier #dropIndex - | ALTER INDEX identifier ON identifier (PARTITION properties)? REBUILD #alterIndex - | SHOW (FORMATTED)? (INDEX | INDEXES) ON identifier ((FROM | IN) qualifiedName)? #showIndex + | CREATE INDEX identifier ON TABLE? qualifiedName columnAliases + AS identifier createIndexOptions* #createIndex + | DROP INDEX (IF EXISTS)? identifier ON qualifiedName #dropIndex + | ALTER INDEX identifier ON qualifiedName (PARTITION properties)? REBUILD #alterIndex + | SHOW (FORMATTED)? (INDEX | INDEXES) ON qualifiedName ((FROM | IN) qualifiedName)? #showIndex | SHOW PARTITIONS qualifiedName (PARTITION properties)? (WHERE where=booleanExpression)? (ORDER BY sortItem (',' sortItem)*)? @@ -188,19 +188,19 @@ statement | (DESCRIBE | DESC) FUNCTION EXTENDED? functionName=expression #describeFunction | CREATE TEMPORARY MACRO identifier '(' (tableElement (',' tableElement)*)? ')' expression #createMacro | DROP TEMPORARY MACRO (IF EXISTS)? identifier #dropMacro - | SHOW LOCKS (DATABASE | SCHEMA)? identifier (PARTITION properties)? EXTENDED? #showLocks + | SHOW LOCKS (DATABASE | SCHEMA)? qualifiedName (PARTITION properties)? EXTENDED? #showLocks | SHOW CONF identifier #showConf | SHOW TRANSACTIONS #showTransactions | SHOW COMPACTIONS #showCompactions | ABORT TRANSACTIONS INTEGER_VALUE (INTEGER_VALUE)* #abortTransactions - | LOAD DATA LOCAL? INPATH identifier OVERWRITE? INTO TABLE identifier + | LOAD DATA LOCAL? INPATH identifier OVERWRITE? INTO TABLE qualifiedName (PARTITION properties)? (INPUTFORMAT identifier SERDE identifier)? #loadData | MERGE INTO qualifiedName AS T USING (qualifiedName | query) AS S ON booleanExpression (WHEN MATCHED (AND booleanExpression)? THEN UPDATE SET property (',' property)*)? (WHEN MATCHED (AND booleanExpression)? THEN DELETE)? - (WHEN NOT MATCHED (AND booleanExpression)? THEN INSERT VALUES expression (',' expression)* )? #merge - | EXPORT TABLE identifier (PARTITION properties)? TO string (FOR REPLICATION '(' identifier ')')? #exportData - | IMPORT (EXTERNAL? TABLE identifier (PARTITION properties)?)? FROM string (LOCATION string)? #importData + (WHEN NOT MATCHED (AND booleanExpression)? THEN INSERT VALUES expression (',' expression)* )? #merge + | EXPORT TABLE qualifiedName (PARTITION properties)? TO string (FOR REPLICATION '(' identifier ')')? #exportData + | IMPORT (EXTERNAL? TABLE qualifiedName (PARTITION properties)?)? FROM string (LOCATION string)? #importData ; assignmentList @@ -224,6 +224,10 @@ createFunctionOption : USING (JAR | FILE | ARCHIVE) expression (',' (JAR | FILE | ARCHIVE) expression)? ; +setProperty + : identifier ('.' identifier)* EQ expression + ; + createMaterializedViewOption : DISABLE REWRITE | COMMENT string @@ -272,6 +276,10 @@ properties : '(' property (',' property)* ')' ; +partitionSpec + : '(' identifier (EQ expression)? (',' identifier (EQ expression)?)* ')' + ; + clusteredBy : expression (',' expression)* ; diff --git a/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/impala/ImpalaSql.g4 b/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/impala/ImpalaSql.g4 index c3c284473..d27ca456c 100644 --- a/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/impala/ImpalaSql.g4 +++ b/hetu-sql-migration-tool/src/main/antlr4/io/hetu/core/migration/source/impala/ImpalaSql.g4 @@ -141,7 +141,7 @@ statement | COMMENT ON (DATABASE|TABLE|COLUMN) qualifiedName IS (string | NULL) #addComments | EXPLAIN statement #explain | SET (ALL | identifier EQ expression)? #setSession - | ':'SHUTDOWN '(' (string)? (':' expression)? (',' expression )? ')' #shutdown + | ':'SHUTDOWN '(' ('\\')? (expression)? (':' expression)? (',' expression )? ')' #shutdown | INVALIDATE METADATA qualifiedName #invalidateMeta | LOAD DATA INPATH STRING (OVERWRITE)? INTO TABLE qualifiedName (PARTITION '('expression (',' expression)?')')? #loadData @@ -844,8 +844,8 @@ PERCENT: '%'; CONCAT: '||'; STRING - : '\'' ( ~'\'' | '\'\'' )* '\'' - | '"' ( ~'"' | '""' )* '"' + : '\'' (~'\'' | '\\' | ('\\' '\''))* '\'' + | '"' (~'"' | '\\' | '\\"')* '"' ; UNICODE_STRING @@ -874,7 +874,7 @@ DOUBLE_VALUE ; IDENTIFIER - : (LETTER | '_') (LETTER | DIGIT | '_' | '@' | ':')* + : (LETTER | '_') (LETTER | DIGIT | '_')* ; DIGIT_IDENTIFIER diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/HiveAstBuilder.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/HiveAstBuilder.java index df5cae8bd..250ce2618 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/HiveAstBuilder.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/HiveAstBuilder.java @@ -1436,9 +1436,9 @@ public class HiveAstBuilder @Override public Node visitSetSession(HiveSqlParser.SetSessionContext context) { - if (context.property() != null) { - addDiif(DiffType.UNSUPPORTED, context.property().getText(), "[SET PROPERTY] is not supported"); - throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported to set session property", context.property()); + if (context.setProperty() != null) { + addDiif(DiffType.UNSUPPORTED, context.setProperty().getText(), "[SET PROPERTY] is not supported"); + throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "Unsupported to set session property", context.setProperty()); } return new ShowSession(getLocation(context)); diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/ImpalaAstBuilder.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/ImpalaAstBuilder.java index 40e73e1a8..3d0d77e02 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/ImpalaAstBuilder.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/parser/ImpalaAstBuilder.java @@ -115,6 +115,7 @@ import io.prestosql.sql.tree.Select; import io.prestosql.sql.tree.SelectItem; import io.prestosql.sql.tree.ShowColumns; import io.prestosql.sql.tree.ShowCreate; +import io.prestosql.sql.tree.ShowFunctions; import io.prestosql.sql.tree.ShowRoles; import io.prestosql.sql.tree.ShowSchemas; import io.prestosql.sql.tree.ShowSession; @@ -320,9 +321,6 @@ public class ImpalaAstBuilder Optional comment = Optional.empty(); if (context.COMMENT() != null) { comment = Optional.of(((StringLiteral) visit(context.comment)).getValue()); - - addDiif(DiffType.DELETED, context.COMMENT().getText(), null, null); - addDiif(DiffType.DELETED, comment.get(), null, format("[COMMENT] is omitted: %s", comment)); } List properties = new ArrayList<>(); @@ -466,9 +464,6 @@ public class ImpalaAstBuilder Optional comment = Optional.empty(); if (context.COMMENT() != null) { comment = Optional.of(((StringLiteral) visit(context.comment)).getValue()); - - addDiif(DiffType.DELETED, context.COMMENT().getText(), null, null); - addDiif(DiffType.DELETED, comment.get(), null, format("[COMMENT] is omitted: %s", comment)); } // like clause @@ -1059,8 +1054,24 @@ public class ImpalaAstBuilder @Override public Node visitShowFunctions(ImpalaSqlParser.ShowFunctionsContext context) { - addDiif(DiffType.UNSUPPORTED, context.FUNCTIONS().getText(), "[SHOW FUNCTIONS] is not supported"); - throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "SHOW FUNCTIONS", context); + if (context.AGGREGATE() != null) { + addDiif(DiffType.UNSUPPORTED, context.AGGREGATE().getText(), "[AGGREGATE] is not supported"); + throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "AGGREGATE is not supported", context); + } + if (context.ANALYTIC() != null) { + addDiif(DiffType.UNSUPPORTED, context.ANALYTIC().getText(), "[ANALYTIC] is not supported"); + throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "ANALYTIC is not supported", context); + } + if (context.IN() != null) { + addDiif(DiffType.UNSUPPORTED, context.IN().getText(), "[IN] is not supported"); + throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "IN is not supported", context); + } + if (context.LIKE() != null) { + addDiif(DiffType.UNSUPPORTED, context.LIKE().getText(), "[LIKE] is not supported"); + throw unsupportedError(ErrorType.UNSUPPORTED_STATEMENT, "LIKE is not supported", context); + } + + return new ShowFunctions(getLocation(context)); } @Override @@ -1302,9 +1313,9 @@ public class ImpalaAstBuilder } @Override - public Node visitProperty(ImpalaSqlParser.PropertyContext ctx) + public Node visitProperty(ImpalaSqlParser.PropertyContext context) { - return super.visitProperty(ctx); + return new Property(getLocation(context), (Identifier) visit(context.identifier()), (Expression) visit(context.expression())); } @Override @@ -1502,6 +1513,11 @@ public class ImpalaAstBuilder Relation left = (Relation) visit(context.left); Relation right; + if (context.CROSS() != null) { + right = (Relation) visit(context.right); + return new Join(getLocation(context), Join.Type.CROSS, left, right, Optional.empty()); + } + if (context.joinType().SEMI() != null) { addDiif(DiffType.UNSUPPORTED, context.joinType().SEMI().getText(), "[SEMI] is not supported"); throw unsupportedError(ErrorType.UNSUPPORTED_KEYWORDS, "SEMI", context); @@ -1518,11 +1534,6 @@ public class ImpalaAstBuilder throw unsupportedError(ErrorType.UNSUPPORTED_KEYWORDS, "LEFT INNER || RIGHT INNER", context); } - if (context.CROSS() != null) { - right = (Relation) visit(context.right); - return new Join(getLocation(context), Join.Type.CROSS, left, right, Optional.empty()); - } - JoinCriteria criteria; right = (Relation) visit(context.rightRelation); if (context.joinCriteria().ON() != null) { diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/CliOptions.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/CliOptions.java index a5a8c7a6d..35824d8ce 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/CliOptions.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/CliOptions.java @@ -33,4 +33,7 @@ public class CliOptions @Option(name = {"-c", "--config"}, title = "config", description = "Config file path") public String configFile; + + @Option(name = {"-d", "--debug"}, title = "debug", description = "Print Debug info") + public String debug; } diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/Console.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/Console.java index 08c012a31..eb9dc64eb 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/Console.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/Console.java @@ -167,7 +167,8 @@ public class Console } } } - session.setConsolePrintEnable(cliOptions.execute != null); + session.setDebugEnable(cliOptions.debug != null && cliOptions.debug.equalsIgnoreCase("true")); + session.setConsolePrintEnable(cliOptions.execute != null || session.isDebugEnable()); return executeCommand(query, outputFile, session); } @@ -354,6 +355,9 @@ public class Console StatementSplitter splitter = new StatementSplitter(query); SqlSyntaxConverter sqlConverter = SqlConverterFactory.getSqlConverter(session); for (StatementSplitter.Statement split : splitter.getCompleteStatements()) { + if (session.isDebugEnable()) { + log.info(String.format("Processing sql: %s", split.toString())); + } JSONObject result = sqlConverter.convert(split.statement()); output.put(result); if (session.isConsolePrintEnable()) { diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/SessionProperties.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/SessionProperties.java index 51f345395..c6691c00a 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/SessionProperties.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/migration/tool/SessionProperties.java @@ -26,6 +26,7 @@ public class SessionProperties private ParsingOptions parsingOptions; private MigrationConfig migrationConfig; private boolean isConsolePrintEnable; + private boolean isDebugEnable; public SqlSyntaxType getSourceType() { @@ -71,4 +72,14 @@ public class SessionProperties { this.migrationConfig = migrationConfig; } + + public boolean isDebugEnable() + { + return isDebugEnable; + } + + public void setDebugEnable(boolean debugEnable) + { + isDebugEnable = debugEnable; + } } diff --git a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/util/SqlResultHandleUtils.java b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/util/SqlResultHandleUtils.java index 0bb07debd..1a1b08cb5 100644 --- a/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/util/SqlResultHandleUtils.java +++ b/hetu-sql-migration-tool/src/main/java/io/hetu/core/sql/util/SqlResultHandleUtils.java @@ -63,12 +63,7 @@ public class SqlResultHandleUtils htmlEscape(convertedSqls); String htmlFileText = htmlTextTemplate.replaceAll("\\$\\{conversionResultData}\\$", convertedSqls.toString()); - if (!"html".equalsIgnoreCase(outputFile.substring(outputFile.lastIndexOf(".")))) { - // if the output file specified by user is not end with html, then append the html suffix - outputFile += ".html"; - } - - OutputStream out = new FileOutputStream(outputFile); + OutputStream out = new FileOutputStream(outputFile + ".html"); writer = new BufferedWriter(new OutputStreamWriter(out, UTF_8), BUFFER_SIZE); writer.write(htmlFileText); } diff --git a/hetu-sql-migration-tool/src/main/resources/htmlResultTemplate.html b/hetu-sql-migration-tool/src/main/resources/htmlResultTemplate.html index 3ff145363..989ddac68 100644 --- a/hetu-sql-migration-tool/src/main/resources/htmlResultTemplate.html +++ b/hetu-sql-migration-tool/src/main/resources/htmlResultTemplate.html @@ -552,38 +552,40 @@ function formatSqlDiffs(entry) { for (let diff of entry.diffs) { if (diff.message && diff.message.length > 0) { - msgText += '
  • ' + diff.message + '
    • '; + let formattedMsg = diff.message.replace(//g, '>'); + msgText += '
  • ' + formattedMsg + '
    • '; } switch (diff.diffType) { case 'deleted': if (diff.source && diff.source.length > 0) { - originalSql = originalSql.replace(diff.source, '' + diff.source + '') + originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '' + diff.source + '') } break; case 'modified': if (diff.source && diff.source.length > 0) { - originalSql = originalSql.replace(diff.source, '' + diff.source + '') + originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '' + diff.source + '') } if (diff.target && diff.target.length > 0) { - convertedSql = convertedSql.replace(diff.target, '' + diff.target + '') + convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '' + diff.target + '') } break; case 'inserted': if (diff.target && diff.target.length > 0) { - convertedSql = convertedSql.replace(diff.target, '' + diff.target + '') + convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '' + diff.target + '') } break; case 'warning': if (diff.source && diff.source.length > 0) { - originalSql = originalSql.replace(diff.source, '' + diff.source + '') + originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '' + diff.source + '') } if (diff.target && diff.target.length > 0) { - convertedSql = convertedSql.replace(diff.target, '' + diff.target + '') + convertedSql = convertedSql.replace(new RegExp(diff.target,'gm'), '' + diff.target + '') } break; case 'unsupported': if (diff.source && diff.source.length > 0) { - originalSql = originalSql.replace(diff.source, '' + diff.source + '') + originalSql = originalSql.replace(new RegExp(diff.source,'gm'), '' + diff.source + '') } break; default: diff --git a/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestHiveSqlMigrate.java b/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestHiveSqlMigrate.java index 452f20303..89562057a 100644 --- a/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestHiveSqlMigrate.java +++ b/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestHiveSqlMigrate.java @@ -314,7 +314,7 @@ public class TestHiveSqlMigrate String sql9 = "ALTER TABLE T1 ADD CONSTRAINT TEST PRIMARY KEY (ID, NAME) DISABLE NOVALIDATE"; assertUnsupported(sql9, Optional.of("ADD CONSTRAINT")); - String sql10 = "ALTER TABLE T1 CHANGE COLUMN ID1 ID2 INT CONSTRAINT TEST NOT NULL ENABLE"; + String sql10 = "ALTER TABLE FOO PARTITION (DS='2008-04-08', HR) CHANGE COLUMN DEC_COLUMN_NAME DEC_COLUMN_NAME DECIMAL(38,18)"; assertUnsupported(sql10, Optional.of("CHANGE COLUMN")); String sql11 = "ALTER TABLE T1 DROP CONSTRAINT TEST"; @@ -730,6 +730,13 @@ public class TestHiveSqlMigrate assertUnsupported(sql, Optional.of("LOAD DATA")); } + @Test + public void testSetSession() + { + String sql = "set hive.support.concurrency=true"; + assertUnsupported(sql, Optional.of("SET PROPERTY")); + } + @Test public void testResetSession() { diff --git a/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestImpalaSqlMigrate.java b/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestImpalaSqlMigrate.java index 9c359c308..a085f0a91 100644 --- a/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestImpalaSqlMigrate.java +++ b/hetu-sql-migration-tool/src/test/java/io/hetu/core/sql/migration/tool/TestImpalaSqlMigrate.java @@ -135,7 +135,10 @@ public class TestImpalaSqlMigrate " format = 'Parquet',\n" + " location = '/user/tmp'\n" + ")"; - assertWarning(sql2, expectedSql2); + assertSuccess(sql2, expectedSql2); + + String sql3 = "create table impala10(impala1 tinyint, impala2 smallint, impala3 int) TBLPROPERTIES('id'='11','cnt'=1)"; + assertUnsupported(sql3, Optional.of("TBLPROPERTIES")); // Test negative sql - row format is not supported String negativeSql1 = "CREATE TABLE tbl_row_format (id INT, name STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY 'char'"; @@ -180,7 +183,7 @@ public class TestImpalaSqlMigrate ")"; assertSuccess(sql2, expectedSql2); - String negativeSql3 = "CREATE TABLE tb3 (c1 STRUCT )"; + String negativeSql3 = "CREATE TABLE tb3 (c1 STRUCT )"; assertUnsupported(negativeSql3, Optional.of("STRUCT")); } @@ -208,7 +211,7 @@ public class TestImpalaSqlMigrate " location = '/USER/TEST',\n" + " format = 'Avro'\n" + ")"; - assertWarning(sql, expectedSql); + assertSuccess(sql, expectedSql); String sql2 = "CREATE EXTERNAL TABLE T2 LIKE PARQUET '/user/test/impala' COMMENT 'HETU' STORED AS AVRO LOCATION '/USER/TEST'"; assertUnsupported(sql2, Optional.of("PARQUET")); @@ -286,6 +289,39 @@ public class TestImpalaSqlMigrate "HAVING (\"max\"(salary) > 110)\n" + "LIMIT 5\n"; assertSuccess(sql4, expectedSql4); + + // teet join + String sql5 = "SELECT * FROM t1 JOIN t2 ON t1.id = t2.id"; + String expectedSql5 = "SELECT *\n" + + "FROM\n" + + " (t1\n" + + "INNER JOIN t2 ON (t1.id = t2.id))\n"; + assertSuccess(sql5, expectedSql5); + + String sql6 = "SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id"; + assertSuccess(sql6, expectedSql5); + + String sql7 = "SELECT * FROM t1 FULL OUTER JOIN t2 ON t1.id = t2.id"; + String expectedSql7 = "SELECT *\n" + + "FROM\n" + + " (t1\n" + + "FULL JOIN t2 ON (t1.id = t2.id))\n"; + assertSuccess(sql7, expectedSql7); + + String sql8 = "SELECT * FROM t1 LEFT OUTER JOIN t2 ON t1.id = t2.id"; + String expectedSql8 = "SELECT *\n" + + "FROM\n" + + " (t1\n" + + "LEFT JOIN t2 ON (t1.id = t2.id))\n"; + assertSuccess(sql8, expectedSql8); + + String sql9 = "SELECT * FROM t1 CROSS JOIN t2 WHERE t1.id > t2.id"; + String expectedSql9 = "SELECT *\n" + + "FROM\n" + + " (t1\n" + + "CROSS JOIN t2)\n" + + "WHERE (t1.id > t2.id)\n"; + assertSuccess(sql9, expectedSql9); } @Test @@ -642,7 +678,9 @@ public class TestImpalaSqlMigrate public void testShutDown() { String sql = ":SHUTDOWN('LOCALHOST' : 8090, 0)"; + String sql2 = ":SHUTDOWN(\\\"hostname:1234\\\")"; assertUnsupported(sql, Optional.of("SHUTDOWN")); + assertUnsupported(sql2, Optional.of("SHUTDOWN")); } @Test @@ -794,9 +832,11 @@ public class TestImpalaSqlMigrate String sql = "SHOW FUNCTIONS IN DB1"; String sql2 = "SHOW AGGREGATE FUNCTIONS IN DB1"; String sql3 = "SHOW ANALYTIC FUNCTIONS IN DB1"; - assertUnsupported(sql, Optional.of("FUNCTIONS")); - assertUnsupported(sql2, Optional.of("FUNCTIONS")); - assertUnsupported(sql3, Optional.of("FUNCTIONS")); + String sql4 = "SHOW FUNCTIONS"; + assertUnsupported(sql, Optional.of("IN")); + assertUnsupported(sql2, Optional.of("AGGREGATE")); + assertUnsupported(sql3, Optional.of("ANALYTIC")); + assertSuccess(sql4, "SHOW FUNCTIONS"); } @Test