Compare commits

...

4 Commits

Author SHA1 Message Date
i-robot 4dc9ddca6c !880 Add ClickHouse connector on version 1.2
Merge pull request !880 from Heatao/branch-1.2
2021-06-04 16:43:37 +08:00
HEATAO e17b5e2dc4 add ClickHouse connector
add base remote udf register framework

remove redundant code

change code style

change the path of testng

fix bug
2021-06-04 16:00:10 +08:00
Raghunandan 8692d038e5 [maven-release-plugin] prepare for next development iteration 2021-03-31 11:23:23 +05:30
Raghunandan 730c848290 [maven-release-plugin] prepare release 1.2.0 2021-03-31 11:23:23 +05:30
103 changed files with 3209 additions and 143 deletions

View File

@ -22,7 +22,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-carbondata</artifactId>

168
hetu-clickhouse/pom.xml Normal file
View File

@ -0,0 +1,168 @@
<?xml version="1.0"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-clickhouse</artifactId>
<description>hetu - ClickHouse Connector</description>
<packaging>hetu-plugin</packaging>
<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<resources.dir>src/main/resources</resources.dir>
<version.clickhouse-jdbc>0.2.4</version.clickhouse-jdbc>
<version.maven-surefire-plugin>3.0.0-M2</version.maven-surefire-plugin>
</properties>
<dependencies>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-base-jdbc</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>configuration</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>log</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>
<!-- Presto SPI -->
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-spi</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>slice</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>units</artifactId>
<scope>provided</scope>
</dependency>
<!-- for testing -->
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-main</artifactId>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>testing</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-tpch</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.airlift.tpch</groupId>
<artifactId>tpch</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-tests</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>testing-mysql-server</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-testing-docker</artifactId>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc -->
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>${version.clickhouse-jdbc}</version>
<exclusions>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<!-- <version>${version.maven-surefire-plugin}</version>-->
<configuration>
<useSystemClassLoader>false</useSystemClassLoader>
<forkCount>1</forkCount>
<parallel>classes</parallel>
<threadCount>1</threadCount>
<suiteXmlFiles>
<file>src/test/testng.xml</file>
</suiteXmlFiles>
</configuration>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,548 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.airlift.log.Logger;
import io.hetu.core.plugin.clickhouse.optimization.ClickHousePushDownParameter;
import io.hetu.core.plugin.clickhouse.optimization.ClickHouseQueryGenerator;
import io.prestosql.plugin.jdbc.BaseJdbcClient;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.ColumnMapping;
import io.prestosql.plugin.jdbc.ConnectionFactory;
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
import io.prestosql.plugin.jdbc.JdbcIdentity;
import io.prestosql.plugin.jdbc.JdbcOutputTableHandle;
import io.prestosql.plugin.jdbc.JdbcSplit;
import io.prestosql.plugin.jdbc.JdbcTableHandle;
import io.prestosql.plugin.jdbc.JdbcTypeHandle;
import io.prestosql.plugin.jdbc.StatsCollecting;
import io.prestosql.plugin.jdbc.WriteMapping;
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorResult;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.SuppressFBWarnings;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.function.ExternalFunctionHub;
import io.prestosql.spi.function.FunctionMetadataManager;
import io.prestosql.spi.function.StandardFunctionResolution;
import io.prestosql.spi.relation.DeterminismEvaluator;
import io.prestosql.spi.relation.RowExpressionService;
import io.prestosql.spi.sql.QueryGenerator;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.DecimalType;
import io.prestosql.spi.type.Type;
import javax.inject.Inject;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.BiFunction;
import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.bigintColumnMapping;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.integerColumnMapping;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.smallintColumnMapping;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varbinaryWriteFunction;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharColumnMapping;
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction;
import static io.prestosql.spi.StandardErrorCode.NOT_FOUND;
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.prestosql.spi.type.BooleanType.BOOLEAN;
import static io.prestosql.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType;
import static io.prestosql.spi.type.Varchars.isVarcharType;
import static java.lang.String.format;
import static java.lang.String.join;
import static java.util.Collections.nCopies;
import static java.util.Locale.ENGLISH;
public class ClickHouseClient
extends BaseJdbcClient
{
private static final Logger logger = Logger.get(ClickHouseClient.class);
private String[] tableTypes;
private ClickHouseConfig clickHouseConfig;
private BaseJdbcConfig baseJdbcConfig;
/**
* If disabled, do not accept sub-query push down.
*/
private final JdbcPushDownModule pushDownModule;
/**
* constructor
*
* @param config Base config
* @param clickHouseConfig clickhouse config
* @param connectionFactory connectionFactory
*/
@Inject
public ClickHouseClient(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig, @StatsCollecting ConnectionFactory connectionFactory, ExternalFunctionHub externalFunctionHub)
{
super(config, "", connectionFactory, externalFunctionHub);
tableTypes = clickHouseConfig.getTableTypes().split(",");
this.pushDownModule = config.getPushDownModule();
this.baseJdbcConfig = config;
this.clickHouseConfig = clickHouseConfig;
}
@Override
public Optional<ColumnMapping> toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle)
{
Optional<ColumnMapping> columnMapping;
String jdbcTypeName = "";
if (typeHandle.getJdbcTypeName().isPresent()) {
jdbcTypeName = typeHandle.getJdbcTypeName().get().toUpperCase(ENGLISH);
}
switch (typeHandle.getJdbcType()) {
case Types.TINYINT:
if (jdbcTypeName.equals("UINT8")) {
columnMapping = Optional.of(smallintColumnMapping());
}
else {
columnMapping = Optional.of(tinyintColumnMapping());
}
break;
case Types.INTEGER:
if (jdbcTypeName.equals("UINT32")) {
columnMapping = Optional.of(bigintColumnMapping());
}
else {
columnMapping = Optional.of(integerColumnMapping());
}
break;
case Types.SMALLINT:
if (jdbcTypeName.equals("UINT16")) {
columnMapping = Optional.of(integerColumnMapping());
}
else {
columnMapping = Optional.of(smallintColumnMapping());
}
break;
case Types.BIGINT:
if (jdbcTypeName.equals("UINT64")) {
logger.warn("openLooKeng doesn't support UInt64, it will convert to Int64");
}
columnMapping = Optional.of(bigintColumnMapping());
break;
case Types.VARCHAR:
columnMapping = Optional.of(varcharColumnMapping(createUnboundedVarcharType()));
break;
case Types.DATE:
case Types.TIMESTAMP:
case Types.OTHER:
default:
columnMapping = super.toPrestoType(session, connection, typeHandle);
}
if (!columnMapping.isPresent()) {
throw new UnsupportedOperationException(
"openLooKeng does not support the clickhouse type " + typeHandle.getJdbcTypeName().orElse("")
+ '(' + typeHandle.getColumnSize() + ", " + typeHandle.getDecimalDigits()
+ ')');
}
return columnMapping;
}
@Override
public WriteMapping toWriteMapping(ConnectorSession session, Type type)
{
if (isVarcharType(type)) {
return WriteMapping.sliceMapping("String", varcharWriteFunction());
}
else if (VARBINARY.equals(type)) {
return WriteMapping.sliceMapping("String", varbinaryWriteFunction());
}
else if (type instanceof CharType) {
return WriteMapping.sliceMapping("FixedString(" + ((CharType) type).getLength() + ')',
charWriteFunction());
}
else if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
String dataType = format("decimal(%s, %s)", decimalType.getPrecision(), decimalType.getScale());
if (decimalType.isShort()) {
return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType));
}
return WriteMapping.sliceMapping(dataType, longDecimalWriteFunction(decimalType));
}
else if (BOOLEAN.equals(type)) {
throw new PrestoException(NOT_SUPPORTED, "There is no separate type for boolean values. Use UInt8 type, restricted to the values 0 or 1.");
}
else if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
throw new PrestoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());
}
return super.toWriteMapping(session, type);
}
@Override
protected Collection<String> listSchemas(Connection connection)
{
try (ResultSet resultSet = connection.getMetaData().getSchemas()) {
ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder();
while (resultSet.next()) {
String schemaName = resultSet.getString(1);
schemaNames.add(schemaName);
}
return schemaNames.build();
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to list schemas");
}
}
@Override
protected ResultSet getTables(Connection connection, Optional<String> schemaName, Optional<String> tableName)
throws SQLException
{
DatabaseMetaData metadata = connection.getMetaData();
Optional<String> escape = Optional.ofNullable(metadata.getSearchStringEscape());
return metadata.getTables(connection.getCatalog(), escapeNamePattern(schemaName, escape).orElse(null), escapeNamePattern(tableName, escape).orElse(null), tableTypes);
}
/**
* create table function should be disabled.
*/
@Override
protected JdbcOutputTableHandle createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, String tableName)
{
throw new UnsupportedOperationException("ClickHouse Connector does not support create table.");
}
@Override
public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column)
{
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session))) {
String columnName = column.getName();
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
columnName = columnName.toUpperCase(ENGLISH);
}
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String sql = format("ALTER TABLE %s ADD column %s", quoted(null, schema, handle.getTableName()), getColumnSql(session, column, columnName));
execute(connection, sql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to add column, check table engine and sql");
}
}
private String getColumnSql(ConnectorSession session, ColumnMetadata column, String columnName)
{
StringBuilder sb = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY)
.append(quoted(columnName))
.append(" ")
.append(toWriteMapping(session, column.getType()).getDataType());
return sb.toString();
}
@Override
public void dropColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column)
{
try (Connection connection = connectionFactory.openConnection(identity)) {
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String sql = format("ALTER TABLE %s DROP column %s", quoted(null, schema, handle.getTableName()), column.getColumnName());
execute(connection, sql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to drop column");
}
}
@Override
protected void renameTable(JdbcIdentity identity, String catalogName, String schemaName, String tableName, SchemaTableName newTable)
{
try (Connection connection = connectionFactory.openConnection(identity)) {
String newSchemaName = newTable.getSchemaName();
String newTableName = newTable.getTableName();
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
newSchemaName = newSchemaName.toUpperCase(ENGLISH);
newTableName = newTableName.toUpperCase(ENGLISH);
}
String schema = checkCatalog(catalogName, schemaName);
String newSchema = checkCatalog(catalogName, newSchemaName);
String sql = format("RENAME TABLE %s TO %s", quoted(null, schema, tableName), quoted(null, newSchema, newTableName));
execute(connection, sql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to rename table");
}
}
@Override
public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
{
try (Connection connection = connectionFactory.openConnection(identity)) {
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
newColumnName = newColumnName.toUpperCase(ENGLISH);
}
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String sql = format(
"ALTER TABLE %s RENAME COLUMN %s TO %s",
quoted(null, schema, handle.getTableName()),
jdbcColumn.getColumnName(),
newColumnName);
execute(connection, sql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, e);
}
}
@Override
public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
{
SchemaTableName schemaTableName = tableMetadata.getTable();
JdbcIdentity identity = JdbcIdentity.from(session);
if (!getSchemaNames(identity).contains(schemaTableName.getSchemaName())) {
throw new PrestoException(NOT_FOUND, "Schema not found: " + schemaTableName.getSchemaName());
}
try (Connection connection = connectionFactory.openConnection(identity)) {
boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers();
String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName());
String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName());
String temporaryTableName = generateTemporaryTableName();
String catalog = connection.getCatalog();
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
ImmutableList.Builder<String> columnList = ImmutableList.builder();
for (ColumnMetadata column : tableMetadata.getColumns()) {
String columnName = column.getName();
if (uppercase) {
columnName = columnName.toUpperCase(ENGLISH);
}
columnNames.add(columnName);
columnTypes.add(column.getType());
columnList.add(getColumnSql(session, column, columnName));
}
String schema = checkCatalog(catalog, remoteSchema);
String sql = format(
"CREATE TABLE %s engine=Log AS SELECT * FROM %s WHERE 0 = 1",
quoted(null, schema, temporaryTableName),
quoted(null, schema, remoteTable));
execute(connection, sql);
return new JdbcOutputTableHandle(
catalog,
remoteSchema,
remoteTable,
columnNames.build(),
columnTypes.build(),
temporaryTableName);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, e);
}
}
@Override
public String buildInsertSql(JdbcOutputTableHandle handle)
{
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String sql = format(
"INSERT INTO %s VALUES (%s)",
quoted(null, schema, handle.getTemporaryTableName()),
join(",", nCopies(handle.getColumnNames().size(), "?")));
return sql;
}
@Override
public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List<JdbcColumnHandle> columns)
throws SQLException
{
if (table.getGeneratedSql().isPresent()) {
// openLooKeng: If the sub-query is pushed down, use it as the table
return new ClickHouseQueryBuilder(identifierQuote, true).buildSql(
this,
session,
connection,
null,
null,
table.getGeneratedSql().get().getSql(),
columns,
table.getConstraint(),
split.getAdditionalPredicate(),
tryApplyLimit(table.getLimit()));
}
return new ClickHouseQueryBuilder(identifierQuote).buildSql(
this,
session,
connection,
table.getCatalogName(),
table.getSchemaName(),
table.getTableName(),
columns,
table.getConstraint(),
split.getAdditionalPredicate(),
tryApplyLimit(table.getLimit()));
}
@Override
public void finishInsertTable(JdbcIdentity identity, JdbcOutputTableHandle handle)
{
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String temporaryTable = quoted(null, schema, handle.getTemporaryTableName());
String targetTable = quoted(null, schema, handle.getTableName());
String insertSql = format("INSERT INTO %s SELECT * FROM %s", targetTable, temporaryTable);
String cleanupSql = "DROP TABLE " + temporaryTable;
try (Connection connection = getConnection(identity, handle)) {
execute(connection, insertSql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, e);
}
try (Connection connection = getConnection(identity, handle)) {
execute(connection, cleanupSql);
}
catch (SQLException e) {
logger.warn(e, "Failed to cleanup temporary table: %s", temporaryTable);
}
}
@Override
public void dropTable(JdbcIdentity identity, JdbcTableHandle handle)
{
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
String sql = "DROP TABLE " + quoted(null, schema, handle.getTableName());
try (Connection connection = connectionFactory.openConnection(identity)) {
execute(connection, sql);
}
catch (SQLException e) {
throw new PrestoException(JDBC_ERROR, e);
}
}
@Override
protected Optional<BiFunction<String, Long, String>> limitFunction()
{
return Optional.of((sql, limit) -> sql + " LIMIT " + limit);
}
@Override
public boolean isLimitGuaranteed()
{
return true;
}
@Override
public Optional<QueryGenerator<JdbcQueryGeneratorResult, JdbcConverterContext>> getQueryGenerator(DeterminismEvaluator determinismEvaluator, RowExpressionService rowExpressionService, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution)
{
ClickHousePushDownParameter pushDownParameter = new ClickHousePushDownParameter(getIdentifierQuote(), this.caseInsensitiveNameMatching, pushDownModule, clickHouseConfig, functionResolution);
return Optional.of(new ClickHouseQueryGenerator(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig));
}
@SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING")
@Override
public Map<String, ColumnHandle> getColumns(ConnectorSession session, String sql, Map<String, Type> types)
{
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
PreparedStatement statement = connection.prepareStatement(sql)) {
ResultSetMetaData metadata = statement.getMetaData();
ImmutableMap.Builder<String, ColumnHandle> builder = new ImmutableMap.Builder<>();
for (int i = 1; i <= metadata.getColumnCount(); i++) {
String columnName = metadata.getColumnLabel(i);
String typeName = metadata.getColumnTypeName(i);
int precision = metadata.getPrecision(i);
int dataType = metadata.getColumnType(i);
int scale = metadata.getScale(i);
boolean isNullAble = metadata.isNullable(i) != ResultSetMetaData.columnNoNulls;
if (dataType == Types.DECIMAL) {
String loweredColumnName = columnName.toLowerCase(ENGLISH);
Type type = types.get(loweredColumnName);
if (type instanceof DecimalType) {
DecimalType decimalType = (DecimalType) type;
precision = decimalType.getPrecision();
scale = decimalType.getScale();
}
}
JdbcTypeHandle typeHandle = new JdbcTypeHandle(dataType, Optional.ofNullable(typeName), precision, scale, Optional.empty());
Optional<ColumnMapping> columnMapping;
try {
columnMapping = toPrestoType(session, connection, typeHandle);
}
catch (UnsupportedOperationException ex) {
// User configured to fail the query if the data type is not supported
return Collections.emptyMap();
}
// skip unsupported column types
if (columnMapping.isPresent()) {
Type type = columnMapping.get().getType();
JdbcColumnHandle handle = new JdbcColumnHandle(columnName, typeHandle, type, isNullAble);
builder.put(columnName.toLowerCase(ENGLISH), handle);
}
else {
return Collections.emptyMap();
}
}
return builder.build();
}
catch (SQLException | PrestoException e) {
logger.error("in clickhouse push down, clickhouse data source error msg[%s] rewrite sql[%s]", e.getMessage(), sql);
return Collections.emptyMap();
}
}
/**
* Clickhouse does not support operations on catalog.schema.table
*/
private String checkCatalog(String catalog, String remoteSchema)
{
String schema;
if (catalog != null && remoteSchema != null) {
schema = remoteSchema;
}
else {
schema = catalog == null ? remoteSchema : catalog;
}
return schema;
}
}

View File

@ -0,0 +1,61 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.inject.Binder;
import com.google.inject.Provides;
import com.google.inject.Scopes;
import com.google.inject.Singleton;
import io.airlift.configuration.AbstractConfigurationAwareModule;
import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.ConnectionFactory;
import io.prestosql.plugin.jdbc.DriverConnectionFactory;
import io.prestosql.plugin.jdbc.JdbcClient;
import io.prestosql.spi.function.ExternalFunctionHub;
import ru.yandex.clickhouse.ClickHouseDriver;
import ru.yandex.clickhouse.settings.ClickHouseConnectionSettings;
import java.util.Optional;
import java.util.Properties;
import static io.airlift.configuration.ConfigBinder.configBinder;
import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
public class ClickHouseClientModule
extends AbstractConfigurationAwareModule
{
@Override
protected void setup(Binder binder)
{
binder.bind(JdbcClient.class).to(ClickHouseClient.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(BaseJdbcConfig.class);
binder.bind(ExternalFunctionHub.class).to(ClickHouseExternalFunctionHub.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(ClickHouseConfig.class);
}
@Provides
@Singleton
public static ConnectionFactory createConnectionFactory(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig)
{
Properties connectionProperties = basicConnectionProperties(config);
connectionProperties.setProperty(ClickHouseConnectionSettings.SOCKET_TIMEOUT.getKey(),
String.valueOf(clickHouseConfig.getSocketTimeout()));
return new DriverConnectionFactory(new ClickHouseDriver(), config.getConnectionUrl(),
Optional.ofNullable(config.getUserCredentialName()),
Optional.ofNullable(config.getPasswordCredentialName()),
connectionProperties);
}
}

View File

@ -0,0 +1,100 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import java.util.Locale;
/**
* To get the custom properties to connect to the database.
*/
public class ClickHouseConfig
{
private static final int DEFAULT_SOCKET_TIMEOUT = 120000;
private int socketTimeout = DEFAULT_SOCKET_TIMEOUT;
private String tableTypes = ClickHouseConstants.DEFAULT_TABLE_TYPES;
private String schemaPattern;
private boolean isQueryPushDownEnabled = true;
private String clickHouseSqlVersion = "DEFAULT";
public int getSocketTimeout()
{
return socketTimeout;
}
@Config("clickhouse.socket_timeout")
@ConfigDescription("Connection ClickHouse socket timeout ")
public ClickHouseConfig setSocketTimeout(int socketTimeout)
{
this.socketTimeout = socketTimeout;
return this;
}
@Config("clickhouse.query.pushdown.enabled")
@ConfigDescription("Enable sub-query push down to clickhouse. It's set by default")
public ClickHouseConfig setQueryPushDownEnabled(boolean isQueryPushDownEnabledParameter)
{
this.isQueryPushDownEnabled = isQueryPushDownEnabledParameter;
return this;
}
public boolean isQueryPushDownEnabled()
{
return this.isQueryPushDownEnabled;
}
public String getTableTypes()
{
return this.tableTypes;
}
public String getSchemaPattern()
{
return schemaPattern;
}
/**
* setTableTypes
*
* @param tableTypes the table types to set
* @return ClickHouseConfig
*/
@Config("clickhouse.table-types")
public ClickHouseConfig setTableTypes(String tableTypes)
{
this.tableTypes = tableTypes.toUpperCase(Locale.ENGLISH);
return this;
}
public String getClickHouseSqlVersion()
{
return this.clickHouseSqlVersion;
}
/**
* setSchemaPattern
*
* @param schemaPattern the schema pattern to set
* @return ClickHouseConfig
*/
@Config("clickhouse.schema-pattern")
public ClickHouseConfig setSchemaPattern(String schemaPattern)
{
this.schemaPattern = schemaPattern;
return this;
}
}

View File

@ -0,0 +1,33 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
public class ClickHouseConstants
{
public static final String CLICKHOUSE_JDBC_DRIVER_CLASS_NAME = "ru.yandex.clickhouse.ClickHouseDriver";
public static final int DEAFULT_STRINGBUFFER_CAPACITY = 30;
/**
* default table type list for clickhouse
*/
public static final String DEFAULT_TABLE_TYPES = "TABLE,VIEW";
public static final String CONNECTOR_NAME = "ClickHouse";
private ClickHouseConstants()
{
}
}

View File

@ -0,0 +1,26 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import io.prestosql.plugin.jdbc.JdbcPlugin;
public class ClickHousePlugin
extends JdbcPlugin
{
public ClickHousePlugin()
{
super("clickhouse", new ClickHouseClientModule());
}
}

View File

@ -0,0 +1,346 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.common.base.Joiner;
import com.google.common.base.VerifyException;
import com.google.common.collect.ImmutableList;
import io.airlift.slice.Slice;
import io.prestosql.plugin.jdbc.BlockWriteFunction;
import io.prestosql.plugin.jdbc.BooleanWriteFunction;
import io.prestosql.plugin.jdbc.DoubleWriteFunction;
import io.prestosql.plugin.jdbc.JdbcClient;
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
import io.prestosql.plugin.jdbc.JdbcTypeHandle;
import io.prestosql.plugin.jdbc.LongWriteFunction;
import io.prestosql.plugin.jdbc.QueryBuilder;
import io.prestosql.plugin.jdbc.SliceWriteFunction;
import io.prestosql.plugin.jdbc.WriteFunction;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.predicate.Domain;
import io.prestosql.spi.predicate.Range;
import io.prestosql.spi.predicate.TupleDomain;
import io.prestosql.spi.type.ArrayType;
import io.prestosql.spi.type.BigintType;
import io.prestosql.spi.type.BooleanType;
import io.prestosql.spi.type.CharType;
import io.prestosql.spi.type.DateType;
import io.prestosql.spi.type.DoubleType;
import io.prestosql.spi.type.IntegerType;
import io.prestosql.spi.type.RealType;
import io.prestosql.spi.type.SmallintType;
import io.prestosql.spi.type.TimeType;
import io.prestosql.spi.type.TimeWithTimeZoneType;
import io.prestosql.spi.type.TimestampType;
import io.prestosql.spi.type.TimestampWithTimeZoneType;
import io.prestosql.spi.type.TinyintType;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarcharType;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.Iterables.getOnlyElement;
import static java.lang.String.format;
import static java.util.Collections.nCopies;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.joining;
public class ClickHouseQueryBuilder
extends QueryBuilder
{
private static final String ALWAYS_TRUE = "1=1";
private static final String ALWAYS_FALSE = "1=0";
private boolean isPushSubQueryDown;
private final String identifierQuote;
public ClickHouseQueryBuilder(String identifierQuote)
{
super(identifierQuote);
this.identifierQuote = requireNonNull(identifierQuote, "quote is null");
}
public ClickHouseQueryBuilder(String identifierQuote, boolean isPushSubQueryDown)
{
this(identifierQuote);
this.isPushSubQueryDown = isPushSubQueryDown;
}
private static class TypeAndValue
{
private final Type type;
private final JdbcTypeHandle typeHandle;
private final Object value;
public TypeAndValue(Type type, JdbcTypeHandle typeHandle, Object value)
{
this.type = requireNonNull(type, "type is null");
this.typeHandle = requireNonNull(typeHandle, "typeHandle is null");
this.value = requireNonNull(value, "value is null");
}
public Type getType()
{
return type;
}
public JdbcTypeHandle getTypeHandle()
{
return typeHandle;
}
public Object getValue()
{
return value;
}
}
private String quote(String name)
{
return identifierQuote + name.replace(identifierQuote, identifierQuote + identifierQuote) + identifierQuote;
}
private static Domain pushDownDomain(JdbcClient client, ConnectorSession session, Connection connection, JdbcColumnHandle column, Domain domain)
{
return client.toPrestoType(session, connection, column.getJdbcTypeHandle())
.orElseThrow(() -> new IllegalStateException(format("Unsupported type %s with handle %s", column.getColumnType(), column.getJdbcTypeHandle())))
.getPushdownConverter().apply(domain);
}
private List<String> toConjuncts(
JdbcClient client,
ConnectorSession session,
Connection connection,
List<JdbcColumnHandle> columns,
TupleDomain<ColumnHandle> tupleDomain,
List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
{
if (tupleDomain.isNone()) {
return ImmutableList.of(ALWAYS_FALSE);
}
ImmutableList.Builder<String> builder = ImmutableList.builder();
for (JdbcColumnHandle column : columns) {
Domain domain = tupleDomain.getDomains().get().get(column);
if (domain != null) {
domain = pushDownDomain(client, session, connection, column, domain);
builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
}
}
return builder.build();
}
private String toPredicate(String columnName, Domain domain, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
{
if (domain.getValues().isNone()) {
return domain.isNullAllowed() ? quote(columnName) + " IS NULL" : ALWAYS_FALSE;
}
if (domain.getValues().isAll()) {
return domain.isNullAllowed() ? ALWAYS_TRUE : quote(columnName) + " IS NOT NULL";
}
List<String> disjuncts = new ArrayList<>();
List<Object> singleValues = new ArrayList<>();
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
checkState(!range.isAll()); // Already checked
if (range.isSingleValue()) {
singleValues.add(range.getLow().getValue());
}
else {
List<String> rangeConjuncts = new ArrayList<>();
if (!range.getLow().isLowerUnbounded()) {
switch (range.getLow().getBound()) {
case ABOVE:
rangeConjuncts.add(toPredicate(columnName, ">", range.getLow().getValue(), column, accumulator));
break;
case EXACTLY:
rangeConjuncts.add(toPredicate(columnName, ">=", range.getLow().getValue(), column, accumulator));
break;
case BELOW:
throw new IllegalArgumentException("Low marker should never use BELOW bound");
default:
throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
}
}
if (!range.getHigh().isUpperUnbounded()) {
switch (range.getHigh().getBound()) {
case ABOVE:
throw new IllegalArgumentException("High marker should never use ABOVE bound");
case EXACTLY:
rangeConjuncts.add(toPredicate(columnName, "<=", range.getHigh().getValue(), column, accumulator));
break;
case BELOW:
rangeConjuncts.add(toPredicate(columnName, "<", range.getHigh().getValue(), column, accumulator));
break;
default:
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
}
}
// If rangeConjuncts is null, then the range was ALL, which should already have been checked for
checkState(!rangeConjuncts.isEmpty());
disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")");
}
}
// Add back all of the possible single values either as an equality or an IN predicate
if (singleValues.size() == 1) {
disjuncts.add(toPredicate(columnName, "=", getOnlyElement(singleValues), column, accumulator));
}
else if (singleValues.size() > 1) {
for (Object value : singleValues) {
bindValue(value, column, accumulator);
}
String values = Joiner.on(",").join(nCopies(singleValues.size(), "?"));
disjuncts.add(quote(columnName) + " IN (" + values + ")");
}
// Add nullability disjuncts
checkState(!disjuncts.isEmpty());
if (domain.isNullAllowed()) {
disjuncts.add(quote(columnName) + " IS NULL");
}
return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
}
private String toPredicate(String columnName, String operator, Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
{
bindValue(value, column, accumulator);
return quote(columnName) + " " + operator + " ?";
}
private static void bindValue(Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
{
Type type = column.getColumnType();
checkArgument(isAcceptedType(type), "Can't handle type: %s", type);
accumulator.add(new TypeAndValue(type, column.getJdbcTypeHandle(), value));
}
private static boolean isAcceptedType(Type type)
{
Type validType = requireNonNull(type, "type is null");
return validType.equals(BigintType.BIGINT) ||
validType.equals(TinyintType.TINYINT) ||
validType.equals(SmallintType.SMALLINT) ||
validType.equals(IntegerType.INTEGER) ||
validType.equals(DoubleType.DOUBLE) ||
validType.equals(RealType.REAL) ||
validType.equals(BooleanType.BOOLEAN) ||
validType.equals(DateType.DATE) ||
validType.equals(TimeType.TIME) ||
validType.equals(TimeWithTimeZoneType.TIME_WITH_TIME_ZONE) ||
validType.equals(TimestampType.TIMESTAMP) ||
validType.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) ||
validType instanceof VarcharType ||
validType instanceof CharType ||
validType instanceof ArrayType;
}
@Override
public PreparedStatement buildSql(JdbcClient client,
ConnectorSession session,
Connection connection,
String catalog,
String schema,
String table,
List<JdbcColumnHandle> columns,
TupleDomain<ColumnHandle> tupleDomain,
Optional<String> additionalPredicate,
Function<String, String> sqlFunction)
throws SQLException
{
StringBuilder sql = new StringBuilder();
String columnNames = columns.stream()
.map(JdbcColumnHandle::getColumnName)
.map(this::quote)
.collect(joining(", "));
sql.append("SELECT ");
sql.append(columnNames);
if (columns.isEmpty()) {
sql.append("null");
}
sql.append(" FROM ");
if (!isNullOrEmpty(schema)) {
sql.append(quote(schema)).append('.');
}
if (isPushSubQueryDown) {
sql.append("(").append(table).append(") pushdown");
}
else {
sql.append(quote(table));
}
List<TypeAndValue> accumulator = new ArrayList<>();
List<String> clauses = toConjuncts(client, session, connection, columns, tupleDomain, accumulator);
if (additionalPredicate.isPresent()) {
clauses = ImmutableList.<String>builder()
.addAll(clauses)
.add(additionalPredicate.get())
.build();
}
if (!clauses.isEmpty()) {
sql.append(" WHERE ")
.append(Joiner.on(" AND ").join(clauses));
}
String query = sqlFunction.apply(sql.toString());
PreparedStatement statement = client.getPreparedStatement(connection, query);
for (int i = 0; i < accumulator.size(); i++) {
TypeAndValue typeAndValue = accumulator.get(i);
int parameterIndex = i + 1;
Type type = typeAndValue.getType();
WriteFunction writeFunction = client.toPrestoType(session, connection, typeAndValue.getTypeHandle())
.orElseThrow(() -> new VerifyException(format("Unsupported type %s with handle %s", type, typeAndValue.getTypeHandle())))
.getWriteFunction();
Class<?> javaType = type.getJavaType();
Object value = typeAndValue.getValue();
if (javaType == boolean.class) {
((BooleanWriteFunction) writeFunction).set(statement, parameterIndex, (boolean) value);
}
else if (javaType == long.class) {
((LongWriteFunction) writeFunction).set(statement, parameterIndex, (long) value);
}
else if (javaType == double.class) {
((DoubleWriteFunction) writeFunction).set(statement, parameterIndex, (double) value);
}
else if (javaType == Slice.class) {
((SliceWriteFunction) writeFunction).set(statement, parameterIndex, (Slice) value);
}
else if (javaType == Block.class) {
((BlockWriteFunction) writeFunction).set(statement, parameterIndex, (Block) value);
}
else {
throw new VerifyException(format("Unexpected type %s with java type %s", type, javaType.getName()));
}
}
return statement;
}
}

View File

@ -0,0 +1,62 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization;
import com.google.common.base.Joiner;
import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
import io.prestosql.spi.function.SqlFunctionHandle;
import io.prestosql.spi.relation.CallExpression;
import io.prestosql.sql.builder.functioncall.ApplyRemoteFunctionPushDown;
import java.util.Map;
import java.util.Optional;
import static java.util.stream.Collectors.toList;
public class ClickHouseApplyRemoteFunctionPushDown
extends ApplyRemoteFunctionPushDown
{
private static Map<String, String> clickHousePrimalFunctionNameMap = ClickHouseExternalFunctionHub.getPrimalFunctionNameMap();
public ClickHouseApplyRemoteFunctionPushDown(BaseJdbcConfig baseJdbcConfig, String connectorName)
{
super(baseJdbcConfig, connectorName);
}
/**
* rewrite the remote function to a executable function in the data source.
*/
public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
{
if (!isConnectorSupportedRemoteFunction(callExpression)) {
return Optional.empty();
}
jdbcConverterContext.setRemoteUdfVisited(true);
String displayName = ((SqlFunctionHandle) callExpression.getFunctionHandle()).getFunctionId().getFunctionName().getObjectName();
String args = Joiner.on(",").join(callExpression.getArguments().stream().map(expression -> expression.accept(rowExpressionConverter, jdbcConverterContext)).collect(toList()));
// The click house is case sensitive and presto only support the lower case function name,
// we need to handle this function into lower and store its primal name.
// The click house do not contain two different functions have the equal function name ignore case.
// If a data base contain two different functions have the equal function name ignore case,
// then we should register them into different function name space
if (!clickHousePrimalFunctionNameMap.containsKey(displayName)) {
return Optional.empty();
}
return Optional.of(String.format("%s(%s)", clickHousePrimalFunctionNameMap.get(displayName), args));
}
}

View File

@ -0,0 +1,40 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization;
import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
import io.prestosql.spi.function.StandardFunctionResolution;
/**
* Push Down parameter module
*/
public class ClickHousePushDownParameter
extends JdbcPushDownParameter
{
private final ClickHouseConfig clickHouseConfig;
public ClickHousePushDownParameter(String identifierQuote, boolean nameCaseInsensitive, JdbcPushDownModule pushDownModule, ClickHouseConfig clickHouseConfig, StandardFunctionResolution functionResolution)
{
super(identifierQuote, nameCaseInsensitive, pushDownModule, functionResolution);
this.clickHouseConfig = clickHouseConfig;
}
public ClickHouseConfig getClickHouseConfig()
{
return clickHouseConfig;
}
}

View File

@ -0,0 +1,37 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.optimization.BaseJdbcQueryGenerator;
import io.prestosql.spi.function.FunctionMetadataManager;
import io.prestosql.spi.function.StandardFunctionResolution;
import io.prestosql.spi.relation.DeterminismEvaluator;
import io.prestosql.spi.relation.RowExpressionService;
public class ClickHouseQueryGenerator
extends BaseJdbcQueryGenerator
{
public ClickHouseQueryGenerator(
DeterminismEvaluator determinismEvaluator,
RowExpressionService rowExpressionService,
FunctionMetadataManager functionManager,
StandardFunctionResolution functionResolution,
ClickHousePushDownParameter pushDownParameter,
BaseJdbcConfig baseJdbcConfig)
{
super(pushDownParameter, new ClickHouseRowExpressionConverter(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig), new ClickHouseSqlStatementWriter(pushDownParameter));
}
}

View File

@ -0,0 +1,257 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization;
import com.google.common.base.Joiner;
import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
import io.hetu.core.plugin.clickhouse.rewrite.BuildInDirectMapFunctionCallRewriter;
import io.hetu.core.plugin.clickhouse.rewrite.ClickHouseUnsupportedFunctionCallRewriter;
import io.hetu.core.plugin.clickhouse.rewrite.UdfFunctionRewriteConstants;
import io.hetu.core.plugin.clickhouse.rewrite.functioncall.DateParseFunctionCallRewriter;
import io.prestosql.configmanager.ConfigSupplier;
import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.function.FunctionHandle;
import io.prestosql.spi.function.FunctionMetadata;
import io.prestosql.spi.function.FunctionMetadataManager;
import io.prestosql.spi.function.OperatorType;
import io.prestosql.spi.function.StandardFunctionResolution;
import io.prestosql.spi.relation.CallExpression;
import io.prestosql.spi.relation.ConstantExpression;
import io.prestosql.spi.relation.DeterminismEvaluator;
import io.prestosql.spi.relation.RowExpression;
import io.prestosql.spi.relation.RowExpressionService;
import io.prestosql.spi.relation.SpecialForm;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarcharType;
import io.prestosql.sql.builder.functioncall.FunctionWriterManager;
import io.prestosql.sql.builder.functioncall.FunctionWriterManagerGroup;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import io.prestosql.sql.builder.functioncall.functions.base.FromBase64CallRewriter;
import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.prestosql.sql.builder.functioncall.BaseFunctionUtil.isDefaultFunction;
import static java.lang.String.format;
import static java.util.Locale.ENGLISH;
public class ClickHouseRowExpressionConverter
extends BaseJdbcRowExpressionConverter
{
private static final Set<String> clickHouseNotSupportFunctions =
Stream.of("try", "try_cast", "at_timezone", "current_user", "current_path", "current_time").collect(toImmutableSet());
private static FunctionWriterManager clickHouseFunctionManager;
private final ClickHouseApplyRemoteFunctionPushDown clickHouseApplyRemoteFunctionPushDown;
/**
* ClickHouse sql query writer
*
* @param clickHouseConfig config
*/
public ClickHouseRowExpressionConverter(
DeterminismEvaluator determinismEvaluator,
RowExpressionService rowExpressionService,
FunctionMetadataManager functionManager,
StandardFunctionResolution functionResolution,
ClickHousePushDownParameter clickHouseConfig,
BaseJdbcConfig baseJdbcConfig)
{
super(functionManager, functionResolution, rowExpressionService, determinismEvaluator);
clickHouseFunctionManager = initFunctionManager(clickHouseConfig.getClickHouseConfig());
clickHouseApplyRemoteFunctionPushDown = new ClickHouseApplyRemoteFunctionPushDown(baseJdbcConfig, ClickHouseConstants.CONNECTOR_NAME);
}
private FunctionWriterManager initFunctionManager(ClickHouseConfig clickHouseConfig)
{
ConfigSupplier configSupplier = new DefaultUdfRewriteConfigSupplier(UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS);
DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter =
new DefaultConnectorConfigFunctionRewriter(ClickHouseConstants.CONNECTOR_NAME, configSupplier);
return FunctionWriterManagerGroup.newFunctionWriterManagerInstance(ClickHouseConstants.CONNECTOR_NAME,
clickHouseConfig.getClickHouseSqlVersion(), getInjectFunctionCallRewritersDefault(clickHouseConfig), connectorConfigFunctionRewriter);
}
private Map<String, FunctionCallRewriter> getInjectFunctionCallRewritersDefault(ClickHouseConfig clickHouseConfig)
{
// add the user define function re-writer
Map<String, FunctionCallRewriter> functionCallRewriters = new HashMap<>(Collections.emptyMap());
// 1. the base function re-writers all connector can use
FromBase64CallRewriter fromBase64CallRewriter = new FromBase64CallRewriter();
functionCallRewriters.put(FromBase64CallRewriter.INNER_FUNC_FROM_BASE64, fromBase64CallRewriter);
// 2. the specific user define function re-writers
FunctionCallRewriter unSupportedFunctionCallRewriter = new ClickHouseUnsupportedFunctionCallRewriter(ClickHouseConstants.CONNECTOR_NAME);
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_DAY2SEC, unSupportedFunctionCallRewriter);
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH, unSupportedFunctionCallRewriter);
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_TIME_WITH_TZ_LITERAL, unSupportedFunctionCallRewriter);
FunctionCallRewriter buildInDirectMapFunctionCallRewriter = new BuildInDirectMapFunctionCallRewriter();
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUIDLIN_AGGR_FUNC_SUM, buildInDirectMapFunctionCallRewriter);
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_AVG, buildInDirectMapFunctionCallRewriter);
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_COUNT, buildInDirectMapFunctionCallRewriter);
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MAX, buildInDirectMapFunctionCallRewriter);
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MIN, buildInDirectMapFunctionCallRewriter);
FunctionCallRewriter dateParseFunctionCallRewriter = new DateParseFunctionCallRewriter();
functionCallRewriters.put(DateParseFunctionCallRewriter.BUILD_IN_FUNC_DATE_PARSE, dateParseFunctionCallRewriter);
return functionCallRewriters;
}
protected static String functionCall(QualifiedName name, boolean isDistinct, List<String> argumentsList, Optional<String> orderBy, Optional<String> filter, Optional<String> window)
{
if (clickHouseFunctionManager == null) {
throw new PrestoException(NOT_SUPPORTED, "Function manager is uninitialized");
}
try {
return clickHouseFunctionManager.getFunctionRewriteResult(name, isDistinct, argumentsList, orderBy, filter, window);
}
catch (UnsupportedOperationException e) {
throw new PrestoException(NOT_SUPPORTED, e.getMessage());
}
}
@Override
public String visitCall(CallExpression call, JdbcConverterContext context)
{
FunctionHandle functionHandle = call.getFunctionHandle();
// remote udf verify
if (!isDefaultFunction(call)) {
Optional<String> result = clickHouseApplyRemoteFunctionPushDown.rewriteRemoteFunction(call, this, context);
if (result.isPresent()) {
return result.get();
}
throw new PrestoException(NOT_SUPPORTED, String.format("ClickHouse connector does not support remote function: %s.%s", call.getDisplayName(), call.getFunctionHandle().getFunctionNamespace()));
}
FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(functionHandle);
String functionName = functionMetadata.getName().getObjectName();
if (clickHouseNotSupportFunctions.contains(functionName)) {
throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support " + functionName);
}
if (standardFunctionResolution.isOperator(functionHandle)) {
return handleOperatorFunction(call, functionMetadata, context);
}
List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
if (standardFunctionResolution.isNotFunction(functionHandle)) {
return format("(NOT %s)", argumentList.get(0));
}
if (standardFunctionResolution.isLikeFunction(functionHandle)) {
return format("(%s LIKE %s)", argumentList.get(0), argumentList.get(1));
}
/*
* Array needs to be tested
*/
if (standardFunctionResolution.isArrayConstructor(functionHandle)) {
return format("ARRAY(%s)", Joiner.on(", ").join(argumentList));
}
return functionCall(new QualifiedName(Collections.singletonList(functionName)), false, argumentList, Optional.empty(), Optional.empty(), Optional.empty());
}
private String handleOperatorFunction(CallExpression call, FunctionMetadata functionMetadata, JdbcConverterContext context)
{
Optional<OperatorType> operatorTypeOptional = functionMetadata.getOperatorType();
OperatorType type = operatorTypeOptional.get();
if (type.equals(OperatorType.CAST)) {
return handleCastOperator(call.getArguments().get(0), call.getType(), context);
}
List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
if (type.isArithmeticOperator()) {
return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
}
if (type.isComparisonOperator()) {
final String[] clickHouseCompareOperators = new String[] {"=", ">", "<", ">=", "<=", "!=", "<>"};
if (Arrays.asList(clickHouseCompareOperators).contains(type.getOperator())) {
return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
}
else {
String exceptionInfo = "ClickHouse Connector does not support comparison operator " + type.getOperator();
throw new PrestoException(NOT_SUPPORTED, exceptionInfo);
}
}
if (type.equals(OperatorType.SUBSCRIPT)) {
throw new PrestoException(NOT_SUPPORTED, "ClickHouse Connector does not support subscript now");
}
/*
* "Negative" needs to be tested
*/
if (call.getArguments().size() == 1 && type.equals(OperatorType.NEGATION)) {
String value = argumentList.get(0);
String separator = value.startsWith("-") ? " " : "";
return format("-%s%s", separator, value);
}
throw new PrestoException(NOT_SUPPORTED, String.format("Unknown operator %s in push down", type.getOperator()));
}
private String handleCastOperator(RowExpression expression, Type dstType, JdbcConverterContext context)
{
/*
* In SqlToRowExpressionTranslator, it will translate GenericLiteral expression to a 'CONSTANT' rowExpression,
* so the 'CAST' operator is not needed.
* */
String value = expression.accept(this, context);
if (expression instanceof ConstantExpression && expression.getType() instanceof VarcharType) {
//dstType needs to be tested here.
//especially DateTime.
return value;
}
if (dstType.getDisplayName().equals(LIKE_PATTERN_NAME)) {
return value;
}
return format("CAST(%s AS %s)", value, dstType.getDisplayName().toLowerCase(ENGLISH));
}
@Override
public String visitSpecialForm(SpecialForm specialForm, JdbcConverterContext context)
{
if (specialForm.getForm().equals(SpecialForm.Form.DEREFERENCE)
|| specialForm.getForm().equals(SpecialForm.Form.ROW_CONSTRUCTOR)
|| specialForm.getForm().equals(SpecialForm.Form.BIND)) {
throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support" + specialForm.getForm().toString());
}
return super.visitSpecialForm(specialForm, context);
}
}

View File

@ -0,0 +1,78 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization;
import io.airlift.log.Logger;
import io.prestosql.plugin.jdbc.optimization.BaseJdbcSqlStatementWriter;
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
import io.prestosql.spi.sql.expression.Types;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
/**
* It is not clear what is the difference between aggregation and the previous version.
*/
public class ClickHouseSqlStatementWriter
extends BaseJdbcSqlStatementWriter
{
protected static final Logger log = Logger.get(ClickHouseSqlStatementWriter.class);
public ClickHouseSqlStatementWriter(JdbcPushDownParameter pushDownParameter)
{
super(pushDownParameter);
}
@Override
public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
{
if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
functionName = "varPop";
}
return super.aggregation(functionName, arguments, isDistinct);
}
@Override
public String windowFrame(Types.WindowFrameType type, String start, Optional<String> end)
{
throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
}
@Override
public String window(String functionName, List<String> functionArgs, List<String> partitionBy, Optional<String> orderBy, Optional<String> frame)
{
throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
}
@Override
public String from(String selections, String from)
{
String[] froms = from.split("\\.");
if (froms.length == 2) {
return selections + " FROM " + from;
}
else if (froms.length == 3) {
StringBuilder sb = new StringBuilder(froms[1]);
sb.append(".");
sb.append(froms[2]);
return selections + " FROM " + sb.toString();
}
else {
log.error("params more than 3 " + from);
return selections + " FROM " + from;
}
}
}

View File

@ -0,0 +1,74 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.prestosql.spi.function.ExternalFunctionInfo;
import io.prestosql.spi.type.StandardTypes;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
public class ClickHouseExternalDateTimeFunctions
{
public static Set<ExternalFunctionInfo> getFunctionsInfo()
{
return ImmutableSet.<ExternalFunctionInfo>builder()
.add(ClickHouse_toUnixTimeStamp_FUNCTION_INFO)
.add(ClickHouse_toDateTime_FUNCTION_INFO)
.build();
}
public static Map<String, String> getDateTimeHubFunctionStringMap()
{
// the click house is case sensitive and presto only support the lower case function name,
// we need to handle this function into lower and store its primal name.
// now the click house do not contain two different functions have the equal function name ignore case.
return ImmutableMap.<String, String>builder()
.put(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH), TO_UNIX_TIME_STAMP_PRIMAL)
.put(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH), TO_DATE_TIME_PRIMAL)
.build();
}
private static final String TO_UNIX_TIME_STAMP_PRIMAL = "toUnixTimestamp";
private static final String TO_DATE_TIME_PRIMAL = "toDateTime";
private static final ExternalFunctionInfo ClickHouse_toUnixTimeStamp_FUNCTION_INFO =
ExternalFunctionInfo.builder()
.functionName(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH))
.inputArgs(StandardTypes.VARCHAR)
.returnType(StandardTypes.BIGINT)
.deterministic(true)
.calledOnNullInput(false)
.description("converts value to the number with type UInt32 -- Unix Timestamp")
.build();
private static final ExternalFunctionInfo ClickHouse_toDateTime_FUNCTION_INFO =
ExternalFunctionInfo.builder()
.functionName(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH))
.inputArgs(StandardTypes.VARCHAR)
.returnType(StandardTypes.TIMESTAMP)
.deterministic(true)
.calledOnNullInput(false)
.description("converts value to the number with type UInt32 -- Unix Timestamp")
.build();
private ClickHouseExternalDateTimeFunctions()
{
}
}

View File

@ -0,0 +1,68 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.spi.connector.CatalogSchemaName;
import io.prestosql.spi.function.ExternalFunctionInfo;
import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
import javax.inject.Inject;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import static java.util.Objects.requireNonNull;
public class ClickHouseExternalFunctionHub
extends JdbcExternalFunctionHub
{
private final BaseJdbcConfig jdbcConfig;
@Inject
public ClickHouseExternalFunctionHub(BaseJdbcConfig jdbcConfig)
{
this.jdbcConfig = requireNonNull(jdbcConfig, "jdbcConfig is null");
}
@Override
public Optional<CatalogSchemaName> getExternalFunctionCatalogSchemaName()
{
return jdbcConfig.getConnectorRegistryFunctionNamespace();
}
@Override
public Set<ExternalFunctionInfo> getExternalFunctions()
{
return ImmutableSet.<ExternalFunctionInfo>builder()
.addAll(ClickHouseExternalDateTimeFunctions.getFunctionsInfo())
.build();
}
public static Map<String, String> getPrimalFunctionNameMap()
{
// The click house is case sensitive and presto only support the lower case function name,
// we need to handle this function into lower and store its primal name.
// The click house do not contain two different functions have the equal function name ignore case.
// If a data base contain two different functions have the equal function name ignore case,
// then we should register them into different function name space
return ImmutableMap.<String, String>builder()
.putAll(ClickHouseExternalDateTimeFunctions.getDateTimeHubFunctionStringMap())
.build();
}
}

View File

@ -0,0 +1,72 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.rewrite;
import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
public class BuildInDirectMapFunctionCallRewriter
implements FunctionCallRewriter
{
// ========= for Aggregate Functions =========
/**
* functioncall name of build-in direct map aggregate function max
*/
public static final String BUILDIN_AGGR_FUNC_MAX = "max";
/**
* functioncall name of build-in direct min aggregate function min
*/
public static final String BUILDIN_AGGR_FUNC_MIN = "min";
/**
* functioncall name of build-in direct map aggregate function count
*/
public static final String BUILDIN_AGGR_FUNC_COUNT = "count";
/**
* functioncall name of build-in direct map aggregate function avg
*/
public static final String BUILDIN_AGGR_FUNC_AVG = "avg";
/**
* functioncall name of build-in direct map aggregate function sum
*/
public static final String BUIDLIN_AGGR_FUNC_SUM = "sum";
@Override
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
{
StringBuilder builder = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY);
String arguments = RewriteUtil.joinExpressions(functionCallArgsPackage.getArgumentsList());
if (functionCallArgsPackage.getArgumentsList().isEmpty() && "count".equalsIgnoreCase(functionCallArgsPackage.getName().getSuffix())) {
arguments = "*";
}
if (functionCallArgsPackage.getIsDistinct()) {
arguments = "DISTINCT " + arguments;
}
builder.append(RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName())).append('(').append(arguments);
functionCallArgsPackage.getOrderBy().ifPresent(exp -> builder.append(' ').append(exp));
builder.append(')');
functionCallArgsPackage.getFilter().ifPresent(exp -> builder.append(" FILTER ").append(exp));
functionCallArgsPackage.getWindow().ifPresent(exp -> builder.append(" OVER ").append(exp));
return builder.toString();
}
}

View File

@ -0,0 +1,52 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.rewrite;
import io.prestosql.spi.type.StandardTypes;
import io.prestosql.sql.builder.functioncall.functions.base.UnsupportedFunctionCallRewriter;
import static io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil.LITERAL_FUNCNAME_PREFIX;
public class ClickHouseUnsupportedFunctionCallRewriter
extends UnsupportedFunctionCallRewriter
{
/**
* functioncall name of INTERVAL_DAY_TO_SECOND literal in HeTu inner
*/
public static final String INNER_FUNC_INTERVAL_LITERAL_DAY2SEC =
LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_DAY_TO_SECOND;
/**
* functioncall name of INTERVAL_YEAR_TO_MONTH literal in HeTu inner
*/
public static final String INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH =
LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_YEAR_TO_MONTH;
/**
* functioncall name of TIME_WITH_TIME_ZONE literal in HeTu inner
*/
public static final String INNER_FUNC_TIME_WITH_TZ_LITERAL =
LITERAL_FUNCNAME_PREFIX + StandardTypes.TIME_WITH_TIME_ZONE;
/**
* the constructor of Unsupported Function Call Re-writer
*
* @param connectorName connector's name
*/
public ClickHouseUnsupportedFunctionCallRewriter(String connectorName)
{
super(connectorName);
}
}

View File

@ -0,0 +1,77 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.rewrite;
import com.google.common.base.Joiner;
import io.prestosql.spi.sql.expression.QualifiedName;
import io.prestosql.spi.sql.expression.Selection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static java.util.stream.Collectors.joining;
/**
* util helper class for rewrite handle.
*/
public class RewriteUtil
{
private RewriteUtil()
{
}
/**
* interal function prefix name
*/
public static final String LITERAL_FUNCNAME_PREFIX = "$literal$";
/**
* expression list
*
* @param expressions expression lists
* @return sql statement
*/
public static final String joinExpressions(List<String> expressions)
{
return Joiner.on(", ").join(expressions);
}
/**
* formate identifier
*
* @param qualifiedNames qualified names
* @param identifier identifier
* @return sql statement
*/
public static final String formatIdentifier(Optional<Map<String, Selection>> qualifiedNames, String identifier)
{
if (qualifiedNames.isPresent()) {
return qualifiedNames.get().get(identifier).getExpression();
}
return identifier;
}
/**
* formate qualified name
*
* @param name qualified name
* @return sql statement
*/
public static final String formatQualifiedName(QualifiedName name)
{
return name.getParts().stream().map(identifier -> formatIdentifier(Optional.empty(), identifier)).collect(joining("."));
}
}

View File

@ -0,0 +1,98 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.rewrite;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
public class UdfFunctionRewriteConstants
{
private UdfFunctionRewriteConstants()
{
}
/**
* udf rewrite pattern map
*/
public static final Map<String, String> DEFAULT_VERSION_UDF_REWRITE_PATTERNS =
new ImmutableMap.Builder<String, String>()
// Statistical aggregate functions
.put("CORR($1,$2)", "CORR($1, $2)")
.put("STDDEV($1)", "stddevSamp($1)")
.put("stddev_pop($1)", "stddev_pop($1)")
.put("stddev_samp($1)", "stddevSamp($1)")
.put("skewness($1)", "skewPop($1)")
.put("kurtosis($1)", "kurtPop($1)")
.put("VARIANCE($1)", "varSamp($1)")
.put("var_samp($1)", "varSamp($1)")
.put("APPROX_DISTINCT($1)", "uniq($1)")
.put("APPROX_DISTINCT($1,$2)", "uniq($1)")
// math functions
.put("ABS($1)", "ABS($1)")
.put("ACOS($1)", "ACOS($1)")
.put("ASIN($1)", "ASIN($1)")
.put("ATAN($1)", "ATAN($1)")
.put("ATAN2($1,$2)", "ATAN2($1, $2)")
.put("CEIL($1)", "CEIL($1)")
.put("CEILING($1)", "CEIL($1)")
.put("COS($1)", "COS($1)")
.put("e()", "e()")
.put("EXP($1)", "EXP($1)")
.put("FLOOR($1)", "FLOOR($1)")
.put("LN($1)", "LN($1)")
.put("LOG10($1)", "log10($1)")
.put("LOG2($1)", "log2($1)")
.put("MOD($1,$2)", "MOD($1, $2)")
.put("pi()", "pi()")
.put("POW($1,$2)", "POW($1, $2)")
.put("POWER($1,$2)", "POWER($1, $2)")
.put("RAND()", "RAND()")
.put("RANDOM()", "RAND()")
.put("ROUND($1)", "ROUND($1)")
.put("ROUND($1,$2)", "ROUND($1, $2)")
.put("SIGN($1)", "SIGN($1)")
.put("SIN($1)", "SIN($1)")
.put("SQRT($1)", "SQRT($1)")
.put("TAN($1)", "TAN($1)")
//character functions
.put("CONCAT($1,$2)", "CONCAT($1, $2)")
.put("LENGTH($1)", "LENGTH($1)")
.put("LOWER($1)", "LOWER($1)")
.put("LTRIM($1)", "trimLeft($1)")
.put("REPLACE($1,$2)", "replaceAll($1, $2, '')")
.put("REPLACE($1,$2,$3)", "replaceAll($1, $2, $3)")
.put("RTRIM($1)", "trimRight($1)")
.put("STRPOS($1,$2)", "position($1, $2)")
.put("SUBSTR($1,$2,$3)", "SUBSTR($1, $2, $3)")
.put("POSITION($1,$2)", "position($2, $1)")
.put("TRIM($1)", "trimBoth($1)")
.put("UPPER($1)", "UPPER($1)")
//date functions
.put("YEAR($1)", "toYear($1)")
.put("QUARTER($1)", "toQuarter($1)")
.put("MONTH($1)", "toMonth($1)")
.put("WEEK($1)", "toWeek($1)")
.put("YEAR_OF_WEEK($1)", "toISOYear($1)")
.put("DAY($1)", "toDayOfMonth($1)")
.put("HOUR($1)", "toHour($1)")
.put("MINUTE($1)", "toMinute($1)")
.put("SECOND($1)", "toSecond($1)")
.put("DAY_OF_WEEK($1)", "toDayOfWeek($1)")
.put("DAY_OF_MONTH($1)", "toDayOfMonth($1)")
.put("DAY_OF_YEAR($1)", "toDayOfYear($1)")
.put("TO_UNIXTIME($1)", "toUnixTimestamp($1)")
.build();
}

View File

@ -0,0 +1,51 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse.rewrite.functioncall;
import io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil;
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
import java.util.List;
import java.util.Locale;
/**
* Clickhouse date time type conversion functions.
* The rewrite of this function limits date_parse.
*/
public class DateParseFunctionCallRewriter
implements FunctionCallRewriter
{
public static final String BUILD_IN_FUNC_DATE_PARSE = "date_parse";
@Override
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
{
String functionName = RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName());
if (!functionName.toLowerCase(Locale.ENGLISH).equals("date_parse") || functionCallArgsPackage.getArgumentsList().size() != 2) {
throw new UnsupportedOperationException("ClickHouse Connector does not support function call of " + RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName()));
}
List<String> argsList = functionCallArgsPackage.getArgumentsList();
String args0 = argsList.get(0);
String args1 = argsList.get(1);
if (args1.equals("'%Y-%m-%d %H:%i:%s'")) {
return "toDateTime(" + args0 + ")";
}
else if (args1.equals("'%Y-%m-%d'")) {
return "toDate(" + args0 + ")";
}
return String.format("parseDateTimeBestEffort(%s)", args0);
}
}

View File

@ -0,0 +1,355 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.common.collect.ImmutableSet;
import io.airlift.log.Logger;
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
import io.prestosql.plugin.jdbc.JdbcIdentity;
import io.prestosql.plugin.jdbc.JdbcTableHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.type.IntegerType;
import io.prestosql.spi.type.Type;
import org.testng.SkipException;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.lang.reflect.Field;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import static io.prestosql.spi.type.VarcharType.VARCHAR;
import static io.prestosql.testing.TestingSession.testSessionBuilder;
import static org.testng.Assert.assertEquals;
/**
* ClickHouseTest
*/
public class ClickHouseClientTest
{
private static final ConnectorSession SESSION = testSessionBuilder().build().toConnectorSession();
private static final Logger LOGGER = Logger.get(ClickHouseClientTest.class);
private DataBaseTest database;
private String catalogName;
private ClickHouseClient clickHouseClient;
private Connection connection;
private ClickHouseServerTest clickHouseServer;
private ClickHouseClientTest()
{
}
/**
* setUp
*
* @throws SQLException SQLException
*/
@BeforeClass
public void setUp()
throws SQLException
{
this.clickHouseServer = ClickHouseServerTest.getInstance();
if (!clickHouseServer.isClickHouseServerAvailable()) {
LOGGER.info("please set correct clickhouse data base info!");
throw new SkipException("skip the test");
}
LOGGER.info("running TestClickHouseClient...");
database = new DataBaseTest(clickHouseServer);
connection = database.getConnection();
catalogName = connection.getCatalog();
clickHouseClient = database.getClickHouseClient();
}
/**
* tearDown
*
* @throws SQLException SQLException
*/
@AfterClass(alwaysRun = true)
public void tearDown()
throws SQLException
{
if (this.clickHouseServer.isClickHouseServerAvailable()) {
database.close();
}
}
/**
* testCreateTable
*/
@Test(enabled = false)
public void testCreateTable()
throws SQLException
{
String tableName = "testCreateTable";
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
JdbcIdentity identity = JdbcIdentity.from(SESSION);
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
Type intType = IntegerType.INTEGER;
ColumnMetadata columnMetadata = new ColumnMetadata("col1", intType);
List<ColumnMetadata> columns = new ArrayList<>();
columns.add(columnMetadata);
ConnectorTableMetadata connectorTableMetadata = new ConnectorTableMetadata(schemaTableName, columns);
clickHouseClient.createTable(SESSION, connectorTableMetadata, tableName);
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
clickHouseClient.dropTable(identity, tableHandle);
}
/**
* testListSchemas
*/
@Test
public void testListSchemas()
{
assertEquals(ImmutableSet.copyOf(clickHouseClient.listSchemas(connection)).contains("test"), true);
}
/**
* testGetTables
*/
@Test
public void testGetTables()
{
List<String> expectedTables = database.getTables();
List<String> allTables = getAllTables(getNameByUpperCaseIdentifiers(database.getSchema()));
List<String> actualTables = getActualTables(allTables, expectedTables);
assertEquals(actualTables, expectedTables);
}
@Test
public void testRenameTable()
{
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
List<String> expectedTables = database.getTables();
List<String> allTables = getAllTables(schemaName);
List<String> actualTables = getActualTables(allTables, expectedTables);
assertEquals(actualTables, expectedTables);
String actualTable = database.getActualTable("number"); // get actual table number_*
String tableNumber = getNameByUpperCaseIdentifiers(actualTable);
String tableNewNumber = getNameByUpperCaseIdentifiers(actualTable.replace("number", "new_number"));
/* rename table number_* to new_number_* */
JdbcIdentity identity = JdbcIdentity.from(SESSION);
SchemaTableName newTableName = new SchemaTableName(schemaName, tableNewNumber);
clickHouseClient.renameTable(identity, null, schemaName, tableNumber, newTableName);
expectedTables.remove(3); //remove table "number_*"
expectedTables.add(actualTable.replace("number", "new_number")); //add table "new_number_*"
allTables = getAllTables(schemaName);
actualTables = getActualTables(allTables, expectedTables);
assertEquals(actualTables, expectedTables);
/* rename table new_number_* to number_* */
newTableName = new SchemaTableName(schemaName, tableNumber);
clickHouseClient.renameTable(identity, null, schemaName, tableNewNumber, newTableName);
expectedTables.remove(3); //remove table "new_number"
expectedTables.add(actualTable); //add table "number"
allTables = getAllTables(schemaName);
actualTables = getActualTables(allTables, expectedTables);
assertEquals(actualTables, expectedTables);
}
/**
* testRenameColumn
*/
@Test
public void testRenameColumn()
throws NoSuchFieldException, IllegalAccessException
{
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
// get actual table table_with_float_col_*
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("table_with_float_col"));
List<String> expectedColumns = Arrays.asList("col1", "col2", "col3", "col4");
List<String> actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
JdbcIdentity identity = JdbcIdentity.from(SESSION);
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
JdbcColumnHandle columnHandle = null;
for (JdbcColumnHandle column : columns) {
if ("col4".equalsIgnoreCase(column.getColumnName())) {
columnHandle = column;
break;
}
}
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
field.setAccessible(true);
field.set(tableHandle, null);
clickHouseClient.renameColumn(identity, tableHandle, columnHandle, "newcol4");
expectedColumns = Arrays.asList("col1", "col2", "col3", "newcol4");
actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
}
/**
* testAddColumn
*/
@Test
public void testAddColumn()
throws NoSuchFieldException, IllegalAccessException
{
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
// get actual table student_*
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("student"));
List<String> expectedColumns = Arrays.asList("id");
List<String> actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
JdbcIdentity identity = JdbcIdentity.from(SESSION);
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
ColumnMetadata columnMetadata = new ColumnMetadata("name", VARCHAR);
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
field.setAccessible(true);
field.set(tableHandle, null);
clickHouseClient.addColumn(SESSION, tableHandle, columnMetadata);
expectedColumns = Arrays.asList("id", "name");
actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
}
/**
* testDropColumn
*/
@Test
public void testDropColumn()
throws NoSuchFieldException, IllegalAccessException
{
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("example")); // get actual table example_*
List<String> expectedColumns = Arrays.asList("text", "text_short", "value");
List<String> actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
JdbcIdentity identity = JdbcIdentity.from(SESSION);
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
JdbcColumnHandle columnHandle = null;
for (JdbcColumnHandle column : columns) {
if ("value".equalsIgnoreCase(column.getColumnName())) {
columnHandle = column;
break;
}
}
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
field.setAccessible(true);
field.set(tableHandle, null);
clickHouseClient.dropColumn(identity, tableHandle, columnHandle);
expectedColumns = Arrays.asList("text", "text_short");
actualColumns = getActualColumns(schemaName, tableName);
assertEquals(actualColumns, expectedColumns);
}
@Test(enabled = false)
public void testGetColumns()
{
}
private String getNameByUpperCaseIdentifiers(String name)
{
try {
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
return name.toUpperCase(Locale.ENGLISH);
}
else {
return name;
}
}
catch (SQLException e) {
throw new RuntimeException("Failed to get metadata or storesUpperCaseIdentifiers", e);
}
}
private List<String> getAllTables(String schemaName)
{
List<String> allTables = new ArrayList<>();
try (ResultSet resultSet = clickHouseClient.getTables(connection, Optional.of(schemaName), Optional.empty())) {
while (resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
allTables.add(tableName.toLowerCase(Locale.ENGLISH));
}
return allTables;
}
catch (SQLException e) {
throw new RuntimeException("Failed to get tables", e);
}
}
private List<String> getActualTables(List<String> allTables, List<String> expectedTables)
{
List<String> actualTables = new ArrayList<>();
for (String table : expectedTables) {
if (allTables.contains(table)) {
actualTables.add(table);
}
}
return actualTables;
}
private List<String> getActualColumns(String schemaName, String tableName)
{
try (ResultSet resultSet = connection.getMetaData().getColumns(null, schemaName, tableName, null)) {
List<String> actualColumns = new ArrayList<>();
while (resultSet.next()) {
String columnName = resultSet.getString("COLUMN_NAME");
actualColumns.add(columnName.toLowerCase(Locale.ENGLISH));
}
return actualColumns;
}
catch (SQLException e) {
throw new RuntimeException("Failed to get metadata or columns", e);
}
}
}

View File

@ -0,0 +1,75 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.common.collect.ImmutableMap;
import org.testng.annotations.Test;
import java.util.Map;
import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
import static org.testng.Assert.assertEquals;
/**
* This is testing ClickHouseConfig functions
*/
public class ClickHouseConfigTest
{
private static final int COMMUNICATION_TIMEOUT = 10;
private static final int PACKET_SIZE = 150000;
/**
* This is testing ClickHousePropertyMappings
*/
@Test
public void testClickHousePropertyMappings()
{
final String tableTypes = "TABLE,VIEW,USER DEFINED,SYNONYM,OLAP VIEW,JOIN VIEW,HIERARCHY VIEW" + ",CALC VIEW,SYSTEM TABLE,NO LOGGING TEMPORARY,GLOBAL TEMPORARY";
final String schemaPattern = "default";
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("clickhouse.table-types", tableTypes)
.put("clickhouse.schema-pattern", schemaPattern)
.put("clickhouse.query.pushdown.enabled", "false")
.put("clickhouse.socket_timeout", "100")
.build();
ClickHouseConfig expected = new ClickHouseConfig()
.setTableTypes(tableTypes)
.setSchemaPattern(schemaPattern)
.setQueryPushDownEnabled(false)
.setSocketTimeout(100);
assertFullMapping(properties, expected);
}
/**
* This is testing ClickHouseConfig class inner functions
*/
@Test
public void testGetFuncions()
{
ClickHouseConfig config = new ClickHouseConfig();
String tableTypes = config.getTableTypes();
assertEquals(tableTypes, ClickHouseConstants.DEFAULT_TABLE_TYPES);
String schemaPattern = config.getSchemaPattern();
assertEquals(schemaPattern, null);
boolean isQueryPushDown = config.isQueryPushDownEnabled();
assertEquals(isQueryPushDown, true);
}
}

View File

@ -0,0 +1,27 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
public class ClickHouseConstantsTest
{
private ClickHouseConstantsTest()
{
}
/**
* "src/test/java/io/hetu/core/plugin/clickhouse/test-clickhouse-dev.properties"
*/
public static final String TEST_PROPERTY_FILE_PATH = "src/test/java/io/hetu/plugin/clickhouse/test-clickhouse-dev.properties";
}

View File

@ -0,0 +1,34 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import com.google.common.collect.ImmutableMap;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.testing.TestingConnectorContext;
import org.testng.annotations.Test;
import static com.google.common.collect.Iterables.getOnlyElement;
public class ClickHousePluginTest
{
@Test
public void testCreateConnector()
{
Plugin plugin = new ClickHousePlugin();
ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories());
factory.create("test", ImmutableMap.of("connection-url", "jdbc:clickhouse://test"), new TestingConnectorContext());
}
}

View File

@ -0,0 +1,221 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import io.airlift.log.Logger;
import io.airlift.tpch.TpchTable;
import javax.annotation.concurrent.GuardedBy;
import java.io.File;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
public final class ClickHouseServerTest
{
private static final Logger LOG = Logger.get(ClickHouseConstantsTest.class);
@GuardedBy("this")
private static int referenceCount;
@GuardedBy("this")
private static ClickHouseServerTest instance;
private final AtomicBoolean tpchLoaded = new AtomicBoolean(false);
private final CountDownLatch tpchLoadComplete = new CountDownLatch(1);
private static List<String> actualTables = new ArrayList<>();
private final AtomicBoolean isServerUsable = new AtomicBoolean(false);
private final AtomicInteger atomicIntegerTimes = new AtomicInteger(0);
private String jdbcUrl;
private String user;
private String password;
private String schema;
public static synchronized ClickHouseServerTest getInstance()
{
if (referenceCount == 0) {
instance = new ClickHouseServerTest();
instance.startup();
}
referenceCount++;
return instance;
}
public static synchronized void shutDown()
throws SQLException
{
referenceCount--;
if (referenceCount == 0) {
instance.shutdown();
instance = null;
}
}
private void startup()
{
for (TpchTable<?> table : TpchTable.getTables()) {
String newTable = generateNewTableName(table.getTableName());
actualTables.add(newTable);
}
}
private void shutdown()
throws SQLException
{
if (isClickHouseServerAvailable() && tpchLoaded.get()) {
for (String table : actualTables) {
String sql = "DROP TABLE " + schema + "." + table;
executeInClickHouse(sql);
}
}
}
private void executeInClickHouse(String sql)
throws SQLException
{
try (Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
Statement statement = connection.createStatement()) {
statement.execute(sql);
}
}
private ClickHouseServerTest()
{
File file = new File(ClickHouseConstantsTest.TEST_PROPERTY_FILE_PATH);
try {
Map<String, String> properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
LOG.info("test-clickhouse properties: %s", properties);
String connectionUrl = properties.get("connection.url");
String connectionUser = properties.get("connection.user");
String connectionPass = properties.get("connection.password");
String connectionSchema = properties.get("connection.schema");
if (connectionUrl != null) {
this.jdbcUrl = connectionUrl;
}
if (connectionUser != null) {
this.user = connectionUser;
}
if (connectionPass != null) {
this.password = connectionPass;
}
if (connectionSchema != null) {
this.schema = connectionSchema;
}
}
catch (IOException e) {
LOG.warn("Failed to load properties for file %s", file);
}
}
public boolean isClickHouseServerAvailable()
{
if (atomicIntegerTimes.getAndIncrement() > 0) {
return isServerUsable.get();
}
try {
Connection connection = DriverManager.getConnection(this.jdbcUrl, this.user, this.password);
isServerUsable.set(true);
return isServerUsable.get();
}
catch (SQLException e) {
isServerUsable.set(false);
return isServerUsable.get();
}
}
public String getJdbcUrl()
{
return jdbcUrl;
}
public String getUser()
{
return user;
}
public String getPassword()
{
return password;
}
public String getSchema()
{
return schema;
}
public boolean isTpchLoaded()
{
return tpchLoaded.getAndSet(true);
}
public void setTpchLoaded()
{
tpchLoadComplete.countDown();
}
public void waitTpchLoaded()
throws InterruptedException
{
tpchLoadComplete.await(2, TimeUnit.MINUTES);
}
public static String generateNewTableName(String tableName)
{
return tableName + "_" + UUID.randomUUID().toString().replace("-", "");
}
public static String getActualTable(String tablePattern)
{
return getActualTable(actualTables, tablePattern);
}
public static String getActualTable(List<String> tables, String tablePattern)
{
String actualTable = tablePattern;
for (String table : tables) { //tableName + _ + UUID
int lastIndex = table.lastIndexOf("_");
if (lastIndex == -1) {
continue;
}
if (table.substring(0, lastIndex).equalsIgnoreCase(tablePattern)) {
actualTable = tablePattern + table.substring(lastIndex);
break;
}
}
return actualTable;
}
}

View File

@ -0,0 +1,145 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.clickhouse;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.ConnectionFactory;
import io.prestosql.plugin.jdbc.DriverConnectionFactory;
import io.prestosql.spi.PrestoException;
import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
import java.lang.reflect.InvocationTargetException;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
public class DataBaseTest
implements AutoCloseable
{
private final Connection connection;
private final ClickHouseClient clickHouseClient;
private ClickHouseServerTest clickHouseServerTest;
private List<String> tables = new ArrayList<>();
/**
* constructor
*/
public DataBaseTest(ClickHouseServerTest clickHouseServer)
throws SQLException
{
this.clickHouseServerTest = clickHouseServer;
BaseJdbcConfig jdbcConfig = new BaseJdbcConfig();
ClickHouseConfig clickHouseConfig = new ClickHouseConfig();
jdbcConfig.setConnectionUrl(clickHouseServer.getJdbcUrl());
jdbcConfig.setConnectionUser(clickHouseServer.getUser());
jdbcConfig.setConnectionPassword(clickHouseServer.getPassword());
clickHouseConfig.setTableTypes("TABLE,VIEW");
clickHouseConfig.setSchemaPattern(clickHouseServer.getSchema());
clickHouseConfig.setQueryPushDownEnabled(false);
Driver driver = null;
try {
driver = (Driver) Class.forName(ClickHouseConstants.CLICKHOUSE_JDBC_DRIVER_CLASS_NAME).getConstructor(((Class<?>[]) null)).newInstance();
}
catch (InstantiationException e) {
throw new PrestoException(JDBC_ERROR, e);
}
catch (IllegalAccessException e) {
throw new PrestoException(JDBC_ERROR, e);
}
catch (ClassNotFoundException e) {
throw new PrestoException(JDBC_ERROR, e);
}
catch (InvocationTargetException e) {
throw new PrestoException(JDBC_ERROR, e);
}
catch (NoSuchMethodException e) {
throw new PrestoException(JDBC_ERROR, e);
}
ConnectionFactory connectionFactory = new DriverConnectionFactory(driver, jdbcConfig.getConnectionUrl(), Optional.ofNullable(jdbcConfig.getUserCredentialName()), Optional.ofNullable(jdbcConfig.getPasswordCredentialName()), basicConnectionProperties(jdbcConfig));
clickHouseClient = new ClickHouseClient(jdbcConfig, clickHouseConfig, connectionFactory, new JdbcExternalFunctionHub());
connection =
DriverManager.getConnection(clickHouseServer.getJdbcUrl(), clickHouseServer.getUser(), clickHouseServer.getPassword());
connection.createStatement()
.execute(buildCreateTableSql("example", "(text varchar,text_short varchar(32), value bigint)"));
connection.createStatement().execute(buildCreateTableSql("student", "(id varchar)"));
connection.createStatement()
.execute(buildCreateTableSql("table_with_float_col", "(col1 bigint, col2 double, col3 float, col4 real)"));
connection.createStatement()
.execute(buildCreateTableSql("number", "(text varchar, text_short varchar(32), value bigint)"));
}
@Override
public void close()
throws SQLException
{
for (String table : tables) {
String sql = "DROP TABLE " + clickHouseServerTest.getSchema() + "." + table;
connection.createStatement().execute(sql);
}
ClickHouseServerTest.shutDown();
}
public Connection getConnection()
{
return connection;
}
public ClickHouseClient getClickHouseClient()
{
return clickHouseClient;
}
public List<String> getTables()
{
return tables;
}
public String getSchema()
{
return clickHouseServerTest.getSchema();
}
public String getActualTable(String tablePattern)
{
return ClickHouseServerTest.getActualTable(tables, tablePattern);
}
private String buildCreateTableSql(String tableName, String columnInfo)
{
String newTableName = ClickHouseServerTest.generateNewTableName(tableName);
tables.add(newTableName);
String sql = "CREATE TABLE " + clickHouseServerTest.getSchema() + "." + newTableName + columnInfo + "engine=MergeTree() order by tuple()";
System.out.println(sql);
return sql;
}
}

View File

@ -0,0 +1,10 @@
#
# WARNING
# ^^^^^^^
# This configuration file is for development only and should NOT be used
# in production. For example configuration, see the Hetu documentation.
# please set your data source info here
connection.url=jdbc:clickhouse://localhost:8123
connection.user=default
connection.password=yourpassword
connection.schema=test

View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
<suite name="TestGroupSuite">
<test name="GroupTest">
<classes>
<class name="io.hetu.core.plugin.clickhouse.ClickHouseConfigTest"/>
<class name="io.hetu.core.plugin.clickhouse.ClickHouseClientTest"/>
<class name="io.hetu.core.plugin.clickhouse.ClickHousePluginTest"/>
</classes>
</test>
</suite>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-common</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-cube</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-datacenter</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-function-namespace-managers</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-hana</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-hazelcast</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-hbase</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -152,7 +152,7 @@
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-postgresql</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
<scope>compile</scope>
<exclusions>
<exclusion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-oracle</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-server-rpm</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-server</artifactId>
@ -18,6 +18,7 @@
<air.check.skip-enforcer>true</air.check.skip-enforcer>
<air.check.skip-duplicate-finder>true</air.check.skip-duplicate-finder>
<air.check.skip-findbugs>true</air.check.skip-findbugs>
<air.check.skip-dependency>true</air.check.skip-dependency>
<!-- Launcher properties -->

View File

@ -313,4 +313,10 @@
<unpack />
</artifact>
</artifactSet>
<artifactSet to="plugin/clickhouse">
<artifact id="${project.groupId}:hetu-clickhouse:zip:${project.version}">
<unpack />
</artifact>
</artifactSet>
</runtime>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-sql-migration-tool</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-startree</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-vdm</artifactId>

View File

@ -10,7 +10,7 @@
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>presto-root</name>
@ -151,6 +151,7 @@
<module>hetu-hazelcast</module>
<module>hetu-function-namespace-managers</module>
<module>hetu-greenplum</module>
<module>hetu-clickhouse</module>
</modules>
<dependencyManagement>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-array</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-atop</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-base-jdbc</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-benchmark-driver</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-benchmark</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-benchto-benchmarks</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-cli</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-client</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-elasticsearch</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-example-http</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-expressions</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-geospatial-toolkit</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-geospatial</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-hive-hadoop2</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-hive</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>hetu-jdbc</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-jmx</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-kafka</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-local-file</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-main</artifactId>
@ -185,73 +185,6 @@
<dependency>
<groupId>io.airlift.resolver</groupId>
<artifactId>resolver</artifactId>
<exclusions>
<exclusion>
<artifactId>netty</artifactId>
<groupId>io.netty</groupId>
</exclusion>
<exclusion>
<artifactId>async-http-client</artifactId>
<groupId>com.ning</groupId>
</exclusion>
<exclusion>
<artifactId>maven-compat</artifactId>
<groupId>org.apache.maven</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-compat</artifactId>
<version>3.5.0</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
<artifactId>maven-core</artifactId>
<groupId>org.apache.maven</groupId>
</exclusion>
<exclusion>
<artifactId>maven-model</artifactId>
<groupId>org.apache.maven</groupId>
</exclusion>
<exclusion>
<artifactId>maven-repository-metadata</artifactId>
<groupId>org.apache.maven</groupId>
</exclusion>
<exclusion>
<artifactId>plexus-classworlds</artifactId>
<groupId>org.codehaus.plexus</groupId>
</exclusion>
<exclusion>
<artifactId>jsr250-api</artifactId>
<groupId>javax.annotation</groupId>
</exclusion>
<exclusion>
<artifactId>maven-resolver-provider</artifactId>
<groupId>org.apache.maven</groupId>
</exclusion>
<exclusion>
<artifactId>org.eclipse.sisu.plexus</artifactId>
<groupId>org.eclipse.sisu</groupId>
</exclusion>
<exclusion>
<artifactId>maven-resolver-api</artifactId>
<groupId>org.apache.maven.resolver</groupId>
</exclusion>
<exclusion>
<artifactId>maven-resolver-impl</artifactId>
<groupId>org.apache.maven.resolver</groupId>
</exclusion>
<exclusion>
<artifactId>maven-resolver-spi</artifactId>
<groupId>org.apache.maven.resolver</groupId>
</exclusion>
<exclusion>
<artifactId>maven-resolver-util</artifactId>
<groupId>org.apache.maven.resolver</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>

View File

@ -18,7 +18,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-matching</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-memory-context</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-memory</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-ml</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-mysql</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-orc</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-parquet</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-parser</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-password-authenticators</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-plugin-toolkit</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-postgresql</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-product-tests</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-proxy</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-rcfile</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-record-decoder</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-resource-group-managers</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-session-property-managers</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-spi</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-teradata-functions</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-testing-docker</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-testing-server-launcher</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-tests</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-thrift-api</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-thrift-testing-server</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.2.0-SNAPSHOT</version>
<version>1.2.1-SNAPSHOT</version>
</parent>
<artifactId>presto-thrift</artifactId>

Some files were not shown because too many files have changed in this diff Show More