Compare commits
2 Commits
master
...
revert-mer
| Author | SHA1 | Date |
|---|---|---|
lifengzi
|
e50b4ce963 | |
|
bryanwong
|
5e742e2ba3 |
|
|
@ -1,5 +1,5 @@
|
|||
### Software Environment:
|
||||
- openLooKeng version (source or binary):
|
||||
- OpenLooKeng version (source or binary):
|
||||
|
||||
- OS platform & distribution (eg., Linux Ubuntu 16.04):
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
*.iml
|
||||
*.ipr
|
||||
*.iws
|
||||
*.icloud
|
||||
target/
|
||||
/var
|
||||
/*/var/
|
||||
|
|
@ -28,4 +27,4 @@ node_modules
|
|||
*/*.iml
|
||||
*/.idea
|
||||
*/.gitignore
|
||||
*/dependency-reduced-pom.xml
|
||||
*/dependency-reduced-pom.xml
|
||||
5
OWNERS
5
OWNERS
|
|
@ -4,7 +4,6 @@ approvers:
|
|||
- karmvs
|
||||
- sraghunandan
|
||||
- haochending
|
||||
- farhan3
|
||||
- slgobinath
|
||||
- fbird2020
|
||||
- lizheng920625
|
||||
- giteezhangjingfang
|
||||
- lizheng920625
|
||||
|
|
@ -53,9 +53,5 @@ For ORC files, the ORC row data cache provides an efficient method to cache freq
|
|||
#### Execution Plan Cache
|
||||
The execution plan is cached after the first query, rather than discarded after each query request, thereby reducing the preprocessing time and resources required for subsequent queries. By caching these plans, time-consuming plan generation steps can be skipped.
|
||||
|
||||
## Obtaining the Design Document
|
||||
|
||||
Click [here](https://openlookeng.slite.com/p/channel/EDMAZKydV2MsM5trxJPmLv#) to obtain the design document.
|
||||
|
||||
## Next Steps
|
||||
[Developer Guide](hetu-docs/en/develop/_index.md)
|
||||
|
|
|
|||
|
|
@ -177,7 +177,6 @@ if [[ $# != 0 ]]; then
|
|||
if [[ $discoveryURI = *" "* ]]; then
|
||||
echo "URI cannot contain spaces"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
|
|
@ -186,7 +185,6 @@ if [[ $# != 0 ]]; then
|
|||
if [[ $includeCoordinator != true && $includeCoordinator != false ]]; then
|
||||
echo "includeCoordinator must be boolean"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
|
|
@ -200,7 +198,6 @@ if [[ $# != 0 ]]; then
|
|||
if [[ ! $xmx =~ ^[0-9]+[g|G|m|M|k|K]$ ]]; then
|
||||
echo "Invalid xmx"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
shift
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
|
|
@ -48,7 +48,7 @@ function test_container {
|
|||
I=0
|
||||
# check if hetu instance is running
|
||||
sleep ${QUERY_PERIOD}
|
||||
until RESULT=$(docker exec "${CONTAINER_ID}" openlk --execute "SELECT 'success'" | tail -1); do
|
||||
until RESULT=$(docker exec "${CONTAINER_ID}" openlk --execute "SELECT 'success'"); do
|
||||
if [[ $((I++)) -ge ${QUERY_RETRIES} ]]; then
|
||||
echo "Too many retries waiting for Hetu to start."
|
||||
break
|
||||
|
|
|
|||
|
|
@ -0,0 +1,14 @@
|
|||
# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
connector.name=memory
|
||||
|
|
@ -22,7 +22,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-carbondata</artifactId>
|
||||
|
|
@ -34,7 +34,7 @@
|
|||
<httpcore.version>4.4.9</httpcore.version>
|
||||
<jacoco.append>true</jacoco.append>
|
||||
<carbon.version>2.0.1</carbon.version>
|
||||
<scala.version>2.11.12</scala.version>
|
||||
<scala.version>2.11.8</scala.version>
|
||||
<scala.binary.version>2.11</scala.binary.version>
|
||||
<hadoop.version>3.2.0</hadoop.version>
|
||||
<spark.version>2.4.5</spark.version>
|
||||
|
|
@ -361,17 +361,13 @@
|
|||
<artifactId>bootstrap</artifactId>
|
||||
<groupId>io.airlift</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<version>1.21</version>
|
||||
<version>1.19</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
@ -687,10 +683,6 @@
|
|||
<artifactId>parquet-hadoop-bundle</artifactId>
|
||||
<groupId>org.apache.parquet</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>groovy-all</artifactId>
|
||||
<groupId>org.codehaus.groovy</groupId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
@ -715,11 +707,6 @@
|
|||
<artifactId>commons-io</artifactId>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-parser</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-codec</groupId>
|
||||
<artifactId>commons-codec</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,306 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.plugin.carbondata;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import io.hetu.core.plugin.carbondata.impl.CarbondataTableReader;
|
||||
import io.prestosql.plugin.hive.HdfsEnvironment;
|
||||
import io.prestosql.plugin.hive.authentication.HiveIdentity;
|
||||
import io.prestosql.plugin.hive.metastore.MetastoreUtil;
|
||||
import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
|
||||
import io.prestosql.plugin.hive.metastore.Table;
|
||||
import io.prestosql.plugin.hive.util.ConfigurationUtils;
|
||||
import io.prestosql.spi.connector.ConnectorVacuumTableInfo;
|
||||
import io.prestosql.spi.security.ConnectorIdentity;
|
||||
import org.apache.carbondata.common.logging.LogServiceFactory;
|
||||
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
|
||||
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
|
||||
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
|
||||
import org.apache.carbondata.hive.util.HiveCarbonUtil;
|
||||
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
|
||||
import org.apache.carbondata.processing.merger.CompactionType;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ConcurrentSkipListSet;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
|
||||
import static java.util.Collections.emptyList;
|
||||
|
||||
public class CarbondataAutoVacuumThread
|
||||
{
|
||||
private static final Logger LOG =
|
||||
LogServiceFactory.getLogService(CarbondataAutoVacuumThread.class.getName());
|
||||
private static final ConcurrentSkipListSet<Future<?>> queuedTasks = new ConcurrentSkipListSet<>();
|
||||
private final CarbondataTableReader carbondataTableReader;
|
||||
private final long majorVacuumSegSize;
|
||||
private final long minorVacuumSegCount;
|
||||
private final ScheduledExecutorService executorService;
|
||||
private final HdfsEnvironment hdfsEnvironment;
|
||||
private final List<ConnectorVacuumTableInfo> connectorVacuumTableInfoList;
|
||||
private final Map<String, Long> needToVacuumTablesMap;
|
||||
private final HdfsEnvironment.HdfsContext hdfsContext;
|
||||
private static CarbondataAutoVacuumThread instance;
|
||||
private static boolean enableTracingCleanupTask;
|
||||
private static CarbondataAutoVacuumThreadInfo carbondataAutoVacuumThreadInfo;
|
||||
|
||||
public static synchronized CarbondataAutoVacuumThread getInstanceAutoVacuum()
|
||||
{
|
||||
if (instance == null) {
|
||||
instance = new CarbondataAutoVacuumThread(carbondataAutoVacuumThreadInfo.carbondataTableReader, carbondataAutoVacuumThreadInfo.hdfsEnvironment,
|
||||
carbondataAutoVacuumThreadInfo.majorVacuumSegSize, carbondataAutoVacuumThreadInfo.minorVacuumSegCount, carbondataAutoVacuumThreadInfo.executorService);
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
public static synchronized void setAutoVacuumInfo(CarbondataTableReader carbondataTableReader, HdfsEnvironment hdfsEnvironment,
|
||||
long majorVacuumSegSize, long minorVacuumSegCount,
|
||||
ScheduledExecutorService vacuumExecutorService)
|
||||
{
|
||||
if (carbondataAutoVacuumThreadInfo == null) {
|
||||
carbondataAutoVacuumThreadInfo = new CarbondataAutoVacuumThreadInfo(carbondataTableReader, hdfsEnvironment,
|
||||
majorVacuumSegSize, minorVacuumSegCount, vacuumExecutorService);
|
||||
}
|
||||
}
|
||||
|
||||
private CarbondataAutoVacuumThread(CarbondataTableReader carbondataTableReader, HdfsEnvironment hdfsEnvironment,
|
||||
long majorVacuumSegSize, long minorVacuumSegCount,
|
||||
ScheduledExecutorService vacuumExecutorService)
|
||||
{
|
||||
this.carbondataTableReader = carbondataTableReader;
|
||||
this.hdfsEnvironment = hdfsEnvironment;
|
||||
this.majorVacuumSegSize = majorVacuumSegSize;
|
||||
this.minorVacuumSegCount = minorVacuumSegCount;
|
||||
this.connectorVacuumTableInfoList = new ArrayList<>();
|
||||
this.needToVacuumTablesMap = new ConcurrentHashMap<>();
|
||||
this.executorService = vacuumExecutorService;
|
||||
this.hdfsContext = new HdfsEnvironment.HdfsContext(
|
||||
new ConnectorIdentity("openLooKeng", Optional.empty(), Optional.empty()));
|
||||
}
|
||||
|
||||
class AutoVacuumScanTask
|
||||
implements Runnable
|
||||
{
|
||||
SemiTransactionalHiveMetastore metastore;
|
||||
String schemaName;
|
||||
|
||||
AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
|
||||
{
|
||||
this(metastore, null);
|
||||
}
|
||||
|
||||
AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
|
||||
{
|
||||
this.metastore = metastore;
|
||||
this.schemaName = schemaName;
|
||||
}
|
||||
|
||||
public void run()
|
||||
{
|
||||
hdfsEnvironment.doAs("openLooKeng", () -> {
|
||||
// it will scan through all DB and Tables
|
||||
if (null == schemaName) {
|
||||
scanSchemasAndTables(metastore);
|
||||
}
|
||||
else {
|
||||
scanSchemas(metastore, schemaName);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private void scanSchemasAndTables(SemiTransactionalHiveMetastore metastore)
|
||||
{
|
||||
try {
|
||||
for (String schemaName : metastore.getAllDatabases()) {
|
||||
//when multiple task are there, every schema is submit to task, to scan and identify vacuum tables.
|
||||
if (enableTracingCleanupTask) {
|
||||
queuedTasks.add(executorService.submit(new AutoVacuumScanTask(metastore, schemaName)));
|
||||
}
|
||||
else {
|
||||
executorService.submit(new AutoVacuumScanTask(metastore, schemaName));
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
LOG.error("Error when scanning schema", e);
|
||||
//ignore error cases
|
||||
}
|
||||
}
|
||||
|
||||
private void identifyVacuumTables(Table target, String schemaName, String tableName, String dbNameTableName)
|
||||
{
|
||||
Configuration initialConfiguration = ConfigurationUtils.toJobConf(hdfsEnvironment
|
||||
.getConfiguration(hdfsContext, new Path(target.getStorage().getLocation())));
|
||||
|
||||
Properties schema = MetastoreUtil.getHiveSchema(target);
|
||||
schema.setProperty("tablePath", target.getStorage().getLocation());
|
||||
CarbonTable carbonTable;
|
||||
try {
|
||||
carbonTable = CarbondataMetadata.getCarbonTable(schemaName, tableName, schema, initialConfiguration, carbondataTableReader);
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (null != carbonTable) {
|
||||
CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(schema, initialConfiguration);
|
||||
// in carbondata we support only miner vacuum
|
||||
List<List<LoadMetadataDetails>> allGroupedSegList = CarbondataHetuCompactorUtil.identifyAndGroupSegmentsToBeMerged(carbonLoadModel,
|
||||
initialConfiguration, CompactionType.MINOR, majorVacuumSegSize, minorVacuumSegCount);
|
||||
|
||||
if (!allGroupedSegList.isEmpty()) {
|
||||
ConnectorVacuumTableInfo connectorVacuumTableInfo = new ConnectorVacuumTableInfo(dbNameTableName, false);
|
||||
|
||||
synchronized (connectorVacuumTableInfoList) {
|
||||
//Selected Minor Vacuum tables are inserted in to list
|
||||
connectorVacuumTableInfoList.add(connectorVacuumTableInfo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
LOG.error("error when identify Vacuum Table " + schemaName + "." + tableName, e);
|
||||
//Ignore error cases and continue
|
||||
}
|
||||
}
|
||||
|
||||
private void scanSchemas(SemiTransactionalHiveMetastore metastore, String schemaName)
|
||||
{
|
||||
try {
|
||||
for (String tableName : metastore.getAllTables(schemaName).orElse(emptyList())) {
|
||||
Optional<Table> target = metastore.getTable(new HiveIdentity(hdfsContext.getIdentity()), schemaName, tableName);
|
||||
if (!target.isPresent()) {
|
||||
continue;
|
||||
}
|
||||
String dbNameTableName = target.get().getDatabaseName() + "." + target.get().getTableName();
|
||||
|
||||
if (needToVacuumTablesMap.containsKey(dbNameTableName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (target.get().getStorage().getStorageFormat().getInputFormat().contains("carbon")) {
|
||||
identifyVacuumTables(target.get(), schemaName, tableName, dbNameTableName);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
LOG.error("error when scanning Tables", e);
|
||||
//Ignore error cases and continue
|
||||
}
|
||||
}
|
||||
|
||||
public static List<ConnectorVacuumTableInfo> getAutoVacuumTableList(SemiTransactionalHiveMetastore metastore)
|
||||
{
|
||||
CarbondataAutoVacuumThread instanceAutoVacuum = getInstanceAutoVacuum();
|
||||
if (null == instanceAutoVacuum) {
|
||||
return null;
|
||||
}
|
||||
|
||||
List<ConnectorVacuumTableInfo> connectorVacuumTableInfos = new ArrayList<>();
|
||||
synchronized (instanceAutoVacuum.connectorVacuumTableInfoList) {
|
||||
connectorVacuumTableInfos.addAll(instanceAutoVacuum.connectorVacuumTableInfoList);
|
||||
instanceAutoVacuum.connectorVacuumTableInfoList.clear();
|
||||
}
|
||||
//trigger task to do scanning of tables
|
||||
instanceAutoVacuum.submitTaskScanning(instanceAutoVacuum, metastore);
|
||||
return connectorVacuumTableInfos;
|
||||
}
|
||||
|
||||
private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
|
||||
{
|
||||
//trigger task to do scanning of tables
|
||||
if (enableTracingCleanupTask) {
|
||||
queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
|
||||
}
|
||||
else {
|
||||
instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
|
||||
}
|
||||
}
|
||||
|
||||
public static void removeTableFromVacuumTablesMap(String schemaTableName)
|
||||
{
|
||||
CarbondataAutoVacuumThread autoVacuumObj = getInstanceAutoVacuum();
|
||||
if ((null != autoVacuumObj) && (null != autoVacuumObj.needToVacuumTablesMap.get(schemaTableName))) {
|
||||
autoVacuumObj.needToVacuumTablesMap.remove(schemaTableName);
|
||||
}
|
||||
}
|
||||
|
||||
public static void addTableToVacuumTablesMap(String schemaTableName)
|
||||
{
|
||||
CarbondataAutoVacuumThread autoVacuumObj = getInstanceAutoVacuum();
|
||||
if (null != autoVacuumObj && (null != autoVacuumObj.needToVacuumTablesMap.get(schemaTableName))) {
|
||||
autoVacuumObj.needToVacuumTablesMap.put(schemaTableName, CarbonUpdateUtil.readCurrentTime());
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void enableTracingVacuumTask(boolean isEnabled)
|
||||
{
|
||||
enableTracingCleanupTask = isEnabled;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public static void waitForSubmittedVacuumTasksFinish()
|
||||
{
|
||||
queuedTasks.stream().forEach(f -> {
|
||||
try {
|
||||
f.get();
|
||||
}
|
||||
catch (InterruptedException e) {
|
||||
LOG.debug("Interrupted to get the result of autoCleanup : " + e);
|
||||
}
|
||||
catch (ExecutionException e) {
|
||||
LOG.debug("Exception to get the result of autoCleanup : " + e);
|
||||
}
|
||||
});
|
||||
queuedTasks.clear();
|
||||
LOG.info("All autocleanup tasks finished");
|
||||
}
|
||||
|
||||
static class CarbondataAutoVacuumThreadInfo
|
||||
{
|
||||
public final CarbondataTableReader carbondataTableReader;
|
||||
public final HdfsEnvironment hdfsEnvironment;
|
||||
public final long majorVacuumSegSize;
|
||||
public final long minorVacuumSegCount;
|
||||
public final ScheduledExecutorService executorService;
|
||||
public final List<ConnectorVacuumTableInfo> connectorVacuumTableInfoList;
|
||||
public final Map<String, Long> needToVacuumTablesMap;
|
||||
|
||||
private CarbondataAutoVacuumThreadInfo(CarbondataTableReader carbondataTableReader, HdfsEnvironment hdfsEnvironment,
|
||||
long majorVacuumSegSize, long minorVacuumSegCount,
|
||||
ScheduledExecutorService vacuumExecutorService)
|
||||
{
|
||||
this.carbondataTableReader = carbondataTableReader;
|
||||
this.hdfsEnvironment = hdfsEnvironment;
|
||||
this.majorVacuumSegSize = majorVacuumSegSize;
|
||||
this.minorVacuumSegCount = minorVacuumSegCount;
|
||||
this.connectorVacuumTableInfoList = new ArrayList<>();
|
||||
this.needToVacuumTablesMap = new ConcurrentHashMap<>();
|
||||
this.executorService = vacuumExecutorService;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -73,17 +73,16 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putShorts(int rowId, int count, short value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
if (filteredRowsExist) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putShort(counter++, value);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
columnVector.putShorts(inputRowId, count, value);
|
||||
columnVector.putShorts(rowId, count, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -98,17 +97,16 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putInts(int rowId, int count, int value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
if (filteredRowsExist) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putInt(counter++, value);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
columnVector.putInts(inputRowId, count, value);
|
||||
columnVector.putInts(rowId, count, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -123,17 +121,16 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putLongs(int rowId, int count, long value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
if (filteredRowsExist) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putLong(counter++, value);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
columnVector.putLongs(inputRowId, count, value);
|
||||
columnVector.putLongs(rowId, count, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -148,12 +145,11 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putDecimals(int rowId, int count, BigDecimal value, int precision)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putDecimal(counter++, value, precision);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -168,17 +164,16 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putDoubles(int rowId, int count, double value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
if (filteredRowsExist) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putDouble(counter++, value);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
columnVector.putDoubles(inputRowId, count, value);
|
||||
columnVector.putDoubles(rowId, count, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -201,12 +196,11 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putByteArray(int rowId, int count, byte[] value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putByteArray(counter++, value);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -229,17 +223,16 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putNulls(int rowId, int count)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
if (filteredRowsExist) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putNull(counter++);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
columnVector.putNulls(inputRowId, count);
|
||||
columnVector.putNulls(rowId, count);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -326,72 +319,66 @@ public class CarbondataColumnVectorWrapper
|
|||
@Override
|
||||
public void putFloats(int rowId, int count, float[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putFloat(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putShorts(int rowId, int count, short[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putShort(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putInts(int rowId, int count, int[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putInt(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putLongs(int rowId, int count, long[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putLong(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putDoubles(int rowId, int count, double[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putDouble(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putBytes(int rowId, int count, byte[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = srcIndex; i < count; i++) {
|
||||
if (!filteredRows[inputRowId]) {
|
||||
if (!filteredRows[rowId]) {
|
||||
columnVector.putByte(counter++, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ public class CarbondataConfig
|
|||
private long minorVacuumSegCount = Long.parseLong(CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD.split(",")[0]);
|
||||
private long majorVacuumSegSize = 1L;
|
||||
private int vacuumServiceThreads = 2;
|
||||
private boolean autoVacuumEnable;
|
||||
private CarbondataStorageFormat carbondataStorageFormat = CarbondataStorageFormat.CARBON;
|
||||
|
||||
@NotNull
|
||||
|
|
@ -83,17 +82,4 @@ public class CarbondataConfig
|
|||
{
|
||||
return vacuumServiceThreads;
|
||||
}
|
||||
|
||||
@Config("carbondata.auto-vacuum-enabled")
|
||||
@ConfigDescription("carbondata auto vacuum default value is true.")
|
||||
public CarbondataConfig setAutoVacuumEnable(boolean autoVacuumEnable)
|
||||
{
|
||||
this.autoVacuumEnable = autoVacuumEnable;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean getAutoVacuumEnable()
|
||||
{
|
||||
return autoVacuumEnable;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import com.google.gson.Gson;
|
|||
import io.prestosql.plugin.hive.HiveACIDWriteType;
|
||||
import io.prestosql.plugin.hive.HiveFileWriter;
|
||||
import io.prestosql.plugin.hive.HiveType;
|
||||
import io.prestosql.plugin.hive.util.FieldSetterFactory;
|
||||
import io.prestosql.plugin.hive.HiveWriteUtils;
|
||||
import io.prestosql.spi.Page;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.block.Block;
|
||||
|
|
@ -64,7 +64,6 @@ import org.apache.hadoop.mapred.Reporter;
|
|||
import org.apache.hadoop.mapred.TaskAttemptID;
|
||||
import org.apache.hadoop.mapreduce.TaskType;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.joda.time.DateTimeZone;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.UncheckedIOException;
|
||||
|
|
@ -94,7 +93,6 @@ public class CarbondataFileWriter
|
|||
{
|
||||
private static final Logger LOG =
|
||||
LogServiceFactory.getLogService(CarbondataFileWriter.class.getName());
|
||||
private static final io.airlift.log.Logger AIR_LOG = io.airlift.log.Logger.get(CarbondataFileWriter.class);
|
||||
|
||||
private static final String LOAD_MODEL = "mapreduce.carbontable.load.model";
|
||||
|
||||
|
|
@ -105,7 +103,7 @@ public class CarbondataFileWriter
|
|||
private final Object row;
|
||||
private final SettableStructObjectInspector tableInspector;
|
||||
private final List<StructField> structFields;
|
||||
private final FieldSetterFactory.FieldSetter[] setters;
|
||||
private final HiveWriteUtils.FieldSetter[] setters;
|
||||
private final Properties properties;
|
||||
private final Optional<AcidOutputFormat.Options> acidOptions;
|
||||
private final HiveACIDWriteType acidWriteType;
|
||||
|
|
@ -127,16 +125,15 @@ public class CarbondataFileWriter
|
|||
private boolean isInitDone;
|
||||
private boolean isCommitDone;
|
||||
|
||||
public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
|
||||
public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
|
||||
JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
|
||||
Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
|
||||
{
|
||||
Path localOutPutPath = paramOutPutPath;
|
||||
this.outPutPath = requireNonNull(localOutPutPath, "path is null");
|
||||
this.outPutPath = requireNonNull(outPutPath, "path is null");
|
||||
// in table creation this can be null
|
||||
if (null != properties.getProperty("location")) {
|
||||
this.outPutPath = new Path(properties.getProperty("location"));
|
||||
localOutPutPath = new Path(properties.getProperty("location"));
|
||||
outPutPath = new Path(properties.getProperty("location"));
|
||||
}
|
||||
this.configuration = requireNonNull(configuration, "conf is null");
|
||||
this.properties = requireNonNull(properties, "Properties is null");
|
||||
|
|
@ -146,7 +143,6 @@ public class CarbondataFileWriter
|
|||
Long.toString(System.currentTimeMillis()));
|
||||
this.taskId = taskId.orElseGet(() -> 0);
|
||||
|
||||
AIR_LOG.debug("[carbonWriterTask] taskId: " + this.taskId + ", outputPath: " + this.outPutPath);
|
||||
try {
|
||||
if (HiveACIDWriteType.isUpdateOrDelete(this.acidWriteType)) {
|
||||
String encodedCarbonTable = configuration.get(CarbondataConstants.CarbonTable);
|
||||
|
|
@ -185,16 +181,13 @@ public class CarbondataFileWriter
|
|||
|
||||
row = tableInspector.create();
|
||||
|
||||
setters = new FieldSetterFactory.FieldSetter[structFields.size()];
|
||||
|
||||
FieldSetterFactory fieldSetterFactory = new FieldSetterFactory(DateTimeZone.UTC);
|
||||
|
||||
setters = new HiveWriteUtils.FieldSetter[structFields.size()];
|
||||
for (int i = 0; i < setters.length; i++) {
|
||||
setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i),
|
||||
setters[i] = HiveWriteUtils.createFieldSetter(tableInspector, row, structFields.get(i),
|
||||
fileColumnTypes.get(structFields.get(i).getFieldID()));
|
||||
}
|
||||
|
||||
if (this.acidWriteType == HiveACIDWriteType.INSERT || this.acidWriteType == HiveACIDWriteType.INSERT_OVERWRITE) {
|
||||
if (this.acidWriteType == HiveACIDWriteType.INSERT | this.acidWriteType == HiveACIDWriteType.INSERT_OVERWRITE) {
|
||||
try {
|
||||
boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
|
||||
|
||||
|
|
@ -212,7 +205,7 @@ public class CarbondataFileWriter
|
|||
Object writer =
|
||||
Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
|
||||
recordWriter = ((MapredCarbonOutputFormat<?>) writer)
|
||||
.getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
|
||||
.getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
|
||||
properties, Reporter.NULL);
|
||||
}
|
||||
|
||||
|
|
@ -227,25 +220,25 @@ public class CarbondataFileWriter
|
|||
|
||||
private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
|
||||
{
|
||||
Path finalOutPutPath = this.outPutPath;
|
||||
Properties finalProperties = this.properties;
|
||||
JobConf finalConfiguration = this.configuration;
|
||||
boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
|
||||
Path outPutPath = this.outPutPath;
|
||||
Properties properties = this.properties;
|
||||
JobConf configuration = this.configuration;
|
||||
boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
|
||||
|
||||
CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
|
||||
CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
|
||||
carbonLoadModel.setSegmentId(segmentId);
|
||||
carbonLoadModel.setTaskNo(String.valueOf(taskNo));
|
||||
carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
|
||||
carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
|
||||
|
||||
CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
|
||||
CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
|
||||
this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
|
||||
|
||||
Object writer =
|
||||
Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
|
||||
return ((MapredCarbonOutputFormat<?>) writer)
|
||||
.getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
|
||||
finalProperties, Reporter.NULL);
|
||||
.getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
|
||||
properties, Reporter.NULL);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -279,14 +272,14 @@ public class CarbondataFileWriter
|
|||
private String getUpdateTupleIdFromRec(Page dataPage, int position)
|
||||
{
|
||||
Block tupleIdBlock = null;
|
||||
tupleIdBlock = dataPage.getBlock(dataPage.getChannelCount() - 1).getLoadedBlock();
|
||||
tupleIdBlock = dataPage.getBlock(fieldCount).getLoadedBlock();
|
||||
|
||||
return tupleIdBlock.getString(position, 0, 0);
|
||||
}
|
||||
|
||||
public void appendRow(Page dataPage, int position)
|
||||
{
|
||||
FileSinkOperator.RecordWriter finalRecordWriter = null;
|
||||
FileSinkOperator.RecordWriter recordWriter = null;
|
||||
if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
|
||||
try {
|
||||
DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
|
||||
|
|
@ -335,7 +328,7 @@ public class CarbondataFileWriter
|
|||
return;
|
||||
}
|
||||
|
||||
finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
|
||||
recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
|
||||
{
|
||||
try {
|
||||
return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
|
||||
|
|
@ -352,7 +345,7 @@ public class CarbondataFileWriter
|
|||
}
|
||||
}
|
||||
else {
|
||||
finalRecordWriter = this.recordWriter;
|
||||
recordWriter = this.recordWriter;
|
||||
}
|
||||
|
||||
for (int field = 0; field < fieldCount; field++) {
|
||||
|
|
@ -366,8 +359,8 @@ public class CarbondataFileWriter
|
|||
}
|
||||
|
||||
try {
|
||||
if (finalRecordWriter != null) {
|
||||
finalRecordWriter.write(serDe.serialize(row, tableInspector));
|
||||
if (recordWriter != null) {
|
||||
recordWriter.write(serDe.serialize(row, tableInspector));
|
||||
}
|
||||
}
|
||||
catch (SerDeException | IOException e) {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ import org.apache.hadoop.mapred.JobConf;
|
|||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.OptionalInt;
|
||||
import java.util.Properties;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
|
@ -68,19 +67,10 @@ public class CarbondataFileWriterFactory
|
|||
Optional<HiveACIDWriteType> acidWriteType)
|
||||
{
|
||||
try {
|
||||
int taskId = session.getTaskId().getAsInt();
|
||||
int driverId = session.getDriverId().getAsInt();
|
||||
int taskWriterCount = session.getTaskWriterCount();
|
||||
//taskId starts from 0.
|
||||
//driverId starts from 0 and will be < taskWriterCount.
|
||||
//taskWriterCount starts from 1
|
||||
// for taskId n, buckets will be between n*taskWriterCount (inclusive) and (n+1)*taskWriterCount (exclusive)
|
||||
int bucketNumber = taskId * taskWriterCount + driverId;
|
||||
|
||||
/* Create a DeleteDeltaFileWriter */
|
||||
return Optional
|
||||
.of(new CarbondataFileWriter(path, inputColumnNames, schema, configuration,
|
||||
typeManager, acidOptions, acidWriteType, OptionalInt.of(bucketNumber)));
|
||||
typeManager, acidOptions, acidWriteType, session.getTaskId()));
|
||||
}
|
||||
catch (SerDeException e) {
|
||||
throw new RuntimeException("Error while creating carbon file writer", e);
|
||||
|
|
|
|||
|
|
@ -48,7 +48,6 @@ public class CarbondataHandleResolver
|
|||
return CarbonDeleteAsInsertTableHandle.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
|
||||
{
|
||||
return CarbondataOutputTableHandle.class;
|
||||
|
|
|
|||
|
|
@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
|
|||
if (rawData instanceof Slice) {
|
||||
String value = ((Slice) rawData).toStringUtf8();
|
||||
if (type.getTypeInfo() instanceof CharTypeInfo) {
|
||||
StringBuilder padding = new StringBuilder();
|
||||
String padding = "";
|
||||
int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
|
||||
int truncatedLength = value.length();
|
||||
for (int i = 0; i < paddedLength - truncatedLength; i++) {
|
||||
padding.append(" ");
|
||||
padding += " ";
|
||||
}
|
||||
return value + padding;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ public class CarbondataHetuOutputFormat<T>
|
|||
OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
|
||||
JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
|
||||
carbonOutputCommitter.setupJob(jobContext);
|
||||
CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
|
||||
org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
|
||||
return new FileSinkOperator.RecordWriter()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ public class CarbondataLocationService
|
|||
{
|
||||
// TODO: check and make it compatible for cloud scenario
|
||||
|
||||
HdfsEnvironment.HdfsContext context =
|
||||
new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
|
||||
Path targetPath = new Path(table.getStorage().getLocation());
|
||||
|
||||
return new LocationHandle(targetPath, targetPath, true,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -21,7 +21,6 @@ import io.airlift.log.Logger;
|
|||
import io.airlift.units.Duration;
|
||||
import io.hetu.core.plugin.carbondata.impl.CarbondataTableReader;
|
||||
import io.prestosql.plugin.hive.ForHive;
|
||||
import io.prestosql.plugin.hive.ForHiveMetastore;
|
||||
import io.prestosql.plugin.hive.ForHiveTransactionHeartbeats;
|
||||
import io.prestosql.plugin.hive.HdfsEnvironment;
|
||||
import io.prestosql.plugin.hive.HiveMetadata;
|
||||
|
|
@ -37,6 +36,7 @@ import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
|
|||
import io.prestosql.plugin.hive.security.AccessControlMetadataFactory;
|
||||
import io.prestosql.plugin.hive.statistics.MetastoreHiveStatisticsProvider;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
import org.joda.time.DateTimeZone;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
|
|
@ -49,6 +49,7 @@ public class CarbondataMetadataFactory
|
|||
extends HiveMetadataFactory
|
||||
{
|
||||
private static final Logger log = Logger.get(HiveMetadataFactory.class);
|
||||
private final boolean allowCorruptWritesForTesting;
|
||||
private final boolean skipDeletionForAlter;
|
||||
private final boolean skipTargetCleanupOnRollback;
|
||||
private final boolean writesToNonManagedTablesEnabled;
|
||||
|
|
@ -58,11 +59,11 @@ public class CarbondataMetadataFactory
|
|||
private final HiveMetastore metastore;
|
||||
private final HdfsEnvironment hdfsEnvironment;
|
||||
private final HivePartitionManager partitionManager;
|
||||
private final DateTimeZone timeZone;
|
||||
private final TypeManager typeManager;
|
||||
private final LocationService locationService;
|
||||
private final BoundedExecutor renameExecution;
|
||||
private final ScheduledExecutorService vacuumExecutorService;
|
||||
private final ScheduledExecutorService hiveMetastoreClientService;
|
||||
private final TypeTranslator typeTranslator;
|
||||
private final String hetuVersion;
|
||||
private final AccessControlMetadataFactory accessControlMetadataFactory;
|
||||
|
|
@ -82,7 +83,6 @@ public class CarbondataMetadataFactory
|
|||
@ForHive ExecutorService executorService,
|
||||
@ForCarbonVacuum ScheduledExecutorService vacuumExecutorService,
|
||||
@ForHiveTransactionHeartbeats ScheduledExecutorService heartbeatService,
|
||||
@ForHiveMetastore ScheduledExecutorService hiveMetastoreClientService,
|
||||
TypeManager typeManager, LocationService locationService,
|
||||
JsonCodec<PartitionUpdate> partitionUpdateCodec,
|
||||
JsonCodec<CarbondataSegmentInfoUtil> segmentInfoCodec,
|
||||
|
|
@ -90,8 +90,9 @@ public class CarbondataMetadataFactory
|
|||
AccessControlMetadataFactory accessControlMetadataFactory,
|
||||
CarbondataTableReader carbondataTableReader)
|
||||
{
|
||||
this(metastore, hdfsEnvironment, partitionManager,
|
||||
this(metastore, hdfsEnvironment, partitionManager, carbondataConfig.getDateTimeZone(),
|
||||
carbondataConfig.getMaxConcurrentFileRenames(),
|
||||
carbondataConfig.getAllowCorruptWritesForTesting(),
|
||||
carbondataConfig.isSkipDeletionForAlter(),
|
||||
carbondataConfig.isSkipTargetCleanupOnRollback(),
|
||||
true,
|
||||
|
|
@ -101,16 +102,15 @@ public class CarbondataMetadataFactory
|
|||
carbondataConfig.getHiveTransactionHeartbeatInterval(),
|
||||
carbondataConfig.getVacuumCleanupRecheckInterval(), typeManager, locationService,
|
||||
partitionUpdateCodec, segmentInfoCodec, executorService,
|
||||
vacuumExecutorService, heartbeatService, hiveMetastoreClientService, typeTranslator, nodeVersion.toString(),
|
||||
vacuumExecutorService, heartbeatService, typeTranslator, nodeVersion.toString(),
|
||||
accessControlMetadataFactory, carbondataTableReader, carbondataConfig.getStoreLocation(),
|
||||
carbondataConfig.getMajorVacuumSegSize(), carbondataConfig.getMinorVacuumSegCount(),
|
||||
carbondataConfig.getAutoVacuumEnable(), carbondataConfig.getMetastoreWriteBatchSize());
|
||||
carbondataConfig.getMajorVacuumSegSize(), carbondataConfig.getMinorVacuumSegCount());
|
||||
}
|
||||
|
||||
public CarbondataMetadataFactory(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment,
|
||||
HivePartitionManager partitionManager,
|
||||
HivePartitionManager partitionManager, DateTimeZone timeZone,
|
||||
int maxConcurrentFileRenames,
|
||||
boolean skipDeletionForAlter,
|
||||
boolean allowCorruptWritesForTesting, boolean skipDeletionForAlter,
|
||||
boolean skipTargetCleanupOnRollback, boolean writesToNonManagedTablesEnabled,
|
||||
boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
|
||||
long perTransactionCacheMaximumSize,
|
||||
|
|
@ -120,16 +120,16 @@ public class CarbondataMetadataFactory
|
|||
JsonCodec<PartitionUpdate> partitionUpdateCodec,
|
||||
JsonCodec<CarbondataSegmentInfoUtil> segmentInfoCodec,
|
||||
ExecutorService executorService, ScheduledExecutorService vacuumExecutorService, ScheduledExecutorService heartbeatService,
|
||||
ScheduledExecutorService hiveMetastoreClientService,
|
||||
TypeTranslator typeTranslator, String hetuVersion,
|
||||
AccessControlMetadataFactory accessControlMetadataFactory,
|
||||
CarbondataTableReader carbondataTableReader, String storeLocation, long majorVacuumSegSize, long minorVacuumSegCount,
|
||||
boolean autoVacuumEnable, int hmsWriteBatchSize)
|
||||
CarbondataTableReader carbondataTableReader, String storeLocation, long majorVacuumSegSize, long minorVacuumSegCount)
|
||||
{
|
||||
super(metastore,
|
||||
hdfsEnvironment,
|
||||
partitionManager,
|
||||
timeZone,
|
||||
maxConcurrentFileRenames,
|
||||
allowCorruptWritesForTesting,
|
||||
skipDeletionForAlter,
|
||||
skipTargetCleanupOnRollback,
|
||||
writesToNonManagedTablesEnabled,
|
||||
|
|
@ -144,13 +144,11 @@ public class CarbondataMetadataFactory
|
|||
executorService,
|
||||
vacuumExecutorService,
|
||||
heartbeatService,
|
||||
hiveMetastoreClientService,
|
||||
typeTranslator,
|
||||
hetuVersion,
|
||||
accessControlMetadataFactory,
|
||||
2, 0.0, false,
|
||||
Optional.of(new Duration(5, TimeUnit.MINUTES)),
|
||||
hmsWriteBatchSize);
|
||||
2, 0.0, false, Optional.of(new Duration(5, TimeUnit.MINUTES)));
|
||||
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
|
||||
this.skipDeletionForAlter = skipDeletionForAlter;
|
||||
this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
|
||||
this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled;
|
||||
|
|
@ -160,6 +158,7 @@ public class CarbondataMetadataFactory
|
|||
this.metastore = requireNonNull(metastore, "metastore is null");
|
||||
this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
|
||||
this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
|
||||
this.timeZone = requireNonNull(timeZone, "timeZone is null");
|
||||
this.typeManager = requireNonNull(typeManager, "typeManager is null");
|
||||
this.locationService = requireNonNull(locationService, "locationService is null");
|
||||
this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
|
||||
|
|
@ -168,9 +167,15 @@ public class CarbondataMetadataFactory
|
|||
this.hetuVersion = requireNonNull(hetuVersion, "hetuVersion is null");
|
||||
this.accessControlMetadataFactory = requireNonNull(accessControlMetadataFactory,
|
||||
"accessControlMetadataFactory is null");
|
||||
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
|
||||
log.warn(
|
||||
"Hive writes are disabled. To write data to Hive, your JVM timezone must match the " +
|
||||
"Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments",
|
||||
timeZone.getID());
|
||||
}
|
||||
|
||||
this.renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
|
||||
this.vacuumExecutorService = requireNonNull(vacuumExecutorService, "vacuumExecutorService is null");
|
||||
this.hiveMetastoreClientService = requireNonNull(hiveMetastoreClientService, "hiveMetastoreClientService is null");
|
||||
this.vacuumCleanupInterval = requireNonNull(vacuumCleanupInterval,
|
||||
"vacuumCleanupInterval is null");
|
||||
this.hiveTransactionHeartbeatInterval = requireNonNull(hiveTransactionHeartbeatInterval,
|
||||
|
|
@ -181,28 +186,25 @@ public class CarbondataMetadataFactory
|
|||
this.carbondataTableStore = storeLocation;
|
||||
this.carbondataMinorVacuumSegmentCount = minorVacuumSegCount;
|
||||
this.carbondataMajorVacuumSegmentSize = majorVacuumSegSize;
|
||||
|
||||
if (autoVacuumEnable) {
|
||||
CarbondataAutoVacuumThread.setAutoVacuumInfo(this.carbondataTableReader, this.hdfsEnvironment,
|
||||
majorVacuumSegSize, minorVacuumSegCount, vacuumExecutorService);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public HiveMetadata get()
|
||||
{
|
||||
SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
|
||||
SemiTransactionalHiveMetastore metastore =
|
||||
new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
|
||||
CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
|
||||
this.renameExecution,
|
||||
vacuumExecutorService, this.vacuumCleanupInterval, this.skipDeletionForAlter,
|
||||
this.skipTargetCleanupOnRollback,
|
||||
this.hiveTransactionHeartbeatInterval,
|
||||
this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
|
||||
this.heartbeatService);
|
||||
|
||||
return new CarbondataMetadata(semiTransactionalHiveMetastore,
|
||||
return new CarbondataMetadata(metastore,
|
||||
this.hdfsEnvironment,
|
||||
this.partitionManager,
|
||||
this.timeZone,
|
||||
this.allowCorruptWritesForTesting,
|
||||
this.writesToNonManagedTablesEnabled,
|
||||
this.createsOfNonManagedTablesEnabled,
|
||||
this.tableCreatesWithLocationAllowed,
|
||||
|
|
@ -212,13 +214,12 @@ public class CarbondataMetadataFactory
|
|||
this.segmentInfoCodec,
|
||||
this.typeTranslator,
|
||||
this.hetuVersion,
|
||||
new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
|
||||
this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
|
||||
new MetastoreHiveStatisticsProvider(metastore),
|
||||
this.accessControlMetadataFactory.create(metastore),
|
||||
carbondataTableReader,
|
||||
this.carbondataTableStore,
|
||||
this.carbondataMajorVacuumSegmentSize,
|
||||
this.carbondataMinorVacuumSegmentCount,
|
||||
vacuumExecutorService,
|
||||
hiveMetastoreClientService);
|
||||
vacuumExecutorService);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -167,9 +167,9 @@ public class CarbondataPageSink
|
|||
{
|
||||
//set flag here if called and change finish accordingly.
|
||||
isCompactionCalled = true;
|
||||
HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
|
||||
HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
|
||||
|
||||
finalHdfsEnvironment.doAs(session.getUser(), () -> {
|
||||
hdfsEnvironment.doAs(session.getUser(), () -> {
|
||||
try {
|
||||
// Worker part: each thread to run this code
|
||||
boolean mergeStatus = false;
|
||||
|
|
|
|||
|
|
@ -33,7 +33,6 @@ import io.prestosql.plugin.hive.HiveWriterStats;
|
|||
import io.prestosql.plugin.hive.LocationService;
|
||||
import io.prestosql.plugin.hive.OrcFileWriterFactory;
|
||||
import io.prestosql.plugin.hive.PartitionUpdate;
|
||||
import io.prestosql.plugin.hive.authentication.HiveIdentity;
|
||||
import io.prestosql.plugin.hive.metastore.HiveMetastore;
|
||||
import io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider;
|
||||
import io.prestosql.plugin.hive.metastore.SortingColumn;
|
||||
|
|
@ -163,7 +162,6 @@ public class CarbondataPageSinkProvider
|
|||
ImmutableMap.of(), handle.getAdditionalConf(), false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
|
||||
{
|
||||
CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;
|
||||
|
|
@ -184,7 +182,6 @@ public class CarbondataPageSinkProvider
|
|||
vacuumTableHandle.getTableStorageFormat(),
|
||||
vacuumTableHandle.getPartitionStorageFormat(),
|
||||
vacuumTableHandle.isFullVacuum(),
|
||||
vacuumTableHandle.isUnifyVacuum(),
|
||||
null);
|
||||
return createPageSink(hiveVacuumTableHandle, session, HiveACIDWriteType.VACUUM, ImmutableMap.of(), vacuumTableHandle.getAdditionalConf(), false);
|
||||
}
|
||||
|
|
@ -219,7 +216,7 @@ public class CarbondataPageSinkProvider
|
|||
locationService,
|
||||
session.getQueryId(),
|
||||
new HivePageSinkMetadataProvider(handle.getPageSinkMetadata(),
|
||||
memoizeMetastore(metastore, perTransactionMetastoreCacheMaximumSize), new HiveIdentity(session)),
|
||||
memoizeMetastore(metastore, perTransactionMetastoreCacheMaximumSize)),
|
||||
typeManager,
|
||||
hdfsEnvironment,
|
||||
pageSorter,
|
||||
|
|
|
|||
|
|
@ -232,15 +232,15 @@ public class CarbondataPageSource
|
|||
nanoStart = System.nanoTime();
|
||||
}
|
||||
CarbondataVectorBatch columnarBatch = null;
|
||||
int columnBatchSize = 0;
|
||||
int batchSize = 0;
|
||||
try {
|
||||
batchId++;
|
||||
if (vectorReader.nextKeyValue()) {
|
||||
Object vectorBatch = vectorReader.getCurrentValue();
|
||||
if (vectorBatch instanceof CarbondataVectorBatch) {
|
||||
columnarBatch = (CarbondataVectorBatch) vectorBatch;
|
||||
columnBatchSize = columnarBatch.numRows();
|
||||
if (columnBatchSize == 0) {
|
||||
batchSize = columnarBatch.numRows();
|
||||
if (batchSize == 0) {
|
||||
close();
|
||||
return null;
|
||||
}
|
||||
|
|
@ -256,9 +256,9 @@ public class CarbondataPageSource
|
|||
|
||||
Block[] blocks = new Block[columnHandles.size()];
|
||||
for (int column = 0; column < blocks.length; column++) {
|
||||
blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
|
||||
blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
|
||||
}
|
||||
Page page = new Page(columnBatchSize, blocks);
|
||||
Page page = new Page(batchSize, blocks);
|
||||
return page;
|
||||
}
|
||||
catch (PrestoException e) {
|
||||
|
|
|
|||
|
|
@ -32,7 +32,6 @@ import io.prestosql.spi.connector.ConnectorSplit;
|
|||
import io.prestosql.spi.connector.ConnectorTableHandle;
|
||||
import io.prestosql.spi.connector.ConnectorTransactionHandle;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.dynamicfilter.DynamicFilterSupplier;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
import org.apache.carbondata.core.constants.CarbonCommonConstants;
|
||||
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
|
||||
|
|
@ -42,7 +41,6 @@ import org.apache.hadoop.fs.Path;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
|
||||
|
|
@ -113,15 +111,6 @@ public class CarbondataPageSourceProvider
|
|||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
|
||||
ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table,
|
||||
List<ColumnHandle> columns, Optional<DynamicFilterSupplier> supplier)
|
||||
{
|
||||
// dynamicFilterSupplier is not supporting carbondata
|
||||
return createPageSource(transactionHandle, session, split, table, columns);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param carbonSplit
|
||||
* @return
|
||||
|
|
|
|||
|
|
@ -15,22 +15,9 @@ package io.hetu.core.plugin.carbondata;
|
|||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.prestosql.plugin.hive.HivePlugin;
|
||||
import io.prestosql.plugin.hive.metastore.thrift.StaticMetastoreConfig;
|
||||
import io.prestosql.spi.connector.ConnectorFactory;
|
||||
import io.prestosql.spi.function.ConnectorConfig;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorUtil;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
|
||||
import org.apache.carbondata.core.datastore.impl.FileFactory;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
|
||||
@ConnectorConfig(connectorLabel = "Carbondata: Query data stored in a Carbondata warehouse",
|
||||
propertiesEnabled = true,
|
||||
catalogConfigFilesEnabled = true,
|
||||
globalConfigFilesEnabled = true,
|
||||
docLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html",
|
||||
configLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html#configuration")
|
||||
public class CarbondataPlugin
|
||||
extends HivePlugin
|
||||
{
|
||||
|
|
@ -49,12 +36,4 @@ public class CarbondataPlugin
|
|||
{
|
||||
return ImmutableList.of(new CarbondataConnectorFactory("carbondata", getClassLoader()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ConnectorWithProperties> getConnectorWithProperties()
|
||||
{
|
||||
ConnectorConfig connectorConfig = CarbondataPlugin.class.getAnnotation(ConnectorConfig.class);
|
||||
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
|
||||
Arrays.asList(StaticMetastoreConfig.class.getDeclaredMethods()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
package io.hetu.core.plugin.carbondata;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.gson.Gson;
|
||||
import io.hetu.core.plugin.carbondata.impl.CarbondataLocalInputSplit;
|
||||
import io.hetu.core.plugin.carbondata.impl.CarbondataLocalMultiBlockSplit;
|
||||
|
|
@ -34,7 +33,6 @@ import io.prestosql.plugin.hive.HiveSplitWrapper;
|
|||
import io.prestosql.plugin.hive.HiveTableHandle;
|
||||
import io.prestosql.plugin.hive.HiveTransactionHandle;
|
||||
import io.prestosql.plugin.hive.NamenodeStats;
|
||||
import io.prestosql.plugin.hive.authentication.HiveIdentity;
|
||||
import io.prestosql.plugin.hive.metastore.MetastoreUtil;
|
||||
import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
|
||||
import io.prestosql.plugin.hive.metastore.Table;
|
||||
|
|
@ -140,20 +138,18 @@ public class CarbondataSplitManager
|
|||
@Override
|
||||
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle,
|
||||
ConnectorSession session, ConnectorTableHandle tableHandle,
|
||||
SplitSchedulingStrategy splitSchedulingStrategy, Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier,
|
||||
SplitSchedulingStrategy splitSchedulingStrategy, Supplier<Set<DynamicFilter>> dynamicFilterSupplier,
|
||||
Optional<QueryType> queryType, Map<String, Object> queryProperties,
|
||||
Set<TupleDomain<ColumnMetadata>> userDefinedCachePredicates,
|
||||
boolean partOfReuse)
|
||||
Set<TupleDomain<ColumnMetadata>> userDefinedCachePredicates)
|
||||
{
|
||||
HiveTableHandle hiveTable = (HiveTableHandle) tableHandle;
|
||||
SchemaTableName schemaTableName = hiveTable.getSchemaTableName();
|
||||
|
||||
// get table metadata
|
||||
HiveIdentity identity = new HiveIdentity(session);
|
||||
SemiTransactionalHiveMetastore metastore =
|
||||
metastoreProvider.apply((HiveTransactionHandle) transactionHandle);
|
||||
Table table =
|
||||
metastore.getTable(identity, schemaTableName.getSchemaName(), schemaTableName.getTableName())
|
||||
metastore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName())
|
||||
.orElseThrow(() -> new TableNotFoundException(schemaTableName));
|
||||
if (!table.getStorage().getStorageFormat().getInputFormat().contains("carbon")) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "Carbondata connector can only read carbondata tables");
|
||||
|
|
@ -201,7 +197,7 @@ public class CarbondataSplitManager
|
|||
0, 0, 0, 0,
|
||||
properties, new ArrayList(), getHostAddresses(split.getLocations()),
|
||||
OptionalInt.empty(), false, new HashMap<>(),
|
||||
Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
|
||||
Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
|
||||
/* Todo: Make this part aligned with rest of the HiveSlipt loading flow...
|
||||
* and figure out how to pass valid transaction Ids to CarbonData? */
|
||||
}
|
||||
|
|
@ -214,8 +210,7 @@ public class CarbondataSplitManager
|
|||
statisticRecorder.logStatisticsAsTableDriver();
|
||||
if (queryType != null && queryType.isPresent() && queryType.get().equals(QueryType.VACUUM)) {
|
||||
// Get Splits for compaction
|
||||
return getSplitsForCompaction(identity,
|
||||
transactionHandle,
|
||||
return getSplitsForCompaction(transactionHandle,
|
||||
tableHandle,
|
||||
cache.getCarbonTable().getTablePath(),
|
||||
queryProperties,
|
||||
|
|
@ -234,7 +229,7 @@ public class CarbondataSplitManager
|
|||
/*
|
||||
* Convert the splits into batches based on task id and wrap around ConnectorSplitSource to send back
|
||||
*/
|
||||
public ConnectorSplitSource getSplitsForCompaction(HiveIdentity identity, ConnectorTransactionHandle transactionHandle, ConnectorTableHandle tableHandle,
|
||||
public ConnectorSplitSource getSplitsForCompaction(ConnectorTransactionHandle transactionHandle, ConnectorTableHandle tableHandle,
|
||||
String tablePath, Map<String, Object> queryProperties, String queryId,
|
||||
ImmutableList.Builder<ConnectorSplit> allSplitsForComp, Configuration configuration) throws PrestoException
|
||||
{
|
||||
|
|
@ -245,7 +240,7 @@ public class CarbondataSplitManager
|
|||
// Step 1: Get table handles and metadata
|
||||
SemiTransactionalHiveMetastore metaStore =
|
||||
metastoreProvider.apply((HiveTransactionHandle) transactionHandle);
|
||||
Table table = metaStore.getTable(identity, schemaTableName.getSchemaName(), schemaTableName.getTableName())
|
||||
Table table = metaStore.getTable(schemaTableName.getSchemaName(), schemaTableName.getTableName())
|
||||
.orElseThrow(() -> new TableNotFoundException(schemaTableName));
|
||||
Properties hiveSchema = MetastoreUtil.getHiveSchema(table);
|
||||
CarbonLoadModel carbonLoadModel = null;
|
||||
|
|
@ -310,7 +305,7 @@ public class CarbondataSplitManager
|
|||
schemaTableName.getTableName(), tablePath, 0L, 0L, 0L, 0L,
|
||||
properties, new ArrayList(), getHostAddresses(currSplit.getLocations()),
|
||||
OptionalInt.empty(), false, new HashMap<>(),
|
||||
Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
|
||||
Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
|
||||
}
|
||||
}
|
||||
LOGGER.info("Splits for compaction built and ready");
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -35,17 +35,4 @@ public class CarbondataTableHandle
|
|||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
/* This method checks if reuse table scan can be used*/
|
||||
@Override
|
||||
public boolean isReuseTableScanSupported()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isUpdateAsInsertSupported()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -90,17 +90,16 @@ public class CarbondataTableProperties
|
|||
|
||||
private static SortingColumn sortingColumnFromString(String name)
|
||||
{
|
||||
String finalName = name;
|
||||
SortingColumn.Order order = SortingColumn.Order.ASCENDING;
|
||||
String lower = name.toUpperCase(ENGLISH);
|
||||
if (lower.endsWith(" ASC")) {
|
||||
finalName = name.substring(0, name.length() - 4).trim();
|
||||
name = name.substring(0, name.length() - 4).trim();
|
||||
}
|
||||
else if (lower.endsWith(" DESC")) {
|
||||
finalName = name.substring(0, name.length() - 5).trim();
|
||||
name = name.substring(0, name.length() - 5).trim();
|
||||
order = SortingColumn.Order.DESCENDING;
|
||||
}
|
||||
return new SortingColumn(finalName, order);
|
||||
return new SortingColumn(name, order);
|
||||
}
|
||||
|
||||
private static String sortingColumnToString(SortingColumn column)
|
||||
|
|
|
|||
|
|
@ -61,7 +61,6 @@ public class CarbondataVacuumTableHandle
|
|||
tableStorageFormat,
|
||||
partitionStorageFormat,
|
||||
full,
|
||||
false,
|
||||
null);
|
||||
// Additional conf is used to store the encoded load model
|
||||
this.additionalConf = ImmutableMap.copyOf(requireNonNull(additionalConf, "additional conf map is null"));
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import io.prestosql.plugin.hive.OrcFileWriterFactory;
|
|||
import io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider;
|
||||
import io.prestosql.plugin.hive.metastore.SortingColumn;
|
||||
import io.prestosql.spi.NodeManager;
|
||||
import io.prestosql.spi.Page;
|
||||
import io.prestosql.spi.PageSorter;
|
||||
import io.prestosql.spi.connector.ConnectorSession;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
|
|
@ -47,7 +48,6 @@ import java.util.Set;
|
|||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION;
|
||||
import static org.joda.time.DateTimeZone.UTC;
|
||||
|
||||
public class CarbondataWriterFactory
|
||||
extends HiveWriterFactory
|
||||
|
|
@ -88,7 +88,7 @@ public class CarbondataWriterFactory
|
|||
additionalTableParameters, bucketCount, sortedBy, locationHandle,
|
||||
locationService, queryId, pageSinkMetadataProvider,
|
||||
typeManager, hdfsEnvironment, pageSorter, sortBufferSize,
|
||||
maxOpenSortFiles, immutablePartitions, UTC, session, nodeManager,
|
||||
maxOpenSortFiles, immutablePartitions, session, nodeManager,
|
||||
eventClient, hiveSessionProperties, hiveWriterStats, orcFileWriterFactory);
|
||||
|
||||
this.additionalJobConf = requireNonNull(additionalJobConf, "Additional JobConf is null");
|
||||
|
|
@ -120,20 +120,19 @@ public class CarbondataWriterFactory
|
|||
}
|
||||
|
||||
@Override
|
||||
public HiveWriter createWriter(List<String> partitionValues, OptionalInt bucketNumber, Optional<Options> vacuumOptions)
|
||||
public HiveWriter createWriter(Page partitionColumns, int position, OptionalInt bucketNumber, Optional<Options> vacuumOptions)
|
||||
{
|
||||
/* set Additional JobConf */
|
||||
JobConf jobConf = getSuperJobConf();
|
||||
|
||||
additionalJobConf.forEach((k, v) -> jobConf.set(k, v));
|
||||
return super.createWriter(partitionValues, bucketNumber, vacuumOptions);
|
||||
return super.createWriter(partitionColumns, position, bucketNumber, vacuumOptions);
|
||||
}
|
||||
|
||||
protected void checkWriteMode(LocationService.WriteInfo writeInfo)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void setAdditionalSchemaProperties(Properties schema)
|
||||
{
|
||||
schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());
|
||||
|
|
|
|||
|
|
@ -143,15 +143,14 @@ class ColumnarVectorWrapperDirect
|
|||
@Override
|
||||
public void putDecimals(int rowId, int count, BigDecimal value, int precision)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putDecimal(inputRowId, value, precision);
|
||||
columnVector.putDecimal(rowId, value, precision);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -186,9 +185,8 @@ class ColumnarVectorWrapperDirect
|
|||
@Override
|
||||
public void putByteArray(int rowId, int count, byte[] value)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
columnVector.putByteArray(inputRowId++, value);
|
||||
columnVector.putByteArray(rowId++, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -305,90 +303,84 @@ class ColumnarVectorWrapperDirect
|
|||
@Override
|
||||
public void putFloats(int rowId, int count, float[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putFloat(inputRowId, src[i]);
|
||||
columnVector.putFloat(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putShorts(int rowId, int count, short[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putShort(inputRowId, src[i]);
|
||||
columnVector.putShort(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putInts(int rowId, int count, int[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putInt(inputRowId, src[i]);
|
||||
columnVector.putInt(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putLongs(int rowId, int count, long[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putLong(inputRowId, src[i]);
|
||||
columnVector.putLong(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putDoubles(int rowId, int count, double[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putDouble(inputRowId, src[i]);
|
||||
columnVector.putDouble(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void putBytes(int rowId, int count, byte[] src, int srcIndex)
|
||||
{
|
||||
int inputRowId = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (nullBitSet.get(inputRowId)) {
|
||||
columnVector.putNull(inputRowId);
|
||||
if (nullBitSet.get(rowId)) {
|
||||
columnVector.putNull(rowId);
|
||||
}
|
||||
else {
|
||||
columnVector.putByte(inputRowId, src[i]);
|
||||
columnVector.putByte(rowId, src[i]);
|
||||
}
|
||||
inputRowId++;
|
||||
rowId++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -56,10 +56,7 @@ import org.apache.hadoop.mapreduce.Job;
|
|||
import org.apache.log4j.Logger;
|
||||
import org.apache.thrift.TBase;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
|
|
@ -142,19 +139,10 @@ public class CarbondataTableReader
|
|||
carbonTableCacheModel.getCarbonTable().isTransactionalTable()) {
|
||||
CarbonTable carbonTable = carbonTableCacheModel.getCarbonTable();
|
||||
try {
|
||||
String schemaFilePath = CarbonTablePath
|
||||
.getSchemaFilePath(
|
||||
carbonTable.getTablePath());
|
||||
FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
|
||||
|
||||
CarbonFile carbonFile = FileFactory.getCarbonFile(schemaFilePath,
|
||||
config);
|
||||
|
||||
long latestTime = carbonFile.getLastModifiedTime();
|
||||
// Workaround for cache updation error due to round off of latestTime inside LocalCarbonFile
|
||||
if (fileType == FileFactory.FileType.LOCAL) {
|
||||
latestTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
|
||||
}
|
||||
long latestTime = FileFactory.getCarbonFile(CarbonTablePath
|
||||
.getSchemaFilePath(
|
||||
carbonTable.getTablePath()),
|
||||
config).getLastModifiedTime();
|
||||
carbonTableCacheModel.setCurrentSchemaTime(latestTime);
|
||||
if (!carbonTableCacheModel.isValid()) {
|
||||
// Invalidate datamaps
|
||||
|
|
@ -162,7 +150,7 @@ public class CarbondataTableReader
|
|||
.clearIndex(carbonTableCacheModel.getCarbonTable().getAbsoluteTableIdentifier());
|
||||
}
|
||||
}
|
||||
catch (CarbonFileException | IOException e) {
|
||||
catch (CarbonFileException e) {
|
||||
carbonCache.get().remove(schemaTableName);
|
||||
}
|
||||
}
|
||||
|
|
@ -192,7 +180,7 @@ public class CarbondataTableReader
|
|||
return cache;
|
||||
}
|
||||
// multiple tasks can be launched in a worker concurrently. Hence need to synchronize this.
|
||||
synchronized (CarbondataTableReader.class) {
|
||||
synchronized (this) {
|
||||
// cache might be filled by another thread, so if filled use that cache.
|
||||
CarbondataTableCacheModel cacheModel = getValidCacheBySchemaTableName(table);
|
||||
if (cacheModel != null) {
|
||||
|
|
@ -223,11 +211,6 @@ public class CarbondataTableReader
|
|||
tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
|
||||
thriftReader.close();
|
||||
modifiedTime = schemaFile.getLastModifiedTime();
|
||||
FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
|
||||
// Workaround for cache updation error due to round off of latesttime inside LocalCarbonFile
|
||||
if (fileType == FileFactory.FileType.LOCAL) {
|
||||
modifiedTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
|
||||
}
|
||||
}
|
||||
else {
|
||||
tableInfo = CarbonUtil.inferSchema(tablePath, table.getTableName(), false, config);
|
||||
|
|
@ -340,7 +323,7 @@ public class CarbondataTableReader
|
|||
inputSplits.get(j).stream().flatMap(f -> Arrays.stream(getLocations(f))).distinct()
|
||||
.toArray(String[]::new)));
|
||||
}
|
||||
LOGGER.error("Size of MultiblockList " + multiBlockSplitList.size());
|
||||
LOGGER.error("Size fo MultiblockList " + multiBlockSplitList.size());
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
|
|
|
|||
|
|
@ -58,9 +58,8 @@ public class BooleanStreamReader
|
|||
@Override
|
||||
public void putBytes(int rowId, int count, byte[] src, int srcIndex)
|
||||
{
|
||||
int srcIdx = srcIndex;
|
||||
for (int i = 0; i < count; i++) {
|
||||
type.writeBoolean(builder, src[srcIdx++] == 1);
|
||||
type.writeBoolean(builder, src[srcIndex++] == 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -76,9 +76,8 @@ public class DecimalSliceStreamReader
|
|||
@Override
|
||||
public void putDecimals(int rowId, int count, BigDecimal value, int precision)
|
||||
{
|
||||
int id = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
putDecimal(id++, value, precision);
|
||||
putDecimal(rowId++, value, precision);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,9 +58,8 @@ public class IntegerStreamReader
|
|||
@Override
|
||||
public void putInts(int rowId, int count, int value)
|
||||
{
|
||||
int id = rowId;
|
||||
for (int i = 0; i < count; i++) {
|
||||
putInt(id++, value);
|
||||
putInt(rowId++, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,237 +0,0 @@
|
|||
/*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.plugin.carbondata.integrationtest;
|
||||
|
||||
import io.hetu.core.plugin.carbondata.CarbondataAutoVacuumThread;
|
||||
import io.hetu.core.plugin.carbondata.server.HetuTestServer;
|
||||
import io.prestosql.metadata.Catalog;
|
||||
import io.prestosql.metadata.CatalogManager;
|
||||
import io.prestosql.spi.connector.Connector;
|
||||
import io.prestosql.spi.connector.ConnectorMetadata;
|
||||
import io.prestosql.spi.connector.ConnectorVacuumTableInfo;
|
||||
import org.apache.carbondata.common.logging.LogServiceFactory;
|
||||
import org.apache.carbondata.core.constants.CarbonCommonConstants;
|
||||
import org.apache.carbondata.core.datastore.impl.FileFactory;
|
||||
import org.apache.carbondata.core.util.CarbonProperties;
|
||||
import org.apache.carbondata.core.util.CarbonUtil;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.testng.Assert.assertTrue;
|
||||
import static org.testng.AssertJUnit.assertNotNull;
|
||||
|
||||
public class TestCarbonAutoVacuum
|
||||
{
|
||||
private final Logger logger = LogServiceFactory.getLogService(TestCarbonAllDataType.class.getCanonicalName());
|
||||
|
||||
private final String rootPath = new File(this.getClass().getResource("/").getPath() + "../..").getCanonicalPath();
|
||||
|
||||
private final String storePath = rootPath + "/target/store_autovacuum";
|
||||
private final HetuTestServer hetuServer = new HetuTestServer();
|
||||
|
||||
public TestCarbonAutoVacuum() throws Exception {
|
||||
}
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public void setup() throws Exception
|
||||
{
|
||||
logger.info("Setup begin: " + this.getClass().getSimpleName());
|
||||
|
||||
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
|
||||
|
||||
Map<String, String> map = new HashMap<>();
|
||||
map.put("hive.metastore", "file");
|
||||
map.put("hive.allow-drop-table", "true");
|
||||
map.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store");
|
||||
map.put("carbondata.store-location", "file://" + storePath + "/carbon.store");
|
||||
map.put("carbondata.minor-vacuum-seg-count", "4");
|
||||
map.put("carbondata.major-vacuum-seg-size", "1");
|
||||
map.put("carbondata.auto-vacuum-enabled", "true");
|
||||
|
||||
if (!FileFactory.isFileExist( storePath + "/carbon.store")) {
|
||||
FileFactory.mkdirs( storePath + "/carbon.store");
|
||||
}
|
||||
Map<String, String> configProperties = new HashMap<>();
|
||||
configProperties.put("auto-vacuum.enabled", "true");
|
||||
configProperties.put("auto-vacuum.scan.interval", "15m");
|
||||
configProperties.put("auto-vacuum.scan.threads", "3");
|
||||
|
||||
hetuServer.startServer("carbontestdb", map, configProperties);
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
|
||||
hetuServer.execute("drop table if exists carbontestdb2.autovacuumtable2");
|
||||
hetuServer.execute("drop table if exists carbontestdb3.autovacuumtable3");
|
||||
hetuServer.execute("drop table if exists carbontestdb4.autovacuumtable4");
|
||||
|
||||
hetuServer.execute("drop schema if exists carbontestdb1");
|
||||
hetuServer.execute("drop schema if exists carbontestdb2");
|
||||
hetuServer.execute("drop schema if exists carbontestdb3");
|
||||
hetuServer.execute("drop schema if exists carbontestdb4");
|
||||
|
||||
hetuServer.execute("create schema carbontestdb1");
|
||||
hetuServer.execute("create schema carbontestdb2");
|
||||
hetuServer.execute("create schema carbontestdb3");
|
||||
hetuServer.execute("create schema carbontestdb4");
|
||||
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public void tearDown() throws SQLException, IOException, InterruptedException
|
||||
{
|
||||
logger.info("TearDown begin: " + this.getClass().getSimpleName());
|
||||
hetuServer.stopServer();
|
||||
CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoVacuum() throws SQLException {
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable1(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable1 VALUES (10, 11)");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable2(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (20, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable2 VALUES (30, 11)");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable3 as select a, b from carbontestdb1.autovacuumtable2 ");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable4(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (20, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (30, 12)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable4 VALUES (40, 13)");
|
||||
CatalogManager catalogManager = hetuServer.getCatalog();
|
||||
List<Catalog> catalogs = catalogManager.getCatalogs();
|
||||
Catalog catalog = null;
|
||||
for (Catalog catalog1 : catalogs) {
|
||||
if (catalog1.getConnectorCatalogName().getCatalogName().equals("carbondata")) {
|
||||
catalog = catalog1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Connector connector;
|
||||
ConnectorMetadata connectorMetadata = null;
|
||||
List<ConnectorVacuumTableInfo> tables;
|
||||
List<String> tablesNames = new ArrayList<>();
|
||||
|
||||
try {
|
||||
CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
|
||||
assertNotNull(catalog);
|
||||
connector = catalog.getConnector(catalog.getConnectorCatalogName());
|
||||
connectorMetadata = connector.getConnectorMetadata();
|
||||
connectorMetadata.getTablesForVacuum();
|
||||
} catch (Exception e) {
|
||||
logger.debug(e.getMessage());
|
||||
}
|
||||
|
||||
CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
|
||||
CarbondataAutoVacuumThread.enableTracingVacuumTask(false);
|
||||
tables = connectorMetadata.getTablesForVacuum();
|
||||
if (tables != null && 0 != tables.size()) {
|
||||
for (ConnectorVacuumTableInfo vacuumTable : tables) {
|
||||
tablesNames.add(vacuumTable.getSchemaTableName());
|
||||
}
|
||||
}
|
||||
|
||||
assertTrue( tablesNames.contains("carbontestdb1.autovacuumtable4"));
|
||||
assertTrue( tablesNames.contains("carbontestdb1.autovacuumtable2"));
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable2");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable3");
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable4");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoVacuumDiffDB() throws SQLException, InterruptedException {
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
|
||||
hetuServer.execute("drop table if exists carbontestdb2.autovacuumtable2");
|
||||
hetuServer.execute("drop table if exists carbontestdb3.autovacuumtable3");
|
||||
hetuServer.execute("drop table if exists carbontestdb4.autovacuumtable4");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb1.autovacuumtable1(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb1.autovacuumtable1 VALUES (10, 11)");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb2.autovacuumtable2(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (20, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (30, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb2.autovacuumtable2 VALUES (30, 11)");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb3.autovacuumtable3 as select a, b from carbontestdb2.autovacuumtable2 ");
|
||||
|
||||
hetuServer.execute("CREATE TABLE carbontestdb4.autovacuumtable4(a int, b int)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (20, 11)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (30, 12)");
|
||||
hetuServer.execute("INSERT INTO carbontestdb4.autovacuumtable4 VALUES (40, 13)");
|
||||
|
||||
CatalogManager catalogManager = hetuServer.getCatalog();
|
||||
List<Catalog> catalogs = catalogManager.getCatalogs();
|
||||
Catalog catalog = null;
|
||||
for (Catalog catalog1 : catalogs) {
|
||||
if (catalog1.getConnectorCatalogName().getCatalogName().equals("carbondata")) {
|
||||
catalog = catalog1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Connector connector;
|
||||
ConnectorMetadata connectorMetadata = null;
|
||||
List<ConnectorVacuumTableInfo> tables = null;
|
||||
List<String> tablesNames = new ArrayList<>();
|
||||
|
||||
try {
|
||||
|
||||
CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
|
||||
connector = catalog.getConnector(catalog.getConnectorCatalogName());
|
||||
connectorMetadata = connector.getConnectorMetadata();
|
||||
connectorMetadata.getTablesForVacuum();
|
||||
} catch (Exception e) {
|
||||
logger.debug(e.getMessage());
|
||||
}
|
||||
|
||||
CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
|
||||
CarbondataAutoVacuumThread.enableTracingVacuumTask(false);
|
||||
tables = connectorMetadata.getTablesForVacuum();
|
||||
if (tables != null && 0 != tables.size()) {
|
||||
for (ConnectorVacuumTableInfo vacuumTable : tables) {
|
||||
tablesNames.add(vacuumTable.getSchemaTableName());
|
||||
}
|
||||
}
|
||||
assertTrue( tablesNames.contains("carbontestdb4.autovacuumtable4"));
|
||||
assertTrue( tablesNames.contains("carbontestdb2.autovacuumtable2"));
|
||||
hetuServer.execute("drop table if exists carbontestdb1.autovacuumtable1");
|
||||
hetuServer.execute("drop table if exists carbontestdb2.autovacuumtable2");
|
||||
hetuServer.execute("drop table if exists carbontestdb3.autovacuumtable3");
|
||||
hetuServer.execute("drop table if exists carbontestdb4.autovacuumtable4");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,411 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.plugin.carbondata.integrationtest;
|
||||
|
||||
import io.hetu.core.plugin.carbondata.CarbondataMetadata;
|
||||
import io.hetu.core.plugin.carbondata.server.HetuTestServer;
|
||||
import org.apache.carbondata.common.logging.LogServiceFactory;
|
||||
import org.apache.carbondata.core.constants.CarbonCommonConstants;
|
||||
import org.apache.carbondata.core.datastore.impl.FileFactory;
|
||||
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
|
||||
import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
|
||||
import org.apache.carbondata.core.util.CarbonProperties;
|
||||
import org.apache.carbondata.core.util.CarbonUtil;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.sql.SQLException;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
public class TestCarbondataAutoCleanup
|
||||
{
|
||||
private final Logger logger = LogServiceFactory.getLogService(TestCarbondataAutoCleanup.class.getCanonicalName());
|
||||
|
||||
private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
|
||||
.getCanonicalPath();
|
||||
|
||||
private String storePath = rootPath + "/target/store_autocleanup";
|
||||
private String systemPath = rootPath + "/target/system";
|
||||
private HetuTestServer hetuServer = new HetuTestServer();
|
||||
private String carbonStoreLocation = storePath + "/carbon.store";
|
||||
|
||||
public TestCarbondataAutoCleanup() throws Exception
|
||||
{
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public void setup() throws Exception
|
||||
{
|
||||
logger.info("Setup begin: " + this.getClass().getSimpleName());
|
||||
|
||||
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
|
||||
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
|
||||
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS, "0");
|
||||
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT, "1");
|
||||
|
||||
Map<String, String> map = new HashMap<String, String>();
|
||||
map.put("hive.metastore", "file");
|
||||
map.put("hive.allow-drop-table", "true");
|
||||
map.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store");
|
||||
map.put("carbondata.store-location", "file://" + carbonStoreLocation);
|
||||
map.put("carbondata.minor-vacuum-seg-count", "4");
|
||||
map.put("carbondata.major-vacuum-seg-size", "1");
|
||||
|
||||
if (!FileFactory.isFileExist(storePath + "/carbon.store")) {
|
||||
FileFactory.mkdirs(storePath + "/carbon.store");
|
||||
}
|
||||
|
||||
hetuServer.startServer("testdb", map);
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup1");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup2");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup3");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup4");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup5");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup6");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup7");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup8");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
|
||||
hetuServer.execute("drop schema if exists testdb");
|
||||
hetuServer.execute("drop schema if exists default");
|
||||
hetuServer.execute("create schema testdb");
|
||||
hetuServer.execute("create schema default");
|
||||
|
||||
logger.info("CarbonStore created at location : " + storePath);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public void tearDown() throws SQLException, IOException, InterruptedException
|
||||
{
|
||||
logger.info("TearDown begin: " + this.getClass().getSimpleName());
|
||||
hetuServer.stopServer();
|
||||
CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAutoCleanupInUpdate() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup1");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup1 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup1 AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup1/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
hetuServer.execute("UPDATE testdb.testtableautocleanup1 SET a=232 WHERE b=511");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
|
||||
}
|
||||
catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoCleanupInVacuum() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup2");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup2 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (130, 411)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup2/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoCleanupInDelete() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup3");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup3");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoCleanupInDeleteWithPushdown() throws SQLException
|
||||
{
|
||||
try {
|
||||
hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = true");
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanupwithpushdown (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanupwithpushdown AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("DELETE FROM testdb.testtableautocleanupwithpushdown WHERE a=130");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
}
|
||||
finally {
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanupwithpushdown");
|
||||
hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = false");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAutoCleanupInInsert() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup4");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup4 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup4 AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup4/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup4");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullAutoCleanupInUpdate() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup5");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup5 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup5 AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup5/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("UPDATE testdb.testtableautocleanup5 SET a=232 WHERE b=511");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
|
||||
}
|
||||
catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup5");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullAutoCleanupInVacuum() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup6");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup6 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (130, 411)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup6/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup6");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullAutoCleanupInDelete() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup7");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup7 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup7 FULL AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup7/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("DELETE FROM testdb.testtableautocleanup7 WHERE a=130");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup7");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullAutoCleanupInInsert() throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists testdb.testtableautocleanup8");
|
||||
hetuServer.execute("CREATE TABLE testdb.testtableautocleanup8 (a int, b int)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (10, 11)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (110, 211)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (120, 311)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 411)");
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)");
|
||||
hetuServer.execute("vacuum table testdb.testtableautocleanup8 FULL AND WAIT");
|
||||
|
||||
reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup8/Metadata");
|
||||
CarbondataMetadata.enableTracingCleanupTask(true);
|
||||
|
||||
hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)");
|
||||
try {
|
||||
CarbondataMetadata.waitForSubmittedTasksFinish();
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_0", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_1", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
|
||||
assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
|
||||
} catch (IOException exception) {
|
||||
logger.debug(exception.getMessage());
|
||||
}
|
||||
|
||||
CarbondataMetadata.enableTracingCleanupTask(false);
|
||||
hetuServer.execute("drop table testdb.testtableautocleanup8");
|
||||
}
|
||||
|
||||
private void reduceModificationOrdeletionTimesStamp(String tableMetadatPath)
|
||||
{
|
||||
LoadMetadataDetails[] metadataDetails =
|
||||
SegmentStatusManager.readLoadMetadata(tableMetadatPath);
|
||||
for (LoadMetadataDetails oneLoad : metadataDetails) {
|
||||
long deletionTime = oneLoad.getModificationOrdeletionTimesStamp();
|
||||
|
||||
String modificationOrdeletionTimesStamp = Long.toString(deletionTime);
|
||||
deletionTime -= 70000L;
|
||||
String replace = Long.toString(deletionTime);
|
||||
System.out.println("replace" + replace);
|
||||
|
||||
Path path = Paths.get(tableMetadatPath + "/tablestatus");
|
||||
Charset charset = StandardCharsets.UTF_8;
|
||||
|
||||
String content = null;
|
||||
try {
|
||||
content = new String(Files.readAllBytes(path), charset);
|
||||
|
||||
content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
|
||||
Files.write(path, content.getBytes(charset));
|
||||
} catch (IOException e) {
|
||||
logger.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
|
|||
"/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
|
||||
} catch (IOException e) {
|
||||
hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
|
||||
logger.error(e.getMessage());
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -15,7 +15,6 @@
|
|||
|
||||
package io.hetu.core.plugin.carbondata.integrationtest;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.plugin.carbondata.server.HetuTestServer;
|
||||
import org.apache.carbondata.core.constants.CarbonCommonConstants;
|
||||
import org.apache.carbondata.core.datastore.impl.FileFactory;
|
||||
|
|
@ -31,13 +30,11 @@ import java.sql.SQLException;
|
|||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertThrows;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
public class TestsWithHiveConnector
|
||||
{
|
||||
private static final Logger log = Logger.get(TestsWithHiveConnector.class);
|
||||
private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
|
||||
.getCanonicalPath();
|
||||
|
||||
|
|
@ -66,9 +63,7 @@ public class TestsWithHiveConnector
|
|||
}
|
||||
|
||||
hetuServer.startServer("default", map);
|
||||
hetuServer.execute("drop schema if exists default");
|
||||
hetuServer.execute("create schema default");
|
||||
hetuServer.addHiveCatalogToQueryRunner(createHiveProperties());
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
|
|
@ -82,6 +77,7 @@ public class TestsWithHiveConnector
|
|||
public void block_Hive_Table_from_Carbondata()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.addHiveCatalogToQueryRunner(createHiveProperties());
|
||||
hetuServer.execute("CREATE TABLE hive.default.demotable (c1 int)");
|
||||
|
||||
hetuServer.execute("use carbondata.default");
|
||||
|
|
@ -101,134 +97,24 @@ public class TestsWithHiveConnector
|
|||
hetuServer.execute("DROP TABLE hive.default.demotable");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteTransactionTableDirc()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists hive.default.parttable");
|
||||
hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = true");
|
||||
hetuServer.execute("create table hive.default.parttable (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
|
||||
hetuServer.execute("insert into hive.default.parttable values (1,2011)");
|
||||
hetuServer.execute("insert into hive.default.parttable values (2,2012)");
|
||||
hetuServer.execute("insert into hive.default.parttable values (3,2013)");
|
||||
hetuServer.execute("delete from hive.default.parttable where year =2013");
|
||||
try {
|
||||
assertEquals(FileFactory.isFileExist(storePath +
|
||||
"hive.store/default/parttable/year=2013", false), false);
|
||||
} catch (IOException exception) {
|
||||
log.error(exception.getMessage());
|
||||
}
|
||||
hetuServer.execute("DROP TABLE hive.default.parttable");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteTransactionTableDirUsing2tables()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists hive.default.parttable2");
|
||||
hetuServer.execute("drop table if exists hive.default.parttable1");
|
||||
hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = true");
|
||||
hetuServer.execute("create table hive.default.parttable2 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
|
||||
hetuServer.execute("insert into hive.default.parttable2 values (1,2011)");
|
||||
hetuServer.execute("insert into hive.default.parttable2 values (2,2012)");
|
||||
hetuServer.execute("insert into hive.default.parttable2 values (3,2013)");
|
||||
|
||||
hetuServer.execute("create table hive.default.parttable1 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
|
||||
hetuServer.execute("insert into hive.default.parttable1 values (1,2011)");
|
||||
hetuServer.execute("insert into hive.default.parttable1 values (2,2012)");
|
||||
hetuServer.execute("insert into hive.default.parttable1 values (3,2013)");
|
||||
|
||||
hetuServer.execute("delete from hive.default.parttable2 where year >= (select max(year) from hive.default.parttable1) ");
|
||||
try {
|
||||
assertEquals(FileFactory.isFileExist(storePath +
|
||||
"hive.store/default/parttable2/year=2013", false), false);
|
||||
} catch (IOException exception) {
|
||||
log.error(exception.getMessage());
|
||||
}
|
||||
|
||||
hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
|
||||
hetuServer.execute("delete from hive.default.parttable2 where year >= (select year from hive.default.parttable1 where orderkey=4 ) ");
|
||||
try {
|
||||
assertEquals(FileFactory.isFileExist(storePath +
|
||||
"hive.store/default/parttable2/year=2014", false), false);
|
||||
} catch (IOException exception) {
|
||||
log.error(exception.getMessage());
|
||||
}
|
||||
|
||||
hetuServer.execute("DROP TABLE hive.default.parttable2");
|
||||
hetuServer.execute("DROP TABLE hive.default.parttable1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deleteTransactionTableDirDisable()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.execute("drop table if exists hive.default.parttable3");
|
||||
hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = false");
|
||||
hetuServer.execute("create table hive.default.parttable3 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
|
||||
hetuServer.execute("insert into hive.default.parttable3 values (1,2011)");
|
||||
hetuServer.execute("insert into hive.default.parttable3 values (2,2012)");
|
||||
hetuServer.execute("insert into hive.default.parttable3 values (3,2013)");
|
||||
|
||||
hetuServer.execute("create table hive.default.parttable4 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
|
||||
hetuServer.execute("insert into hive.default.parttable4 values (1,2011)");
|
||||
hetuServer.execute("insert into hive.default.parttable4 values (2,2012)");
|
||||
hetuServer.execute("insert into hive.default.parttable4 values (3,2013)");
|
||||
|
||||
hetuServer.execute("delete from hive.default.parttable3 where year >= (select max(year) from hive.default.parttable4) ");
|
||||
try {
|
||||
assertEquals(FileFactory.isFileExist(storePath +
|
||||
"/hive.store/default/parttable3/year=2013", false), true);
|
||||
} catch (IOException exception) {
|
||||
log.error(exception.getMessage());
|
||||
}
|
||||
hetuServer.execute("DROP TABLE hive.default.parttable3");
|
||||
}
|
||||
|
||||
@Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
|
||||
public void block_Carbondata_Table_from_Hive()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.execute("CREATE TABLE carbondata.default.demotable1 (c1 int)");
|
||||
hetuServer.execute("CREATE TABLE carbondata.default.demotable (c1 int)");
|
||||
|
||||
hetuServer.execute("use hive.default");
|
||||
|
||||
runQueryAndAssertErrorMessage("SELECT * FROM demotable1",
|
||||
runQueryAndAssertErrorMessage("SELECT * FROM demotable",
|
||||
"Hive connector can't read carbondata tables");
|
||||
|
||||
runQueryAndAssertErrorMessage("INSERT INTO demotable1 VALUES(1)",
|
||||
runQueryAndAssertErrorMessage("INSERT INTO demotable VALUES(1)",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
|
||||
runQueryAndAssertErrorMessage("UPDATE demotable1 SET c1=1",
|
||||
runQueryAndAssertErrorMessage("UPDATE demotable SET c1=1",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
|
||||
hetuServer.execute("use carbondata.default");
|
||||
hetuServer.execute("DROP TABLE carbondata.default.demotable1");
|
||||
}
|
||||
|
||||
@Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
|
||||
public void block_Carbondata_Table_change_from_Hive()
|
||||
throws SQLException
|
||||
{
|
||||
hetuServer.execute("CREATE TABLE carbondata.default.changetable (a int, b int, c int)");
|
||||
hetuServer.execute("use hive.default");
|
||||
//rename column
|
||||
runQueryAndAssertErrorMessage("ALTER TABLE default.changetable RENAME COLUMN a TO aa",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
//add column
|
||||
runQueryAndAssertErrorMessage("ALTER TABLE default.changetable ADD COLUMN e int COMMENT 'Hello World'",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
//drop column
|
||||
runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable DROP COLUMN c",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
//change table name
|
||||
runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable RENAME TO hive.default.changetable_N",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
//drop table
|
||||
runQueryAndAssertErrorMessage("DROP TABLE default.changetable",
|
||||
"Tables with MapredCarbonInputFormat are not supported by Hive connector");
|
||||
hetuServer.execute("use carbondata.default");
|
||||
hetuServer.execute("DROP TABLE carbondata.default.changetable");
|
||||
hetuServer.execute("DROP TABLE hive.default.demotable");
|
||||
}
|
||||
|
||||
private Map<String, String> createHiveProperties()
|
||||
|
|
@ -237,11 +123,6 @@ public class TestsWithHiveConnector
|
|||
hiveProperties.put("hive.metastore", "file");
|
||||
hiveProperties.put("hive.allow-drop-table", "true");
|
||||
hiveProperties.put("hive.non-managed-table-writes-enabled", "true");
|
||||
hiveProperties.put("hive.allow-add-column", "true");
|
||||
hiveProperties.put("hive.allow-drop-column", "true");
|
||||
hiveProperties.put("hive.allow-rename-table", "true");
|
||||
hiveProperties.put("hive.allow-comment-table", "true");
|
||||
hiveProperties.put("hive.allow-rename-column", "true");
|
||||
hiveProperties.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store");
|
||||
return hiveProperties;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -17,14 +17,13 @@ package io.hetu.core.plugin.carbondata.server;
|
|||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.hetu.core.plugin.carbondata.CarbondataPlugin;
|
||||
import io.hetu.core.plugin.carbondata.impl.CarbondataTableReader;
|
||||
import io.prestosql.Session;
|
||||
import io.prestosql.execution.QueryIdGenerator;
|
||||
import io.prestosql.jdbc.PrestoStatement;
|
||||
import io.prestosql.metadata.CatalogManager;
|
||||
import io.prestosql.metadata.SessionPropertyManager;
|
||||
import io.prestosql.plugin.hive.HivePlugin;
|
||||
import io.prestosql.spi.security.Identity;
|
||||
import io.prestosql.sql.parser.SqlParserOptions;
|
||||
import io.prestosql.tests.DistributedQueryRunner;
|
||||
import org.apache.carbondata.common.logging.LogServiceFactory;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
|
@ -72,30 +71,19 @@ public class HetuTestServer
|
|||
{
|
||||
}
|
||||
|
||||
public void startServer(String dbName, Map<String, String> properties, Map<String, String> configProperties) throws Exception
|
||||
{
|
||||
queryRunner = new DistributedQueryRunner(createSession(), 4, hetuProperties, configProperties, new SqlParserOptions(), "testing", Optional.empty());
|
||||
this.dbName = dbName;
|
||||
startServer(properties);
|
||||
}
|
||||
|
||||
public void startServer(String dbName, Map<String, String> properties) throws Exception
|
||||
{
|
||||
queryRunner = new DistributedQueryRunner(createSession(), 4, hetuProperties);
|
||||
this.dbName = dbName;
|
||||
startServer(properties);
|
||||
}
|
||||
|
||||
public void startServer(Map<String, String> properties) throws Exception
|
||||
{
|
||||
this.dbName = dbName;
|
||||
carbonProperties.putAll(properties);
|
||||
|
||||
logger.info("------------ Starting Presto Server -------------");
|
||||
DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
|
||||
DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
|
||||
Connection connection = createJdbcConnection(dbName);
|
||||
statement = (PrestoStatement) connection.createStatement();
|
||||
|
||||
logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
|
||||
logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
|
||||
}
|
||||
|
||||
public void stopServer() throws SQLException
|
||||
|
|
@ -111,8 +99,7 @@ public class HetuTestServer
|
|||
boolean result = false;
|
||||
try {
|
||||
result = statement.execute(query);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
} catch (SQLException e) {
|
||||
logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
|
||||
throw e;
|
||||
}
|
||||
|
|
@ -123,20 +110,13 @@ public class HetuTestServer
|
|||
public List<Map<String, Object>> executeQuery(String query) throws SQLException
|
||||
{
|
||||
logger.info(">>>>> Executing Query: " + query);
|
||||
ResultSet rs = null;
|
||||
try {
|
||||
rs = statement.executeQuery(query);
|
||||
ResultSet rs = statement.executeQuery(query);
|
||||
return convertResultSetToList(rs);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
} catch (SQLException e) {
|
||||
logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
|
||||
throw e;
|
||||
}
|
||||
finally {
|
||||
if (rs != null) {
|
||||
rs.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
|
||||
|
|
@ -175,8 +155,7 @@ public class HetuTestServer
|
|||
|
||||
if (StringUtils.isEmpty(dbName)) {
|
||||
url = "jdbc:presto://localhost:" + port + "/carbondata/default";
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
url = "jdbc:presto://localhost:" + port + "/carbondata/" + dbName;
|
||||
}
|
||||
|
||||
|
|
@ -192,21 +171,20 @@ public class HetuTestServer
|
|||
{
|
||||
try {
|
||||
queryRunner.installPlugin(new CarbondataPlugin());
|
||||
Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
|
||||
Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
|
||||
.putAll(this.carbonProperties)
|
||||
.put("carbon.unsafe.working.memory.in.mb", "512")
|
||||
.build();
|
||||
Map<String, String> carbonPropertiesLocationDisabled = ImmutableMap.<String, String>builder()
|
||||
.putAll(this.carbonProperties)
|
||||
.put("carbon.unsafe.working.memory.in.mb", "512")
|
||||
.put("hive.table-creates-with-location-allowed", "false")
|
||||
.put("hive.table-creates-with-location-allowed","false")
|
||||
.build();
|
||||
|
||||
// CreateCatalog will create a catalog for CarbonData in etc/catalog.
|
||||
queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
|
||||
queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
|
||||
queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
|
||||
}
|
||||
catch (RuntimeException e) {
|
||||
} catch (RuntimeException e) {
|
||||
queryRunner.close();
|
||||
throw e;
|
||||
}
|
||||
|
|
@ -219,9 +197,4 @@ public class HetuTestServer
|
|||
queryRunner.installPlugin(new HivePlugin("hive"));
|
||||
queryRunner.createCatalog("hive", "hive", hiveProperties);
|
||||
}
|
||||
|
||||
public CatalogManager getCatalog()
|
||||
{
|
||||
return queryRunner.getCatalogManager();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,167 +0,0 @@
|
|||
<?xml version="1.0"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-clickhouse</artifactId>
|
||||
<description>hetu - ClickHouse Connector</description>
|
||||
<packaging>hetu-plugin</packaging>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
<resources.dir>src/main/resources</resources.dir>
|
||||
<version.clickhouse-jdbc>0.2.4</version.clickhouse-jdbc>
|
||||
<version.maven-surefire-plugin>3.0.0-M2</version.maven-surefire-plugin>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.google.code.findbugs</groupId>
|
||||
<artifactId>jsr305</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-base-jdbc</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>configuration</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>log</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.guava</groupId>
|
||||
<artifactId>guava</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.google.inject</groupId>
|
||||
<artifactId>guice</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>javax.inject</groupId>
|
||||
<artifactId>javax.inject</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Presto SPI -->
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-spi</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>slice</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>units</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- for testing -->
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-main</artifactId>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcprov-jdk15on</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-tpch</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift.tpch</groupId>
|
||||
<artifactId>tpch</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-tests</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>testing-mysql-server</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-testing-docker</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc -->
|
||||
<dependency>
|
||||
<groupId>ru.yandex.clickhouse</groupId>
|
||||
<artifactId>clickhouse-jdbc</artifactId>
|
||||
<version>${version.clickhouse-jdbc}</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>javax.xml.bind</groupId>
|
||||
<artifactId>jaxb-api</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<!-- <version>${version.maven-surefire-plugin}</version>-->
|
||||
<configuration>
|
||||
<useSystemClassLoader>false</useSystemClassLoader>
|
||||
<forkCount>1</forkCount>
|
||||
<parallel>classes</parallel>
|
||||
<threadCount>1</threadCount>
|
||||
<suiteXmlFiles>
|
||||
<file>src/test/testng.xml</file>
|
||||
</suiteXmlFiles>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
|
||||
</project>
|
||||
|
|
@ -1,584 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.plugin.clickhouse.optimization.ClickHousePushDownParameter;
|
||||
import io.hetu.core.plugin.clickhouse.optimization.ClickHouseQueryGenerator;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcClient;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.ColumnMapping;
|
||||
import io.prestosql.plugin.jdbc.ConnectionFactory;
|
||||
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
|
||||
import io.prestosql.plugin.jdbc.JdbcIdentity;
|
||||
import io.prestosql.plugin.jdbc.JdbcOutputTableHandle;
|
||||
import io.prestosql.plugin.jdbc.JdbcSplit;
|
||||
import io.prestosql.plugin.jdbc.JdbcTableHandle;
|
||||
import io.prestosql.plugin.jdbc.JdbcTypeHandle;
|
||||
import io.prestosql.plugin.jdbc.StatsCollecting;
|
||||
import io.prestosql.plugin.jdbc.WriteMapping;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorResult;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.SuppressFBWarnings;
|
||||
import io.prestosql.spi.connector.ColumnHandle;
|
||||
import io.prestosql.spi.connector.ColumnMetadata;
|
||||
import io.prestosql.spi.connector.ConnectorSession;
|
||||
import io.prestosql.spi.connector.ConnectorTableMetadata;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.function.ExternalFunctionHub;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpressionService;
|
||||
import io.prestosql.spi.sql.QueryGenerator;
|
||||
import io.prestosql.spi.type.CharType;
|
||||
import io.prestosql.spi.type.DecimalType;
|
||||
import io.prestosql.spi.type.Type;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.DatabaseMetaData;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.ResultSetMetaData;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Types;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.bigintColumnMapping;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.integerColumnMapping;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.smallintColumnMapping;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varbinaryWriteFunction;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharColumnMapping;
|
||||
import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction;
|
||||
import static io.prestosql.spi.StandardErrorCode.NOT_FOUND;
|
||||
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
|
||||
import static io.prestosql.spi.type.BooleanType.BOOLEAN;
|
||||
import static io.prestosql.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
|
||||
import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
|
||||
import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
|
||||
import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType;
|
||||
import static io.prestosql.spi.type.Varchars.isVarcharType;
|
||||
import static java.lang.String.format;
|
||||
import static java.lang.String.join;
|
||||
import static java.util.Collections.nCopies;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public class ClickHouseClient
|
||||
extends BaseJdbcClient
|
||||
{
|
||||
private static final Logger logger = Logger.get(ClickHouseClient.class);
|
||||
|
||||
private String[] tableTypes;
|
||||
|
||||
private ClickHouseConfig clickHouseConfig;
|
||||
|
||||
private BaseJdbcConfig baseJdbcConfig;
|
||||
|
||||
/**
|
||||
* If disabled, do not accept sub-query push down.
|
||||
*/
|
||||
private final JdbcPushDownModule pushDownModule;
|
||||
|
||||
/**
|
||||
* constructor
|
||||
*
|
||||
* @param config Base config
|
||||
* @param clickHouseConfig clickhouse config
|
||||
* @param connectionFactory connectionFactory
|
||||
*/
|
||||
@Inject
|
||||
public ClickHouseClient(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig, @StatsCollecting ConnectionFactory connectionFactory, ExternalFunctionHub externalFunctionHub)
|
||||
{
|
||||
super(config, "", connectionFactory, externalFunctionHub);
|
||||
tableTypes = clickHouseConfig.getTableTypes().split(",");
|
||||
this.pushDownModule = config.getPushDownModule();
|
||||
this.baseJdbcConfig = config;
|
||||
this.clickHouseConfig = clickHouseConfig;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ColumnMapping> toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle)
|
||||
{
|
||||
Optional<ColumnMapping> columnMapping;
|
||||
|
||||
String jdbcTypeName = "";
|
||||
if (typeHandle.getJdbcTypeName().isPresent()) {
|
||||
jdbcTypeName = typeHandle.getJdbcTypeName().get().toUpperCase(ENGLISH);
|
||||
}
|
||||
switch (typeHandle.getJdbcType()) {
|
||||
case Types.TINYINT:
|
||||
if (jdbcTypeName.equals("UINT8")) {
|
||||
columnMapping = Optional.of(smallintColumnMapping());
|
||||
}
|
||||
else {
|
||||
columnMapping = Optional.of(tinyintColumnMapping());
|
||||
}
|
||||
break;
|
||||
case Types.INTEGER:
|
||||
if (jdbcTypeName.equals("UINT32")) {
|
||||
columnMapping = Optional.of(bigintColumnMapping());
|
||||
}
|
||||
else {
|
||||
columnMapping = Optional.of(integerColumnMapping());
|
||||
}
|
||||
break;
|
||||
case Types.SMALLINT:
|
||||
if (jdbcTypeName.equals("UINT16")) {
|
||||
columnMapping = Optional.of(integerColumnMapping());
|
||||
}
|
||||
else {
|
||||
columnMapping = Optional.of(smallintColumnMapping());
|
||||
}
|
||||
break;
|
||||
case Types.BIGINT:
|
||||
if (jdbcTypeName.equals("UINT64")) {
|
||||
logger.warn("openLooKeng doesn't support UInt64, it will convert to Int64");
|
||||
}
|
||||
columnMapping = Optional.of(bigintColumnMapping());
|
||||
break;
|
||||
case Types.VARCHAR:
|
||||
columnMapping = Optional.of(varcharColumnMapping(createUnboundedVarcharType()));
|
||||
break;
|
||||
case Types.DATE:
|
||||
case Types.TIMESTAMP:
|
||||
case Types.OTHER:
|
||||
default:
|
||||
columnMapping = super.toPrestoType(session, connection, typeHandle);
|
||||
}
|
||||
if (!columnMapping.isPresent()) {
|
||||
throw new UnsupportedOperationException(
|
||||
"openLooKeng does not support the clickhouse type " + typeHandle.getJdbcTypeName().orElse("")
|
||||
+ '(' + typeHandle.getColumnSize() + ", " + typeHandle.getDecimalDigits()
|
||||
+ ')');
|
||||
}
|
||||
return columnMapping;
|
||||
}
|
||||
|
||||
@Override
|
||||
public WriteMapping toWriteMapping(ConnectorSession session, Type type)
|
||||
{
|
||||
if (isVarcharType(type)) {
|
||||
return WriteMapping.sliceMapping("String", varcharWriteFunction());
|
||||
}
|
||||
else if (VARBINARY.equals(type)) {
|
||||
return WriteMapping.sliceMapping("String", varbinaryWriteFunction());
|
||||
}
|
||||
else if (type instanceof CharType) {
|
||||
return WriteMapping.sliceMapping("FixedString(" + ((CharType) type).getLength() + ')',
|
||||
charWriteFunction());
|
||||
}
|
||||
else if (type instanceof DecimalType) {
|
||||
DecimalType decimalType = (DecimalType) type;
|
||||
String dataType = format("decimal(%s, %s)", decimalType.getPrecision(), decimalType.getScale());
|
||||
if (decimalType.isShort()) {
|
||||
return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType));
|
||||
}
|
||||
return WriteMapping.sliceMapping(dataType, longDecimalWriteFunction(decimalType));
|
||||
}
|
||||
else if (BOOLEAN.equals(type)) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "There is no separate type for boolean values. Use UInt8 type, restricted to the values 0 or 1.");
|
||||
}
|
||||
else if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());
|
||||
}
|
||||
|
||||
return super.toWriteMapping(session, type);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<String> listSchemas(Connection connection)
|
||||
{
|
||||
try (ResultSet resultSet = connection.getMetaData().getSchemas()) {
|
||||
ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder();
|
||||
while (resultSet.next()) {
|
||||
String schemaName = resultSet.getString(1);
|
||||
schemaNames.add(schemaName);
|
||||
}
|
||||
return schemaNames.build();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to list schemas");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected ResultSet getTables(Connection connection, Optional<String> schemaName, Optional<String> tableName)
|
||||
throws SQLException
|
||||
{
|
||||
DatabaseMetaData metadata = connection.getMetaData();
|
||||
Optional<String> escape = Optional.ofNullable(metadata.getSearchStringEscape());
|
||||
return metadata.getTables(connection.getCatalog(), escapeNamePattern(schemaName, escape).orElse(null), escapeNamePattern(tableName, escape).orElse(null), tableTypes);
|
||||
}
|
||||
|
||||
/**
|
||||
* create table function should be disabled.
|
||||
*/
|
||||
@Override
|
||||
protected JdbcOutputTableHandle createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, String tableName)
|
||||
{
|
||||
throw new UnsupportedOperationException("ClickHouse Connector does not support create table.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column)
|
||||
{
|
||||
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session))) {
|
||||
String columnName = column.getName();
|
||||
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
|
||||
columnName = columnName.toUpperCase(ENGLISH);
|
||||
}
|
||||
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String sql = format("ALTER TABLE %s ADD column %s", quoted(null, schema, handle.getTableName()), getColumnSql(session, column, columnName));
|
||||
execute(connection, sql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to add column, check table engine and sql");
|
||||
}
|
||||
}
|
||||
|
||||
private String getColumnSql(ConnectorSession session, ColumnMetadata column, String columnName)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY)
|
||||
.append(quoted(columnName))
|
||||
.append(" ")
|
||||
.append(toWriteMapping(session, column.getType()).getDataType());
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column)
|
||||
{
|
||||
try (Connection connection = connectionFactory.openConnection(identity)) {
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String sql = format("ALTER TABLE %s DROP column %s", quoted(null, schema, handle.getTableName()), column.getColumnName());
|
||||
execute(connection, sql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to drop column");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTableModificationTime(ConnectorSession session, JdbcTableHandle handle)
|
||||
{
|
||||
String sql = format(
|
||||
"SELECT max(modification_time) as modified_time FROM system.parts WHERE database='%s' and table='%s'",
|
||||
handle.getSchemaTableName().getSchemaName(),
|
||||
handle.getSchemaTableName().getTableName());
|
||||
|
||||
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
|
||||
PreparedStatement preparedStatement = getPreparedStatement(connection, sql)) {
|
||||
ResultSet resultSet = preparedStatement.executeQuery();
|
||||
if (resultSet.next()) {
|
||||
String updateTime = resultSet.getString("modified_time");
|
||||
SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
|
||||
return parser.parse(updateTime).getTime();
|
||||
}
|
||||
else {
|
||||
// In the case where for some reason the table doesn't exist anymore in system.parts
|
||||
return -1L;
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
// We want to make sure the query doesn't fail because of star-tree not being able to get last modified time
|
||||
logger.error("Exception thrown while trying to get modified time", e);
|
||||
return -1L;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isPreAggregationSupported(ConnectorSession session)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void renameTable(JdbcIdentity identity, String catalogName, String schemaName, String tableName, SchemaTableName newTable)
|
||||
{
|
||||
try (Connection connection = connectionFactory.openConnection(identity)) {
|
||||
String newSchemaName = newTable.getSchemaName();
|
||||
String newTableName = newTable.getTableName();
|
||||
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
|
||||
newSchemaName = newSchemaName.toUpperCase(ENGLISH);
|
||||
newTableName = newTableName.toUpperCase(ENGLISH);
|
||||
}
|
||||
|
||||
String schema = checkCatalog(catalogName, schemaName);
|
||||
String newSchema = checkCatalog(catalogName, newSchemaName);
|
||||
String sql = format("RENAME TABLE %s TO %s", quoted(null, schema, tableName), quoted(null, newSchema, newTableName));
|
||||
execute(connection, sql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to rename table");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
|
||||
{
|
||||
String newColumnName = inputNewColumnName;
|
||||
try (Connection connection = connectionFactory.openConnection(identity)) {
|
||||
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
|
||||
newColumnName = newColumnName.toUpperCase(ENGLISH);
|
||||
}
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String sql = format(
|
||||
"ALTER TABLE %s RENAME COLUMN %s TO %s",
|
||||
quoted(null, schema, handle.getTableName()),
|
||||
jdbcColumn.getColumnName(),
|
||||
newColumnName);
|
||||
execute(connection, sql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
|
||||
{
|
||||
SchemaTableName schemaTableName = tableMetadata.getTable();
|
||||
|
||||
JdbcIdentity identity = JdbcIdentity.from(session);
|
||||
if (!getSchemaNames(identity).contains(schemaTableName.getSchemaName())) {
|
||||
throw new PrestoException(NOT_FOUND, "Schema not found: " + schemaTableName.getSchemaName());
|
||||
}
|
||||
|
||||
try (Connection connection = connectionFactory.openConnection(identity)) {
|
||||
boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers();
|
||||
String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName());
|
||||
String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName());
|
||||
String temporaryTableName = generateTemporaryTableName();
|
||||
String catalog = connection.getCatalog();
|
||||
|
||||
ImmutableList.Builder<String> columnNames = ImmutableList.builder();
|
||||
ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
|
||||
ImmutableList.Builder<String> columnList = ImmutableList.builder();
|
||||
for (ColumnMetadata column : tableMetadata.getColumns()) {
|
||||
String columnName = column.getName();
|
||||
if (uppercase) {
|
||||
columnName = columnName.toUpperCase(ENGLISH);
|
||||
}
|
||||
columnNames.add(columnName);
|
||||
columnTypes.add(column.getType());
|
||||
columnList.add(getColumnSql(session, column, columnName));
|
||||
}
|
||||
|
||||
String schema = checkCatalog(catalog, remoteSchema);
|
||||
String sql = format(
|
||||
"CREATE TABLE %s engine=Log AS SELECT * FROM %s WHERE 0 = 1",
|
||||
quoted(null, schema, temporaryTableName),
|
||||
quoted(null, schema, remoteTable));
|
||||
execute(connection, sql);
|
||||
|
||||
return new JdbcOutputTableHandle(
|
||||
catalog,
|
||||
remoteSchema,
|
||||
remoteTable,
|
||||
columnNames.build(),
|
||||
columnTypes.build(),
|
||||
temporaryTableName);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String buildInsertSql(JdbcOutputTableHandle handle)
|
||||
{
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String sql = format(
|
||||
"INSERT INTO %s VALUES (%s)",
|
||||
quoted(null, schema, handle.getTemporaryTableName()),
|
||||
join(",", nCopies(handle.getColumnNames().size(), "?")));
|
||||
return sql;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List<JdbcColumnHandle> columns)
|
||||
throws SQLException
|
||||
{
|
||||
if (table.getGeneratedSql().isPresent()) {
|
||||
// openLooKeng: If the sub-query is pushed down, use it as the table
|
||||
return new ClickHouseQueryBuilder(identifierQuote, true).buildSql(
|
||||
this,
|
||||
session,
|
||||
connection,
|
||||
null,
|
||||
null,
|
||||
table.getGeneratedSql().get().getSql(),
|
||||
columns,
|
||||
table.getConstraint(),
|
||||
split.getAdditionalPredicate(),
|
||||
tryApplyLimit(table.getLimit()));
|
||||
}
|
||||
return new ClickHouseQueryBuilder(identifierQuote).buildSql(
|
||||
this,
|
||||
session,
|
||||
connection,
|
||||
table.getCatalogName(),
|
||||
table.getSchemaName(),
|
||||
table.getTableName(),
|
||||
columns,
|
||||
table.getConstraint(),
|
||||
split.getAdditionalPredicate(),
|
||||
tryApplyLimit(table.getLimit()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finishInsertTable(JdbcIdentity identity, JdbcOutputTableHandle handle)
|
||||
{
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String temporaryTable = quoted(null, schema, handle.getTemporaryTableName());
|
||||
String targetTable = quoted(null, schema, handle.getTableName());
|
||||
String insertSql = format("INSERT INTO %s SELECT * FROM %s", targetTable, temporaryTable);
|
||||
String cleanupSql = "DROP TABLE " + temporaryTable;
|
||||
|
||||
try (Connection connection = getConnection(identity, handle)) {
|
||||
execute(connection, insertSql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
|
||||
try (Connection connection = getConnection(identity, handle)) {
|
||||
execute(connection, cleanupSql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
logger.warn(e, "Failed to cleanup temporary table: %s", temporaryTable);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropTable(JdbcIdentity identity, JdbcTableHandle handle)
|
||||
{
|
||||
String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
|
||||
String sql = "DROP TABLE " + quoted(null, schema, handle.getTableName());
|
||||
|
||||
try (Connection connection = connectionFactory.openConnection(identity)) {
|
||||
execute(connection, sql);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Optional<BiFunction<String, Long, String>> limitFunction()
|
||||
{
|
||||
return Optional.of((sql, limit) -> sql + " LIMIT " + limit);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isLimitGuaranteed()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<QueryGenerator<JdbcQueryGeneratorResult, JdbcConverterContext>> getQueryGenerator(DeterminismEvaluator determinismEvaluator, RowExpressionService rowExpressionService, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution)
|
||||
{
|
||||
ClickHousePushDownParameter pushDownParameter = new ClickHousePushDownParameter(getIdentifierQuote(), this.caseInsensitiveNameMatching, pushDownModule, clickHouseConfig, functionResolution);
|
||||
return Optional.of(new ClickHouseQueryGenerator(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig));
|
||||
}
|
||||
|
||||
@SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING")
|
||||
@Override
|
||||
public Map<String, ColumnHandle> getColumns(ConnectorSession session, String sql, Map<String, Type> types)
|
||||
{
|
||||
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
|
||||
PreparedStatement statement = connection.prepareStatement(sql)) {
|
||||
ResultSetMetaData metadata = statement.getMetaData();
|
||||
ImmutableMap.Builder<String, ColumnHandle> builder = new ImmutableMap.Builder<>();
|
||||
for (int i = 1; i <= metadata.getColumnCount(); i++) {
|
||||
String columnName = metadata.getColumnLabel(i);
|
||||
String typeName = metadata.getColumnTypeName(i);
|
||||
int precision = metadata.getPrecision(i);
|
||||
int dataType = metadata.getColumnType(i);
|
||||
int scale = metadata.getScale(i);
|
||||
|
||||
boolean isNullAble = metadata.isNullable(i) != ResultSetMetaData.columnNoNulls;
|
||||
if (dataType == Types.DECIMAL) {
|
||||
String loweredColumnName = columnName.toLowerCase(ENGLISH);
|
||||
Type type = types.get(loweredColumnName);
|
||||
if (type instanceof DecimalType) {
|
||||
DecimalType decimalType = (DecimalType) type;
|
||||
precision = decimalType.getPrecision();
|
||||
scale = decimalType.getScale();
|
||||
}
|
||||
}
|
||||
|
||||
JdbcTypeHandle typeHandle = new JdbcTypeHandle(dataType, Optional.ofNullable(typeName), precision, scale, Optional.empty());
|
||||
Optional<ColumnMapping> columnMapping;
|
||||
try {
|
||||
columnMapping = toPrestoType(session, connection, typeHandle);
|
||||
}
|
||||
catch (UnsupportedOperationException ex) {
|
||||
// User configured to fail the query if the data type is not supported
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
// skip unsupported column types
|
||||
if (columnMapping.isPresent()) {
|
||||
Type type = columnMapping.get().getType();
|
||||
JdbcColumnHandle handle = new JdbcColumnHandle(columnName, typeHandle, type, isNullAble);
|
||||
builder.put(columnName.toLowerCase(ENGLISH), handle);
|
||||
}
|
||||
else {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
catch (SQLException | PrestoException e) {
|
||||
logger.error("in clickhouse push down, clickhouse data source error msg[%s] rewrite sql[%s]", e.getMessage(), sql);
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clickhouse does not support operations on catalog.schema.table
|
||||
*/
|
||||
private String checkCatalog(String catalog, String remoteSchema)
|
||||
{
|
||||
String schema;
|
||||
if (catalog != null && remoteSchema != null) {
|
||||
schema = remoteSchema;
|
||||
}
|
||||
else {
|
||||
schema = catalog == null ? remoteSchema : catalog;
|
||||
}
|
||||
return schema;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,61 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.inject.Binder;
|
||||
import com.google.inject.Provides;
|
||||
import com.google.inject.Scopes;
|
||||
import com.google.inject.Singleton;
|
||||
import io.airlift.configuration.AbstractConfigurationAwareModule;
|
||||
import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.ConnectionFactory;
|
||||
import io.prestosql.plugin.jdbc.DriverConnectionFactory;
|
||||
import io.prestosql.plugin.jdbc.JdbcClient;
|
||||
import io.prestosql.spi.function.ExternalFunctionHub;
|
||||
import ru.yandex.clickhouse.ClickHouseDriver;
|
||||
import ru.yandex.clickhouse.settings.ClickHouseConnectionSettings;
|
||||
|
||||
import java.util.Optional;
|
||||
import java.util.Properties;
|
||||
|
||||
import static io.airlift.configuration.ConfigBinder.configBinder;
|
||||
import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
|
||||
|
||||
public class ClickHouseClientModule
|
||||
extends AbstractConfigurationAwareModule
|
||||
{
|
||||
@Override
|
||||
protected void setup(Binder binder)
|
||||
{
|
||||
binder.bind(JdbcClient.class).to(ClickHouseClient.class).in(Scopes.SINGLETON);
|
||||
configBinder(binder).bindConfig(BaseJdbcConfig.class);
|
||||
binder.bind(ExternalFunctionHub.class).to(ClickHouseExternalFunctionHub.class).in(Scopes.SINGLETON);
|
||||
configBinder(binder).bindConfig(ClickHouseConfig.class);
|
||||
}
|
||||
|
||||
@Provides
|
||||
@Singleton
|
||||
public static ConnectionFactory createConnectionFactory(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig)
|
||||
{
|
||||
Properties connectionProperties = basicConnectionProperties(config);
|
||||
connectionProperties.setProperty(ClickHouseConnectionSettings.SOCKET_TIMEOUT.getKey(),
|
||||
String.valueOf(clickHouseConfig.getSocketTimeout()));
|
||||
return new DriverConnectionFactory(new ClickHouseDriver(), config.getConnectionUrl(),
|
||||
Optional.ofNullable(config.getUserCredentialName()),
|
||||
Optional.ofNullable(config.getPasswordCredentialName()),
|
||||
connectionProperties);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,100 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import io.airlift.configuration.Config;
|
||||
import io.airlift.configuration.ConfigDescription;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* To get the custom properties to connect to the database.
|
||||
*/
|
||||
public class ClickHouseConfig
|
||||
{
|
||||
private static final int DEFAULT_SOCKET_TIMEOUT = 120000;
|
||||
private int socketTimeout = DEFAULT_SOCKET_TIMEOUT;
|
||||
private String tableTypes = ClickHouseConstants.DEFAULT_TABLE_TYPES;
|
||||
private String schemaPattern;
|
||||
private boolean isQueryPushDownEnabled = true;
|
||||
private String clickHouseSqlVersion = "DEFAULT";
|
||||
|
||||
public int getSocketTimeout()
|
||||
{
|
||||
return socketTimeout;
|
||||
}
|
||||
|
||||
@Config("clickhouse.socket_timeout")
|
||||
@ConfigDescription("Connection ClickHouse socket timeout ")
|
||||
public ClickHouseConfig setSocketTimeout(int socketTimeout)
|
||||
{
|
||||
this.socketTimeout = socketTimeout;
|
||||
return this;
|
||||
}
|
||||
|
||||
@Config("clickhouse.query.pushdown.enabled")
|
||||
@ConfigDescription("Enable sub-query push down to clickhouse. It's set by default")
|
||||
public ClickHouseConfig setQueryPushDownEnabled(boolean isQueryPushDownEnabledParameter)
|
||||
{
|
||||
this.isQueryPushDownEnabled = isQueryPushDownEnabledParameter;
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean isQueryPushDownEnabled()
|
||||
{
|
||||
return this.isQueryPushDownEnabled;
|
||||
}
|
||||
|
||||
public String getTableTypes()
|
||||
{
|
||||
return this.tableTypes;
|
||||
}
|
||||
|
||||
public String getSchemaPattern()
|
||||
{
|
||||
return schemaPattern;
|
||||
}
|
||||
|
||||
/**
|
||||
* setTableTypes
|
||||
*
|
||||
* @param tableTypes the table types to set
|
||||
* @return ClickHouseConfig
|
||||
*/
|
||||
@Config("clickhouse.table-types")
|
||||
public ClickHouseConfig setTableTypes(String tableTypes)
|
||||
{
|
||||
this.tableTypes = tableTypes.toUpperCase(Locale.ENGLISH);
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getClickHouseSqlVersion()
|
||||
{
|
||||
return this.clickHouseSqlVersion;
|
||||
}
|
||||
|
||||
/**
|
||||
* setSchemaPattern
|
||||
*
|
||||
* @param schemaPattern the schema pattern to set
|
||||
* @return ClickHouseConfig
|
||||
*/
|
||||
@Config("clickhouse.schema-pattern")
|
||||
public ClickHouseConfig setSchemaPattern(String schemaPattern)
|
||||
{
|
||||
this.schemaPattern = schemaPattern;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.JdbcPlugin;
|
||||
import io.prestosql.spi.function.ConnectorConfig;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorUtil;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
|
||||
@ConnectorConfig(connectorLabel = "Clickhouse: Query data in Clickhouse system",
|
||||
propertiesEnabled = true,
|
||||
catalogConfigFilesEnabled = true,
|
||||
globalConfigFilesEnabled = true,
|
||||
docLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html",
|
||||
configLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html#configuration")
|
||||
public class ClickHousePlugin
|
||||
extends JdbcPlugin
|
||||
{
|
||||
public ClickHousePlugin()
|
||||
{
|
||||
super("clickhouse", new ClickHouseClientModule());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ConnectorWithProperties> getConnectorWithProperties()
|
||||
{
|
||||
ConnectorConfig connectorConfig = ClickHousePlugin.class.getAnnotation(ConnectorConfig.class);
|
||||
ArrayList<Method> methods = new ArrayList<>();
|
||||
methods.addAll(Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
|
||||
methods.addAll(Arrays.asList(ClickHouseConfig.class.getDeclaredMethods()));
|
||||
return ConnectorUtil.assembleConnectorProperties(connectorConfig, methods);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,346 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.VerifyException;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import io.airlift.slice.Slice;
|
||||
import io.prestosql.plugin.jdbc.BlockWriteFunction;
|
||||
import io.prestosql.plugin.jdbc.BooleanWriteFunction;
|
||||
import io.prestosql.plugin.jdbc.DoubleWriteFunction;
|
||||
import io.prestosql.plugin.jdbc.JdbcClient;
|
||||
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
|
||||
import io.prestosql.plugin.jdbc.JdbcTypeHandle;
|
||||
import io.prestosql.plugin.jdbc.LongWriteFunction;
|
||||
import io.prestosql.plugin.jdbc.QueryBuilder;
|
||||
import io.prestosql.plugin.jdbc.SliceWriteFunction;
|
||||
import io.prestosql.plugin.jdbc.WriteFunction;
|
||||
import io.prestosql.spi.block.Block;
|
||||
import io.prestosql.spi.connector.ColumnHandle;
|
||||
import io.prestosql.spi.connector.ConnectorSession;
|
||||
import io.prestosql.spi.predicate.Domain;
|
||||
import io.prestosql.spi.predicate.Range;
|
||||
import io.prestosql.spi.predicate.TupleDomain;
|
||||
import io.prestosql.spi.type.ArrayType;
|
||||
import io.prestosql.spi.type.BigintType;
|
||||
import io.prestosql.spi.type.BooleanType;
|
||||
import io.prestosql.spi.type.CharType;
|
||||
import io.prestosql.spi.type.DateType;
|
||||
import io.prestosql.spi.type.DoubleType;
|
||||
import io.prestosql.spi.type.IntegerType;
|
||||
import io.prestosql.spi.type.RealType;
|
||||
import io.prestosql.spi.type.SmallintType;
|
||||
import io.prestosql.spi.type.TimeType;
|
||||
import io.prestosql.spi.type.TimeWithTimeZoneType;
|
||||
import io.prestosql.spi.type.TimestampType;
|
||||
import io.prestosql.spi.type.TimestampWithTimeZoneType;
|
||||
import io.prestosql.spi.type.TinyintType;
|
||||
import io.prestosql.spi.type.Type;
|
||||
import io.prestosql.spi.type.VarcharType;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.function.Function;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Collections.nCopies;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static java.util.stream.Collectors.joining;
|
||||
|
||||
public class ClickHouseQueryBuilder
|
||||
extends QueryBuilder
|
||||
{
|
||||
private static final String ALWAYS_TRUE = "1=1";
|
||||
private static final String ALWAYS_FALSE = "1=0";
|
||||
private boolean isPushSubQueryDown;
|
||||
private final String identifierQuote;
|
||||
|
||||
public ClickHouseQueryBuilder(String identifierQuote)
|
||||
{
|
||||
super(identifierQuote);
|
||||
this.identifierQuote = requireNonNull(identifierQuote, "quote is null");
|
||||
}
|
||||
|
||||
public ClickHouseQueryBuilder(String identifierQuote, boolean isPushSubQueryDown)
|
||||
{
|
||||
this(identifierQuote);
|
||||
this.isPushSubQueryDown = isPushSubQueryDown;
|
||||
}
|
||||
|
||||
private static class TypeAndValue
|
||||
{
|
||||
private final Type type;
|
||||
private final JdbcTypeHandle typeHandle;
|
||||
private final Object value;
|
||||
|
||||
public TypeAndValue(Type type, JdbcTypeHandle typeHandle, Object value)
|
||||
{
|
||||
this.type = requireNonNull(type, "type is null");
|
||||
this.typeHandle = requireNonNull(typeHandle, "typeHandle is null");
|
||||
this.value = requireNonNull(value, "value is null");
|
||||
}
|
||||
|
||||
public Type getType()
|
||||
{
|
||||
return type;
|
||||
}
|
||||
|
||||
public JdbcTypeHandle getTypeHandle()
|
||||
{
|
||||
return typeHandle;
|
||||
}
|
||||
|
||||
public Object getValue()
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
private String quote(String name)
|
||||
{
|
||||
return identifierQuote + name.replace(identifierQuote, identifierQuote + identifierQuote) + identifierQuote;
|
||||
}
|
||||
|
||||
private static Domain pushDownDomain(JdbcClient client, ConnectorSession session, Connection connection, JdbcColumnHandle column, Domain domain)
|
||||
{
|
||||
return client.toPrestoType(session, connection, column.getJdbcTypeHandle())
|
||||
.orElseThrow(() -> new IllegalStateException(format("Unsupported type %s with handle %s", column.getColumnType(), column.getJdbcTypeHandle())))
|
||||
.getPushdownConverter().apply(domain);
|
||||
}
|
||||
|
||||
private List<String> toConjuncts(
|
||||
JdbcClient client,
|
||||
ConnectorSession session,
|
||||
Connection connection,
|
||||
List<JdbcColumnHandle> columns,
|
||||
TupleDomain<ColumnHandle> tupleDomain,
|
||||
List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
|
||||
{
|
||||
if (tupleDomain.isNone()) {
|
||||
return ImmutableList.of(ALWAYS_FALSE);
|
||||
}
|
||||
ImmutableList.Builder<String> builder = ImmutableList.builder();
|
||||
for (JdbcColumnHandle column : columns) {
|
||||
Domain domain = tupleDomain.getDomains().get().get(column);
|
||||
if (domain != null) {
|
||||
domain = pushDownDomain(client, session, connection, column, domain);
|
||||
builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
|
||||
}
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private String toPredicate(String columnName, Domain domain, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
|
||||
{
|
||||
if (domain.getValues().isNone()) {
|
||||
return domain.isNullAllowed() ? quote(columnName) + " IS NULL" : ALWAYS_FALSE;
|
||||
}
|
||||
|
||||
if (domain.getValues().isAll()) {
|
||||
return domain.isNullAllowed() ? ALWAYS_TRUE : quote(columnName) + " IS NOT NULL";
|
||||
}
|
||||
|
||||
List<String> disjuncts = new ArrayList<>();
|
||||
List<Object> singleValues = new ArrayList<>();
|
||||
for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
|
||||
checkState(!range.isAll()); // Already checked
|
||||
if (range.isSingleValue()) {
|
||||
singleValues.add(range.getLow().getValue());
|
||||
}
|
||||
else {
|
||||
List<String> rangeConjuncts = new ArrayList<>();
|
||||
if (!range.getLow().isLowerUnbounded()) {
|
||||
switch (range.getLow().getBound()) {
|
||||
case ABOVE:
|
||||
rangeConjuncts.add(toPredicate(columnName, ">", range.getLow().getValue(), column, accumulator));
|
||||
break;
|
||||
case EXACTLY:
|
||||
rangeConjuncts.add(toPredicate(columnName, ">=", range.getLow().getValue(), column, accumulator));
|
||||
break;
|
||||
case BELOW:
|
||||
throw new IllegalArgumentException("Low marker should never use BELOW bound");
|
||||
default:
|
||||
throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
|
||||
}
|
||||
}
|
||||
if (!range.getHigh().isUpperUnbounded()) {
|
||||
switch (range.getHigh().getBound()) {
|
||||
case ABOVE:
|
||||
throw new IllegalArgumentException("High marker should never use ABOVE bound");
|
||||
case EXACTLY:
|
||||
rangeConjuncts.add(toPredicate(columnName, "<=", range.getHigh().getValue(), column, accumulator));
|
||||
break;
|
||||
case BELOW:
|
||||
rangeConjuncts.add(toPredicate(columnName, "<", range.getHigh().getValue(), column, accumulator));
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
|
||||
}
|
||||
}
|
||||
// If rangeConjuncts is null, then the range was ALL, which should already have been checked for
|
||||
checkState(!rangeConjuncts.isEmpty());
|
||||
disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")");
|
||||
}
|
||||
}
|
||||
|
||||
// Add back all of the possible single values either as an equality or an IN predicate
|
||||
if (singleValues.size() == 1) {
|
||||
disjuncts.add(toPredicate(columnName, "=", getOnlyElement(singleValues), column, accumulator));
|
||||
}
|
||||
else if (singleValues.size() > 1) {
|
||||
for (Object value : singleValues) {
|
||||
bindValue(value, column, accumulator);
|
||||
}
|
||||
String values = Joiner.on(",").join(nCopies(singleValues.size(), "?"));
|
||||
disjuncts.add(quote(columnName) + " IN (" + values + ")");
|
||||
}
|
||||
|
||||
// Add nullability disjuncts
|
||||
checkState(!disjuncts.isEmpty());
|
||||
if (domain.isNullAllowed()) {
|
||||
disjuncts.add(quote(columnName) + " IS NULL");
|
||||
}
|
||||
|
||||
return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
|
||||
}
|
||||
|
||||
private String toPredicate(String columnName, String operator, Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
|
||||
{
|
||||
bindValue(value, column, accumulator);
|
||||
return quote(columnName) + " " + operator + " ?";
|
||||
}
|
||||
|
||||
private static void bindValue(Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
|
||||
{
|
||||
Type type = column.getColumnType();
|
||||
checkArgument(isAcceptedType(type), "Can't handle type: %s", type);
|
||||
accumulator.add(new TypeAndValue(type, column.getJdbcTypeHandle(), value));
|
||||
}
|
||||
|
||||
private static boolean isAcceptedType(Type type)
|
||||
{
|
||||
Type validType = requireNonNull(type, "type is null");
|
||||
return validType.equals(BigintType.BIGINT) ||
|
||||
validType.equals(TinyintType.TINYINT) ||
|
||||
validType.equals(SmallintType.SMALLINT) ||
|
||||
validType.equals(IntegerType.INTEGER) ||
|
||||
validType.equals(DoubleType.DOUBLE) ||
|
||||
validType.equals(RealType.REAL) ||
|
||||
validType.equals(BooleanType.BOOLEAN) ||
|
||||
validType.equals(DateType.DATE) ||
|
||||
validType.equals(TimeType.TIME) ||
|
||||
validType.equals(TimeWithTimeZoneType.TIME_WITH_TIME_ZONE) ||
|
||||
validType.equals(TimestampType.TIMESTAMP) ||
|
||||
validType.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) ||
|
||||
validType instanceof VarcharType ||
|
||||
validType instanceof CharType ||
|
||||
validType instanceof ArrayType;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PreparedStatement buildSql(JdbcClient client,
|
||||
ConnectorSession session,
|
||||
Connection connection,
|
||||
String catalog,
|
||||
String schema,
|
||||
String table,
|
||||
List<JdbcColumnHandle> columns,
|
||||
TupleDomain<ColumnHandle> tupleDomain,
|
||||
Optional<String> additionalPredicate,
|
||||
Function<String, String> sqlFunction)
|
||||
throws SQLException
|
||||
{
|
||||
StringBuilder sql = new StringBuilder();
|
||||
|
||||
String columnNames = columns.stream()
|
||||
.map(JdbcColumnHandle::getColumnName)
|
||||
.map(this::quote)
|
||||
.collect(joining(", "));
|
||||
|
||||
sql.append("SELECT ");
|
||||
sql.append(columnNames);
|
||||
if (columns.isEmpty()) {
|
||||
sql.append("null");
|
||||
}
|
||||
|
||||
sql.append(" FROM ");
|
||||
if (!isNullOrEmpty(schema)) {
|
||||
sql.append(quote(schema)).append('.');
|
||||
}
|
||||
if (isPushSubQueryDown) {
|
||||
sql.append("(").append(table).append(") pushdown");
|
||||
}
|
||||
else {
|
||||
sql.append(quote(table));
|
||||
}
|
||||
|
||||
List<TypeAndValue> accumulator = new ArrayList<>();
|
||||
|
||||
List<String> clauses = toConjuncts(client, session, connection, columns, tupleDomain, accumulator);
|
||||
if (additionalPredicate.isPresent()) {
|
||||
clauses = ImmutableList.<String>builder()
|
||||
.addAll(clauses)
|
||||
.add(additionalPredicate.get())
|
||||
.build();
|
||||
}
|
||||
if (!clauses.isEmpty()) {
|
||||
sql.append(" WHERE ")
|
||||
.append(Joiner.on(" AND ").join(clauses));
|
||||
}
|
||||
|
||||
String query = sqlFunction.apply(sql.toString());
|
||||
PreparedStatement statement = client.getPreparedStatement(connection, query);
|
||||
|
||||
for (int i = 0; i < accumulator.size(); i++) {
|
||||
TypeAndValue typeAndValue = accumulator.get(i);
|
||||
int parameterIndex = i + 1;
|
||||
Type type = typeAndValue.getType();
|
||||
WriteFunction writeFunction = client.toPrestoType(session, connection, typeAndValue.getTypeHandle())
|
||||
.orElseThrow(() -> new VerifyException(format("Unsupported type %s with handle %s", type, typeAndValue.getTypeHandle())))
|
||||
.getWriteFunction();
|
||||
Class<?> javaType = type.getJavaType();
|
||||
Object value = typeAndValue.getValue();
|
||||
if (javaType == boolean.class) {
|
||||
((BooleanWriteFunction) writeFunction).set(statement, parameterIndex, (boolean) value);
|
||||
}
|
||||
else if (javaType == long.class) {
|
||||
((LongWriteFunction) writeFunction).set(statement, parameterIndex, (long) value);
|
||||
}
|
||||
else if (javaType == double.class) {
|
||||
((DoubleWriteFunction) writeFunction).set(statement, parameterIndex, (double) value);
|
||||
}
|
||||
else if (javaType == Slice.class) {
|
||||
((SliceWriteFunction) writeFunction).set(statement, parameterIndex, (Slice) value);
|
||||
}
|
||||
else if (javaType == Block.class) {
|
||||
((BlockWriteFunction) writeFunction).set(statement, parameterIndex, (Block) value);
|
||||
}
|
||||
else {
|
||||
throw new VerifyException(format("Unexpected type %s with java type %s", type, javaType.getName()));
|
||||
}
|
||||
}
|
||||
|
||||
return statement;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
|
||||
import io.prestosql.spi.function.SqlFunctionHandle;
|
||||
import io.prestosql.spi.relation.CallExpression;
|
||||
import io.prestosql.sql.builder.functioncall.ApplyRemoteFunctionPushDown;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static java.util.stream.Collectors.toList;
|
||||
|
||||
public class ClickHouseApplyRemoteFunctionPushDown
|
||||
extends ApplyRemoteFunctionPushDown
|
||||
{
|
||||
private static Map<String, String> clickHousePrimalFunctionNameMap = ClickHouseExternalFunctionHub.getPrimalFunctionNameMap();
|
||||
|
||||
public ClickHouseApplyRemoteFunctionPushDown(BaseJdbcConfig baseJdbcConfig, String connectorName)
|
||||
{
|
||||
super(baseJdbcConfig, connectorName);
|
||||
}
|
||||
|
||||
/**
|
||||
* rewrite the remote function to a executable function in the data source.
|
||||
*/
|
||||
@Override
|
||||
public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
|
||||
{
|
||||
if (!isConnectorSupportedRemoteFunction(callExpression)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
jdbcConverterContext.setRemoteUdfVisited(true);
|
||||
String displayName = ((SqlFunctionHandle) callExpression.getFunctionHandle()).getFunctionId().getFunctionName().getObjectName();
|
||||
String args = Joiner.on(",").join(callExpression.getArguments().stream().map(expression -> expression.accept(rowExpressionConverter, jdbcConverterContext)).collect(toList()));
|
||||
// The click house is case sensitive and presto only support the lower case function name,
|
||||
// we need to handle this function into lower and store its primal name.
|
||||
// The click house do not contain two different functions have the equal function name ignore case.
|
||||
// If a data base contain two different functions have the equal function name ignore case,
|
||||
// then we should register them into different function name space
|
||||
if (!clickHousePrimalFunctionNameMap.containsKey(displayName)) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(String.format("%s(%s)", clickHousePrimalFunctionNameMap.get(displayName), args));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization;
|
||||
|
||||
import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
|
||||
/**
|
||||
* Push Down parameter module
|
||||
*/
|
||||
public class ClickHousePushDownParameter
|
||||
extends JdbcPushDownParameter
|
||||
{
|
||||
private final ClickHouseConfig clickHouseConfig;
|
||||
|
||||
public ClickHousePushDownParameter(String identifierQuote, boolean nameCaseInsensitive, JdbcPushDownModule pushDownModule, ClickHouseConfig clickHouseConfig, StandardFunctionResolution functionResolution)
|
||||
{
|
||||
super(identifierQuote, nameCaseInsensitive, pushDownModule, functionResolution);
|
||||
this.clickHouseConfig = clickHouseConfig;
|
||||
}
|
||||
|
||||
public ClickHouseConfig getClickHouseConfig()
|
||||
{
|
||||
return clickHouseConfig;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization;
|
||||
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcQueryGenerator;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpressionService;
|
||||
|
||||
public class ClickHouseQueryGenerator
|
||||
extends BaseJdbcQueryGenerator
|
||||
{
|
||||
public ClickHouseQueryGenerator(
|
||||
DeterminismEvaluator determinismEvaluator,
|
||||
RowExpressionService rowExpressionService,
|
||||
FunctionMetadataManager functionManager,
|
||||
StandardFunctionResolution functionResolution,
|
||||
ClickHousePushDownParameter pushDownParameter,
|
||||
BaseJdbcConfig baseJdbcConfig)
|
||||
{
|
||||
super(pushDownParameter, new ClickHouseRowExpressionConverter(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig), new ClickHouseSqlStatementWriter(pushDownParameter));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,257 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
|
||||
import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
|
||||
import io.hetu.core.plugin.clickhouse.rewrite.BuildInDirectMapFunctionCallRewriter;
|
||||
import io.hetu.core.plugin.clickhouse.rewrite.ClickHouseUnsupportedFunctionCallRewriter;
|
||||
import io.hetu.core.plugin.clickhouse.rewrite.UdfFunctionRewriteConstants;
|
||||
import io.hetu.core.plugin.clickhouse.rewrite.functioncall.DateParseFunctionCallRewriter;
|
||||
import io.prestosql.configmanager.ConfigSupplier;
|
||||
import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.function.FunctionHandle;
|
||||
import io.prestosql.spi.function.FunctionMetadata;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.OperatorType;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.relation.CallExpression;
|
||||
import io.prestosql.spi.relation.ConstantExpression;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpression;
|
||||
import io.prestosql.spi.relation.RowExpressionService;
|
||||
import io.prestosql.spi.relation.SpecialForm;
|
||||
import io.prestosql.spi.sql.expression.QualifiedName;
|
||||
import io.prestosql.spi.type.Type;
|
||||
import io.prestosql.spi.type.VarcharType;
|
||||
import io.prestosql.sql.builder.functioncall.FunctionWriterManager;
|
||||
import io.prestosql.sql.builder.functioncall.FunctionWriterManagerGroup;
|
||||
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
|
||||
import io.prestosql.sql.builder.functioncall.functions.base.FromBase64CallRewriter;
|
||||
import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static com.google.common.collect.ImmutableSet.toImmutableSet;
|
||||
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
|
||||
import static io.prestosql.sql.builder.functioncall.BaseFunctionUtil.isDefaultFunction;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Locale.ENGLISH;
|
||||
|
||||
public class ClickHouseRowExpressionConverter
|
||||
extends BaseJdbcRowExpressionConverter
|
||||
{
|
||||
private static final Set<String> clickHouseNotSupportFunctions =
|
||||
Stream.of("try", "try_cast", "at_timezone", "current_user", "current_path", "current_time").collect(toImmutableSet());
|
||||
private static FunctionWriterManager clickHouseFunctionManager;
|
||||
private final ClickHouseApplyRemoteFunctionPushDown clickHouseApplyRemoteFunctionPushDown;
|
||||
|
||||
/**
|
||||
* ClickHouse sql query writer
|
||||
*
|
||||
* @param clickHouseConfig config
|
||||
*/
|
||||
public ClickHouseRowExpressionConverter(
|
||||
DeterminismEvaluator determinismEvaluator,
|
||||
RowExpressionService rowExpressionService,
|
||||
FunctionMetadataManager functionManager,
|
||||
StandardFunctionResolution functionResolution,
|
||||
ClickHousePushDownParameter clickHouseConfig,
|
||||
BaseJdbcConfig baseJdbcConfig)
|
||||
{
|
||||
super(functionManager, functionResolution, rowExpressionService, determinismEvaluator);
|
||||
clickHouseFunctionManager = initFunctionManager(clickHouseConfig.getClickHouseConfig());
|
||||
clickHouseApplyRemoteFunctionPushDown = new ClickHouseApplyRemoteFunctionPushDown(baseJdbcConfig, ClickHouseConstants.CONNECTOR_NAME);
|
||||
}
|
||||
|
||||
private FunctionWriterManager initFunctionManager(ClickHouseConfig clickHouseConfig)
|
||||
{
|
||||
ConfigSupplier configSupplier = new DefaultUdfRewriteConfigSupplier(UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS);
|
||||
DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter =
|
||||
new DefaultConnectorConfigFunctionRewriter(ClickHouseConstants.CONNECTOR_NAME, configSupplier);
|
||||
|
||||
return FunctionWriterManagerGroup.newFunctionWriterManagerInstance(ClickHouseConstants.CONNECTOR_NAME,
|
||||
clickHouseConfig.getClickHouseSqlVersion(), getInjectFunctionCallRewritersDefault(clickHouseConfig), connectorConfigFunctionRewriter);
|
||||
}
|
||||
|
||||
private Map<String, FunctionCallRewriter> getInjectFunctionCallRewritersDefault(ClickHouseConfig clickHouseConfig)
|
||||
{
|
||||
// add the user define function re-writer
|
||||
Map<String, FunctionCallRewriter> functionCallRewriters = new HashMap<>(Collections.emptyMap());
|
||||
// 1. the base function re-writers all connector can use
|
||||
FromBase64CallRewriter fromBase64CallRewriter = new FromBase64CallRewriter();
|
||||
functionCallRewriters.put(FromBase64CallRewriter.INNER_FUNC_FROM_BASE64, fromBase64CallRewriter);
|
||||
|
||||
// 2. the specific user define function re-writers
|
||||
FunctionCallRewriter unSupportedFunctionCallRewriter = new ClickHouseUnsupportedFunctionCallRewriter(ClickHouseConstants.CONNECTOR_NAME);
|
||||
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_DAY2SEC, unSupportedFunctionCallRewriter);
|
||||
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH, unSupportedFunctionCallRewriter);
|
||||
functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_TIME_WITH_TZ_LITERAL, unSupportedFunctionCallRewriter);
|
||||
|
||||
FunctionCallRewriter buildInDirectMapFunctionCallRewriter = new BuildInDirectMapFunctionCallRewriter();
|
||||
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUIDLIN_AGGR_FUNC_SUM, buildInDirectMapFunctionCallRewriter);
|
||||
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_AVG, buildInDirectMapFunctionCallRewriter);
|
||||
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_COUNT, buildInDirectMapFunctionCallRewriter);
|
||||
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MAX, buildInDirectMapFunctionCallRewriter);
|
||||
functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MIN, buildInDirectMapFunctionCallRewriter);
|
||||
|
||||
FunctionCallRewriter dateParseFunctionCallRewriter = new DateParseFunctionCallRewriter();
|
||||
functionCallRewriters.put(DateParseFunctionCallRewriter.BUILD_IN_FUNC_DATE_PARSE, dateParseFunctionCallRewriter);
|
||||
|
||||
return functionCallRewriters;
|
||||
}
|
||||
|
||||
protected static String functionCall(QualifiedName name, boolean isDistinct, List<String> argumentsList, Optional<String> orderBy, Optional<String> filter, Optional<String> window)
|
||||
{
|
||||
if (clickHouseFunctionManager == null) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "Function manager is uninitialized");
|
||||
}
|
||||
|
||||
try {
|
||||
return clickHouseFunctionManager.getFunctionRewriteResult(name, isDistinct, argumentsList, orderBy, filter, window);
|
||||
}
|
||||
catch (UnsupportedOperationException e) {
|
||||
throw new PrestoException(NOT_SUPPORTED, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String visitCall(CallExpression call, JdbcConverterContext context)
|
||||
{
|
||||
FunctionHandle functionHandle = call.getFunctionHandle();
|
||||
// remote udf verify
|
||||
if (!isDefaultFunction(call)) {
|
||||
Optional<String> result = clickHouseApplyRemoteFunctionPushDown.rewriteRemoteFunction(call, this, context);
|
||||
if (result.isPresent()) {
|
||||
return result.get();
|
||||
}
|
||||
throw new PrestoException(NOT_SUPPORTED, String.format("ClickHouse connector does not support remote function: %s.%s", call.getDisplayName(), call.getFunctionHandle().getFunctionNamespace()));
|
||||
}
|
||||
|
||||
FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(functionHandle);
|
||||
String functionName = functionMetadata.getName().getObjectName();
|
||||
|
||||
if (clickHouseNotSupportFunctions.contains(functionName)) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support " + functionName);
|
||||
}
|
||||
|
||||
if (standardFunctionResolution.isOperator(functionHandle)) {
|
||||
return handleOperatorFunction(call, functionMetadata, context);
|
||||
}
|
||||
|
||||
List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
|
||||
if (standardFunctionResolution.isNotFunction(functionHandle)) {
|
||||
return format("(NOT %s)", argumentList.get(0));
|
||||
}
|
||||
|
||||
if (standardFunctionResolution.isLikeFunction(functionHandle)) {
|
||||
return format("(%s LIKE %s)", argumentList.get(0), argumentList.get(1));
|
||||
}
|
||||
|
||||
/*
|
||||
* Array needs to be tested
|
||||
*/
|
||||
if (standardFunctionResolution.isArrayConstructor(functionHandle)) {
|
||||
return format("ARRAY(%s)", Joiner.on(", ").join(argumentList));
|
||||
}
|
||||
|
||||
return functionCall(new QualifiedName(Collections.singletonList(functionName)), false, argumentList, Optional.empty(), Optional.empty(), Optional.empty());
|
||||
}
|
||||
|
||||
private String handleOperatorFunction(CallExpression call, FunctionMetadata functionMetadata, JdbcConverterContext context)
|
||||
{
|
||||
Optional<OperatorType> operatorTypeOptional = functionMetadata.getOperatorType();
|
||||
OperatorType type = operatorTypeOptional.get();
|
||||
if (type.equals(OperatorType.CAST)) {
|
||||
return handleCastOperator(call.getArguments().get(0), call.getType(), context);
|
||||
}
|
||||
|
||||
List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
|
||||
if (type.isArithmeticOperator()) {
|
||||
return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
|
||||
}
|
||||
|
||||
if (type.isComparisonOperator()) {
|
||||
final String[] clickHouseCompareOperators = new String[] {"=", ">", "<", ">=", "<=", "!=", "<>"};
|
||||
if (Arrays.asList(clickHouseCompareOperators).contains(type.getOperator())) {
|
||||
return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
|
||||
}
|
||||
else {
|
||||
String exceptionInfo = "ClickHouse Connector does not support comparison operator " + type.getOperator();
|
||||
throw new PrestoException(NOT_SUPPORTED, exceptionInfo);
|
||||
}
|
||||
}
|
||||
|
||||
if (type.equals(OperatorType.SUBSCRIPT)) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "ClickHouse Connector does not support subscript now");
|
||||
}
|
||||
|
||||
/*
|
||||
* "Negative" needs to be tested
|
||||
*/
|
||||
if (call.getArguments().size() == 1 && type.equals(OperatorType.NEGATION)) {
|
||||
String value = argumentList.get(0);
|
||||
String separator = value.startsWith("-") ? " " : "";
|
||||
return format("-%s%s", separator, value);
|
||||
}
|
||||
|
||||
throw new PrestoException(NOT_SUPPORTED, String.format("Unknown operator %s in push down", type.getOperator()));
|
||||
}
|
||||
|
||||
private String handleCastOperator(RowExpression expression, Type dstType, JdbcConverterContext context)
|
||||
{
|
||||
/*
|
||||
* In SqlToRowExpressionTranslator, it will translate GenericLiteral expression to a 'CONSTANT' rowExpression,
|
||||
* so the 'CAST' operator is not needed.
|
||||
* */
|
||||
String value = expression.accept(this, context);
|
||||
if (expression instanceof ConstantExpression && expression.getType() instanceof VarcharType) {
|
||||
//dstType needs to be tested here.
|
||||
//especially DateTime.
|
||||
return value;
|
||||
}
|
||||
|
||||
if (dstType.getDisplayName().equals(LIKE_PATTERN_NAME)) {
|
||||
return value;
|
||||
}
|
||||
|
||||
return format("CAST(%s AS %s)", value, dstType.getDisplayName().toLowerCase(ENGLISH));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String visitSpecialForm(SpecialForm specialForm, JdbcConverterContext context)
|
||||
{
|
||||
if (specialForm.getForm().equals(SpecialForm.Form.DEREFERENCE)
|
||||
|| specialForm.getForm().equals(SpecialForm.Form.ROW_CONSTRUCTOR)
|
||||
|| specialForm.getForm().equals(SpecialForm.Form.BIND)) {
|
||||
throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support" + specialForm.getForm().toString());
|
||||
}
|
||||
|
||||
return super.visitSpecialForm(specialForm, context);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,79 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcSqlStatementWriter;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
|
||||
import io.prestosql.spi.sql.expression.Types;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* It is not clear what is the difference between aggregation and the previous version.
|
||||
*/
|
||||
public class ClickHouseSqlStatementWriter
|
||||
extends BaseJdbcSqlStatementWriter
|
||||
{
|
||||
protected static final Logger log = Logger.get(ClickHouseSqlStatementWriter.class);
|
||||
|
||||
public ClickHouseSqlStatementWriter(JdbcPushDownParameter pushDownParameter)
|
||||
{
|
||||
super(pushDownParameter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
|
||||
{
|
||||
String functionName = inputFunctionName;
|
||||
if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
|
||||
functionName = "varPop";
|
||||
}
|
||||
return super.aggregation(functionName, arguments, isDistinct);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String windowFrame(Types.WindowFrameType type, String start, Optional<String> end)
|
||||
{
|
||||
throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String window(String functionName, List<String> functionArgs, List<String> partitionBy, Optional<String> orderBy, Optional<String> frame)
|
||||
{
|
||||
throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
|
||||
}
|
||||
|
||||
@Override
|
||||
public String from(String selections, String from)
|
||||
{
|
||||
String[] froms = from.split("\\.");
|
||||
if (froms.length == 2) {
|
||||
return selections + " FROM " + from;
|
||||
}
|
||||
else if (froms.length == 3) {
|
||||
StringBuilder sb = new StringBuilder(froms[1]);
|
||||
sb.append(".");
|
||||
sb.append(froms[2]);
|
||||
return selections + " FROM " + sb.toString();
|
||||
}
|
||||
else {
|
||||
log.error("params more than 3 " + from);
|
||||
return selections + " FROM " + from;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.prestosql.spi.function.ExternalFunctionInfo;
|
||||
import io.prestosql.spi.type.StandardTypes;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
public class ClickHouseExternalDateTimeFunctions
|
||||
{
|
||||
public static Set<ExternalFunctionInfo> getFunctionsInfo()
|
||||
{
|
||||
return ImmutableSet.<ExternalFunctionInfo>builder()
|
||||
.add(ClickHouse_toUnixTimeStamp_FUNCTION_INFO)
|
||||
.add(ClickHouse_toDateTime_FUNCTION_INFO)
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Map<String, String> getDateTimeHubFunctionStringMap()
|
||||
{
|
||||
// the click house is case sensitive and presto only support the lower case function name,
|
||||
// we need to handle this function into lower and store its primal name.
|
||||
// now the click house do not contain two different functions have the equal function name ignore case.
|
||||
return ImmutableMap.<String, String>builder()
|
||||
.put(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH), TO_UNIX_TIME_STAMP_PRIMAL)
|
||||
.put(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH), TO_DATE_TIME_PRIMAL)
|
||||
.build();
|
||||
}
|
||||
|
||||
private static final String TO_UNIX_TIME_STAMP_PRIMAL = "toUnixTimestamp";
|
||||
|
||||
private static final String TO_DATE_TIME_PRIMAL = "toDateTime";
|
||||
|
||||
private static final ExternalFunctionInfo ClickHouse_toUnixTimeStamp_FUNCTION_INFO =
|
||||
ExternalFunctionInfo.builder()
|
||||
.functionName(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH))
|
||||
.inputArgs(StandardTypes.VARCHAR)
|
||||
.returnType(StandardTypes.BIGINT)
|
||||
.deterministic(true)
|
||||
.calledOnNullInput(false)
|
||||
.description("converts value to the number with type UInt32 -- Unix Timestamp")
|
||||
.build();
|
||||
|
||||
private static final ExternalFunctionInfo ClickHouse_toDateTime_FUNCTION_INFO =
|
||||
ExternalFunctionInfo.builder()
|
||||
.functionName(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH))
|
||||
.inputArgs(StandardTypes.VARCHAR)
|
||||
.returnType(StandardTypes.TIMESTAMP)
|
||||
.deterministic(true)
|
||||
.calledOnNullInput(false)
|
||||
.description("converts value to the number with type UInt32 -- Unix Timestamp")
|
||||
.build();
|
||||
|
||||
private ClickHouseExternalDateTimeFunctions()
|
||||
{
|
||||
}
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.spi.connector.CatalogSchemaName;
|
||||
import io.prestosql.spi.function.ExternalFunctionInfo;
|
||||
import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public class ClickHouseExternalFunctionHub
|
||||
extends JdbcExternalFunctionHub
|
||||
{
|
||||
private final BaseJdbcConfig jdbcConfig;
|
||||
|
||||
@Inject
|
||||
public ClickHouseExternalFunctionHub(BaseJdbcConfig jdbcConfig)
|
||||
{
|
||||
this.jdbcConfig = requireNonNull(jdbcConfig, "jdbcConfig is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<CatalogSchemaName> getExternalFunctionCatalogSchemaName()
|
||||
{
|
||||
return jdbcConfig.getConnectorRegistryFunctionNamespace();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<ExternalFunctionInfo> getExternalFunctions()
|
||||
{
|
||||
return ImmutableSet.<ExternalFunctionInfo>builder()
|
||||
.addAll(ClickHouseExternalDateTimeFunctions.getFunctionsInfo())
|
||||
.build();
|
||||
}
|
||||
|
||||
public static Map<String, String> getPrimalFunctionNameMap()
|
||||
{
|
||||
// The click house is case sensitive and presto only support the lower case function name,
|
||||
// we need to handle this function into lower and store its primal name.
|
||||
// The click house do not contain two different functions have the equal function name ignore case.
|
||||
// If a data base contain two different functions have the equal function name ignore case,
|
||||
// then we should register them into different function name space
|
||||
return ImmutableMap.<String, String>builder()
|
||||
.putAll(ClickHouseExternalDateTimeFunctions.getDateTimeHubFunctionStringMap())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.rewrite;
|
||||
|
||||
import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
|
||||
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
|
||||
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
|
||||
|
||||
public class BuildInDirectMapFunctionCallRewriter
|
||||
implements FunctionCallRewriter
|
||||
{
|
||||
// ========= for Aggregate Functions =========
|
||||
|
||||
/**
|
||||
* functioncall name of build-in direct map aggregate function max
|
||||
*/
|
||||
public static final String BUILDIN_AGGR_FUNC_MAX = "max";
|
||||
|
||||
/**
|
||||
* functioncall name of build-in direct min aggregate function min
|
||||
*/
|
||||
public static final String BUILDIN_AGGR_FUNC_MIN = "min";
|
||||
|
||||
/**
|
||||
* functioncall name of build-in direct map aggregate function count
|
||||
*/
|
||||
public static final String BUILDIN_AGGR_FUNC_COUNT = "count";
|
||||
|
||||
/**
|
||||
* functioncall name of build-in direct map aggregate function avg
|
||||
*/
|
||||
public static final String BUILDIN_AGGR_FUNC_AVG = "avg";
|
||||
|
||||
/**
|
||||
* functioncall name of build-in direct map aggregate function sum
|
||||
*/
|
||||
public static final String BUIDLIN_AGGR_FUNC_SUM = "sum";
|
||||
|
||||
@Override
|
||||
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
|
||||
{
|
||||
StringBuilder builder = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY);
|
||||
|
||||
String arguments = RewriteUtil.joinExpressions(functionCallArgsPackage.getArgumentsList());
|
||||
if (functionCallArgsPackage.getArgumentsList().isEmpty() && "count".equalsIgnoreCase(functionCallArgsPackage.getName().getSuffix())) {
|
||||
arguments = "*";
|
||||
}
|
||||
if (functionCallArgsPackage.getIsDistinct()) {
|
||||
arguments = "DISTINCT " + arguments;
|
||||
}
|
||||
|
||||
builder.append(RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName())).append('(').append(arguments);
|
||||
functionCallArgsPackage.getOrderBy().ifPresent(exp -> builder.append(' ').append(exp));
|
||||
builder.append(')');
|
||||
functionCallArgsPackage.getFilter().ifPresent(exp -> builder.append(" FILTER ").append(exp));
|
||||
functionCallArgsPackage.getWindow().ifPresent(exp -> builder.append(" OVER ").append(exp));
|
||||
|
||||
return builder.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.rewrite;
|
||||
|
||||
import io.prestosql.spi.type.StandardTypes;
|
||||
import io.prestosql.sql.builder.functioncall.functions.base.UnsupportedFunctionCallRewriter;
|
||||
|
||||
import static io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil.LITERAL_FUNCNAME_PREFIX;
|
||||
|
||||
public class ClickHouseUnsupportedFunctionCallRewriter
|
||||
extends UnsupportedFunctionCallRewriter
|
||||
{
|
||||
/**
|
||||
* functioncall name of INTERVAL_DAY_TO_SECOND literal in HeTu inner
|
||||
*/
|
||||
public static final String INNER_FUNC_INTERVAL_LITERAL_DAY2SEC =
|
||||
LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_DAY_TO_SECOND;
|
||||
|
||||
/**
|
||||
* functioncall name of INTERVAL_YEAR_TO_MONTH literal in HeTu inner
|
||||
*/
|
||||
public static final String INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH =
|
||||
LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_YEAR_TO_MONTH;
|
||||
|
||||
/**
|
||||
* functioncall name of TIME_WITH_TIME_ZONE literal in HeTu inner
|
||||
*/
|
||||
public static final String INNER_FUNC_TIME_WITH_TZ_LITERAL =
|
||||
LITERAL_FUNCNAME_PREFIX + StandardTypes.TIME_WITH_TIME_ZONE;
|
||||
|
||||
/**
|
||||
* the constructor of Unsupported Function Call Re-writer
|
||||
*
|
||||
* @param connectorName connector's name
|
||||
*/
|
||||
public ClickHouseUnsupportedFunctionCallRewriter(String connectorName)
|
||||
{
|
||||
super(connectorName);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.rewrite;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import io.prestosql.spi.sql.expression.QualifiedName;
|
||||
import io.prestosql.spi.sql.expression.Selection;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static java.util.stream.Collectors.joining;
|
||||
|
||||
/**
|
||||
* util helper class for rewrite handle.
|
||||
*/
|
||||
public class RewriteUtil
|
||||
{
|
||||
private RewriteUtil()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* interal function prefix name
|
||||
*/
|
||||
public static final String LITERAL_FUNCNAME_PREFIX = "$literal$";
|
||||
|
||||
/**
|
||||
* expression list
|
||||
*
|
||||
* @param expressions expression lists
|
||||
* @return sql statement
|
||||
*/
|
||||
public static final String joinExpressions(List<String> expressions)
|
||||
{
|
||||
return Joiner.on(", ").join(expressions);
|
||||
}
|
||||
|
||||
/**
|
||||
* formate identifier
|
||||
*
|
||||
* @param qualifiedNames qualified names
|
||||
* @param identifier identifier
|
||||
* @return sql statement
|
||||
*/
|
||||
public static final String formatIdentifier(Optional<Map<String, Selection>> qualifiedNames, String identifier)
|
||||
{
|
||||
if (qualifiedNames.isPresent()) {
|
||||
return qualifiedNames.get().get(identifier).getExpression();
|
||||
}
|
||||
return identifier;
|
||||
}
|
||||
|
||||
/**
|
||||
* formate qualified name
|
||||
*
|
||||
* @param name qualified name
|
||||
* @return sql statement
|
||||
*/
|
||||
public static final String formatQualifiedName(QualifiedName name)
|
||||
{
|
||||
return name.getParts().stream().map(identifier -> formatIdentifier(Optional.empty(), identifier)).collect(joining("."));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,98 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.rewrite;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class UdfFunctionRewriteConstants
|
||||
{
|
||||
private UdfFunctionRewriteConstants()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* udf rewrite pattern map
|
||||
*/
|
||||
public static final Map<String, String> DEFAULT_VERSION_UDF_REWRITE_PATTERNS =
|
||||
new ImmutableMap.Builder<String, String>()
|
||||
// Statistical aggregate functions
|
||||
.put("CORR($1,$2)", "CORR($1, $2)")
|
||||
.put("STDDEV($1)", "stddevSamp($1)")
|
||||
.put("stddev_pop($1)", "stddev_pop($1)")
|
||||
.put("stddev_samp($1)", "stddevSamp($1)")
|
||||
.put("skewness($1)", "skewPop($1)")
|
||||
.put("kurtosis($1)", "kurtPop($1)")
|
||||
.put("VARIANCE($1)", "varSamp($1)")
|
||||
.put("var_samp($1)", "varSamp($1)")
|
||||
.put("APPROX_DISTINCT($1)", "uniq($1)")
|
||||
.put("APPROX_DISTINCT($1,$2)", "uniq($1)")
|
||||
// math functions
|
||||
.put("ABS($1)", "ABS($1)")
|
||||
.put("ACOS($1)", "ACOS($1)")
|
||||
.put("ASIN($1)", "ASIN($1)")
|
||||
.put("ATAN($1)", "ATAN($1)")
|
||||
.put("ATAN2($1,$2)", "ATAN2($1, $2)")
|
||||
.put("CEIL($1)", "CEIL($1)")
|
||||
.put("CEILING($1)", "CEIL($1)")
|
||||
.put("COS($1)", "COS($1)")
|
||||
.put("e()", "e()")
|
||||
.put("EXP($1)", "EXP($1)")
|
||||
.put("FLOOR($1)", "FLOOR($1)")
|
||||
.put("LN($1)", "LN($1)")
|
||||
.put("LOG10($1)", "log10($1)")
|
||||
.put("LOG2($1)", "log2($1)")
|
||||
.put("MOD($1,$2)", "MOD($1, $2)")
|
||||
.put("pi()", "pi()")
|
||||
.put("POW($1,$2)", "POW($1, $2)")
|
||||
.put("POWER($1,$2)", "POWER($1, $2)")
|
||||
.put("RAND()", "RAND()")
|
||||
.put("RANDOM()", "RAND()")
|
||||
.put("ROUND($1)", "ROUND($1)")
|
||||
.put("ROUND($1,$2)", "ROUND($1, $2)")
|
||||
.put("SIGN($1)", "SIGN($1)")
|
||||
.put("SIN($1)", "SIN($1)")
|
||||
.put("SQRT($1)", "SQRT($1)")
|
||||
.put("TAN($1)", "TAN($1)")
|
||||
//character functions
|
||||
.put("CONCAT($1,$2)", "CONCAT($1, $2)")
|
||||
.put("LENGTH($1)", "LENGTH($1)")
|
||||
.put("LOWER($1)", "LOWER($1)")
|
||||
.put("LTRIM($1)", "trimLeft($1)")
|
||||
.put("REPLACE($1,$2)", "replaceAll($1, $2, '')")
|
||||
.put("REPLACE($1,$2,$3)", "replaceAll($1, $2, $3)")
|
||||
.put("RTRIM($1)", "trimRight($1)")
|
||||
.put("STRPOS($1,$2)", "position($1, $2)")
|
||||
.put("SUBSTR($1,$2,$3)", "SUBSTR($1, $2, $3)")
|
||||
.put("POSITION($1,$2)", "position($2, $1)")
|
||||
.put("TRIM($1)", "trimBoth($1)")
|
||||
.put("UPPER($1)", "UPPER($1)")
|
||||
//date functions
|
||||
.put("YEAR($1)", "toYear($1)")
|
||||
.put("QUARTER($1)", "toQuarter($1)")
|
||||
.put("MONTH($1)", "toMonth($1)")
|
||||
.put("WEEK($1)", "toWeek($1)")
|
||||
.put("YEAR_OF_WEEK($1)", "toISOYear($1)")
|
||||
.put("DAY($1)", "toDayOfMonth($1)")
|
||||
.put("HOUR($1)", "toHour($1)")
|
||||
.put("MINUTE($1)", "toMinute($1)")
|
||||
.put("SECOND($1)", "toSecond($1)")
|
||||
.put("DAY_OF_WEEK($1)", "toDayOfWeek($1)")
|
||||
.put("DAY_OF_MONTH($1)", "toDayOfMonth($1)")
|
||||
.put("DAY_OF_YEAR($1)", "toDayOfYear($1)")
|
||||
.put("TO_UNIXTIME($1)", "toUnixTimestamp($1)")
|
||||
.build();
|
||||
}
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse.rewrite.functioncall;
|
||||
|
||||
import io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil;
|
||||
import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
|
||||
import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Clickhouse date time type conversion functions.
|
||||
* The rewrite of this function limits date_parse.
|
||||
*/
|
||||
public class DateParseFunctionCallRewriter
|
||||
implements FunctionCallRewriter
|
||||
{
|
||||
public static final String BUILD_IN_FUNC_DATE_PARSE = "date_parse";
|
||||
|
||||
@Override
|
||||
public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
|
||||
{
|
||||
String functionName = RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName());
|
||||
if (!functionName.toLowerCase(Locale.ENGLISH).equals("date_parse") || functionCallArgsPackage.getArgumentsList().size() != 2) {
|
||||
throw new UnsupportedOperationException("ClickHouse Connector does not support function call of " + RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName()));
|
||||
}
|
||||
List<String> argsList = functionCallArgsPackage.getArgumentsList();
|
||||
String args0 = argsList.get(0);
|
||||
String args1 = argsList.get(1);
|
||||
if (args1.equals("'%Y-%m-%d %H:%i:%s'")) {
|
||||
return "toDateTime(" + args0 + ")";
|
||||
}
|
||||
else if (args1.equals("'%Y-%m-%d'")) {
|
||||
return "toDate(" + args0 + ")";
|
||||
}
|
||||
return String.format("parseDateTimeBestEffort(%s)", args0);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,355 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.airlift.log.Logger;
|
||||
import io.prestosql.plugin.jdbc.JdbcColumnHandle;
|
||||
import io.prestosql.plugin.jdbc.JdbcIdentity;
|
||||
import io.prestosql.plugin.jdbc.JdbcTableHandle;
|
||||
import io.prestosql.spi.connector.ColumnMetadata;
|
||||
import io.prestosql.spi.connector.ConnectorSession;
|
||||
import io.prestosql.spi.connector.ConnectorTableMetadata;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.type.IntegerType;
|
||||
import io.prestosql.spi.type.Type;
|
||||
import org.testng.SkipException;
|
||||
import org.testng.annotations.AfterClass;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.sql.Connection;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Optional;
|
||||
|
||||
import static io.prestosql.spi.type.VarcharType.VARCHAR;
|
||||
import static io.prestosql.testing.TestingSession.testSessionBuilder;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* ClickHouseTest
|
||||
*/
|
||||
public class ClickHouseClientTest
|
||||
{
|
||||
private static final ConnectorSession SESSION = testSessionBuilder().build().toConnectorSession();
|
||||
|
||||
private static final Logger LOGGER = Logger.get(ClickHouseClientTest.class);
|
||||
|
||||
private DataBaseTest database;
|
||||
|
||||
private String catalogName;
|
||||
|
||||
private ClickHouseClient clickHouseClient;
|
||||
|
||||
private Connection connection;
|
||||
|
||||
private ClickHouseServerTest clickHouseServer;
|
||||
|
||||
private ClickHouseClientTest()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* setUp
|
||||
*
|
||||
* @throws SQLException SQLException
|
||||
*/
|
||||
@BeforeClass
|
||||
public void setUp()
|
||||
throws SQLException
|
||||
{
|
||||
this.clickHouseServer = ClickHouseServerTest.getInstance();
|
||||
if (!clickHouseServer.isClickHouseServerAvailable()) {
|
||||
LOGGER.info("please set correct clickhouse data base info!");
|
||||
throw new SkipException("skip the test");
|
||||
}
|
||||
LOGGER.info("running TestClickHouseClient...");
|
||||
database = new DataBaseTest(clickHouseServer);
|
||||
connection = database.getConnection();
|
||||
catalogName = connection.getCatalog();
|
||||
clickHouseClient = database.getClickHouseClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* tearDown
|
||||
*
|
||||
* @throws SQLException SQLException
|
||||
*/
|
||||
@AfterClass(alwaysRun = true)
|
||||
public void tearDown()
|
||||
throws SQLException
|
||||
{
|
||||
if (this.clickHouseServer.isClickHouseServerAvailable()) {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* testCreateTable
|
||||
*/
|
||||
@Test(enabled = false)
|
||||
public void testCreateTable()
|
||||
throws SQLException
|
||||
{
|
||||
String tableName = "testCreateTable";
|
||||
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
|
||||
JdbcIdentity identity = JdbcIdentity.from(SESSION);
|
||||
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
|
||||
|
||||
Type intType = IntegerType.INTEGER;
|
||||
ColumnMetadata columnMetadata = new ColumnMetadata("col1", intType);
|
||||
List<ColumnMetadata> columns = new ArrayList<>();
|
||||
columns.add(columnMetadata);
|
||||
|
||||
ConnectorTableMetadata connectorTableMetadata = new ConnectorTableMetadata(schemaTableName, columns);
|
||||
clickHouseClient.createTable(SESSION, connectorTableMetadata, tableName);
|
||||
|
||||
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
|
||||
clickHouseClient.dropTable(identity, tableHandle);
|
||||
}
|
||||
|
||||
/**
|
||||
* testListSchemas
|
||||
*/
|
||||
@Test
|
||||
public void testListSchemas()
|
||||
{
|
||||
assertEquals(ImmutableSet.copyOf(clickHouseClient.listSchemas(connection)).contains("test"), true);
|
||||
}
|
||||
|
||||
/**
|
||||
* testGetTables
|
||||
*/
|
||||
@Test
|
||||
public void testGetTables()
|
||||
{
|
||||
List<String> expectedTables = database.getTables();
|
||||
List<String> allTables = getAllTables(getNameByUpperCaseIdentifiers(database.getSchema()));
|
||||
List<String> actualTables = getActualTables(allTables, expectedTables);
|
||||
assertEquals(actualTables, expectedTables);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRenameTable()
|
||||
{
|
||||
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
|
||||
|
||||
List<String> expectedTables = database.getTables();
|
||||
List<String> allTables = getAllTables(schemaName);
|
||||
List<String> actualTables = getActualTables(allTables, expectedTables);
|
||||
assertEquals(actualTables, expectedTables);
|
||||
|
||||
String actualTable = database.getActualTable("number"); // get actual table number_*
|
||||
String tableNumber = getNameByUpperCaseIdentifiers(actualTable);
|
||||
String tableNewNumber = getNameByUpperCaseIdentifiers(actualTable.replace("number", "new_number"));
|
||||
|
||||
/* rename table number_* to new_number_* */
|
||||
JdbcIdentity identity = JdbcIdentity.from(SESSION);
|
||||
SchemaTableName newTableName = new SchemaTableName(schemaName, tableNewNumber);
|
||||
clickHouseClient.renameTable(identity, null, schemaName, tableNumber, newTableName);
|
||||
|
||||
expectedTables.remove(3); //remove table "number_*"
|
||||
expectedTables.add(actualTable.replace("number", "new_number")); //add table "new_number_*"
|
||||
allTables = getAllTables(schemaName);
|
||||
actualTables = getActualTables(allTables, expectedTables);
|
||||
assertEquals(actualTables, expectedTables);
|
||||
|
||||
/* rename table new_number_* to number_* */
|
||||
newTableName = new SchemaTableName(schemaName, tableNumber);
|
||||
clickHouseClient.renameTable(identity, null, schemaName, tableNewNumber, newTableName);
|
||||
|
||||
expectedTables.remove(3); //remove table "new_number"
|
||||
expectedTables.add(actualTable); //add table "number"
|
||||
allTables = getAllTables(schemaName);
|
||||
actualTables = getActualTables(allTables, expectedTables);
|
||||
assertEquals(actualTables, expectedTables);
|
||||
}
|
||||
|
||||
/**
|
||||
* testRenameColumn
|
||||
*/
|
||||
@Test
|
||||
public void testRenameColumn()
|
||||
throws NoSuchFieldException, IllegalAccessException
|
||||
{
|
||||
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
|
||||
// get actual table table_with_float_col_*
|
||||
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("table_with_float_col"));
|
||||
|
||||
List<String> expectedColumns = Arrays.asList("col1", "col2", "col3", "col4");
|
||||
List<String> actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
|
||||
JdbcIdentity identity = JdbcIdentity.from(SESSION);
|
||||
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
|
||||
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
|
||||
|
||||
List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
|
||||
JdbcColumnHandle columnHandle = null;
|
||||
for (JdbcColumnHandle column : columns) {
|
||||
if ("col4".equalsIgnoreCase(column.getColumnName())) {
|
||||
columnHandle = column;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
|
||||
field.setAccessible(true);
|
||||
field.set(tableHandle, null);
|
||||
|
||||
clickHouseClient.renameColumn(identity, tableHandle, columnHandle, "newcol4");
|
||||
|
||||
expectedColumns = Arrays.asList("col1", "col2", "col3", "newcol4");
|
||||
actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
}
|
||||
|
||||
/**
|
||||
* testAddColumn
|
||||
*/
|
||||
@Test
|
||||
public void testAddColumn()
|
||||
throws NoSuchFieldException, IllegalAccessException
|
||||
{
|
||||
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
|
||||
// get actual table student_*
|
||||
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("student"));
|
||||
|
||||
List<String> expectedColumns = Arrays.asList("id");
|
||||
List<String> actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
|
||||
JdbcIdentity identity = JdbcIdentity.from(SESSION);
|
||||
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
|
||||
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
|
||||
ColumnMetadata columnMetadata = new ColumnMetadata("name", VARCHAR);
|
||||
|
||||
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
|
||||
field.setAccessible(true);
|
||||
field.set(tableHandle, null);
|
||||
clickHouseClient.addColumn(SESSION, tableHandle, columnMetadata);
|
||||
|
||||
expectedColumns = Arrays.asList("id", "name");
|
||||
actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
}
|
||||
|
||||
/**
|
||||
* testDropColumn
|
||||
*/
|
||||
@Test
|
||||
public void testDropColumn()
|
||||
throws NoSuchFieldException, IllegalAccessException
|
||||
{
|
||||
String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
|
||||
String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("example")); // get actual table example_*
|
||||
|
||||
List<String> expectedColumns = Arrays.asList("text", "text_short", "value");
|
||||
List<String> actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
|
||||
JdbcIdentity identity = JdbcIdentity.from(SESSION);
|
||||
SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
|
||||
JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
|
||||
List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
|
||||
|
||||
JdbcColumnHandle columnHandle = null;
|
||||
for (JdbcColumnHandle column : columns) {
|
||||
if ("value".equalsIgnoreCase(column.getColumnName())) {
|
||||
columnHandle = column;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
|
||||
field.setAccessible(true);
|
||||
field.set(tableHandle, null);
|
||||
clickHouseClient.dropColumn(identity, tableHandle, columnHandle);
|
||||
|
||||
expectedColumns = Arrays.asList("text", "text_short");
|
||||
actualColumns = getActualColumns(schemaName, tableName);
|
||||
assertEquals(actualColumns, expectedColumns);
|
||||
}
|
||||
|
||||
@Test(enabled = false)
|
||||
public void testGetColumns()
|
||||
{
|
||||
}
|
||||
|
||||
private String getNameByUpperCaseIdentifiers(String name)
|
||||
{
|
||||
try {
|
||||
if (connection.getMetaData().storesUpperCaseIdentifiers()) {
|
||||
return name.toUpperCase(Locale.ENGLISH);
|
||||
}
|
||||
else {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to get metadata or storesUpperCaseIdentifiers", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> getAllTables(String schemaName)
|
||||
{
|
||||
List<String> allTables = new ArrayList<>();
|
||||
|
||||
try (ResultSet resultSet = clickHouseClient.getTables(connection, Optional.of(schemaName), Optional.empty())) {
|
||||
while (resultSet.next()) {
|
||||
String tableName = resultSet.getString("TABLE_NAME");
|
||||
allTables.add(tableName.toLowerCase(Locale.ENGLISH));
|
||||
}
|
||||
|
||||
return allTables;
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to get tables", e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> getActualTables(List<String> allTables, List<String> expectedTables)
|
||||
{
|
||||
List<String> actualTables = new ArrayList<>();
|
||||
for (String table : expectedTables) {
|
||||
if (allTables.contains(table)) {
|
||||
actualTables.add(table);
|
||||
}
|
||||
}
|
||||
return actualTables;
|
||||
}
|
||||
|
||||
private List<String> getActualColumns(String schemaName, String tableName)
|
||||
{
|
||||
try (ResultSet resultSet = connection.getMetaData().getColumns(null, schemaName, tableName, null)) {
|
||||
List<String> actualColumns = new ArrayList<>();
|
||||
while (resultSet.next()) {
|
||||
String columnName = resultSet.getString("COLUMN_NAME");
|
||||
actualColumns.add(columnName.toLowerCase(Locale.ENGLISH));
|
||||
}
|
||||
|
||||
return actualColumns;
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to get metadata or columns", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* This is testing ClickHouseConfig functions
|
||||
*/
|
||||
public class ClickHouseConfigTest
|
||||
{
|
||||
private static final int COMMUNICATION_TIMEOUT = 10;
|
||||
|
||||
private static final int PACKET_SIZE = 150000;
|
||||
|
||||
/**
|
||||
* This is testing ClickHousePropertyMappings
|
||||
*/
|
||||
@Test
|
||||
public void testClickHousePropertyMappings()
|
||||
{
|
||||
final String tableTypes = "TABLE,VIEW,USER DEFINED,SYNONYM,OLAP VIEW,JOIN VIEW,HIERARCHY VIEW" + ",CALC VIEW,SYSTEM TABLE,NO LOGGING TEMPORARY,GLOBAL TEMPORARY";
|
||||
final String schemaPattern = "default";
|
||||
|
||||
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
|
||||
.put("clickhouse.table-types", tableTypes)
|
||||
.put("clickhouse.schema-pattern", schemaPattern)
|
||||
.put("clickhouse.query.pushdown.enabled", "false")
|
||||
.put("clickhouse.socket_timeout", "100")
|
||||
.build();
|
||||
|
||||
ClickHouseConfig expected = new ClickHouseConfig()
|
||||
.setTableTypes(tableTypes)
|
||||
.setSchemaPattern(schemaPattern)
|
||||
.setQueryPushDownEnabled(false)
|
||||
.setSocketTimeout(100);
|
||||
assertFullMapping(properties, expected);
|
||||
}
|
||||
|
||||
/**
|
||||
* This is testing ClickHouseConfig class inner functions
|
||||
*/
|
||||
@Test
|
||||
public void testGetFuncions()
|
||||
{
|
||||
ClickHouseConfig config = new ClickHouseConfig();
|
||||
|
||||
String tableTypes = config.getTableTypes();
|
||||
assertEquals(tableTypes, ClickHouseConstants.DEFAULT_TABLE_TYPES);
|
||||
|
||||
String schemaPattern = config.getSchemaPattern();
|
||||
assertEquals(schemaPattern, null);
|
||||
|
||||
boolean isQueryPushDown = config.isQueryPushDownEnabled();
|
||||
assertEquals(isQueryPushDown, true);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
public class ClickHouseConstantsTest
|
||||
{
|
||||
private ClickHouseConstantsTest()
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* "src/test/java/io/hetu/core/plugin/clickhouse/test-clickhouse-dev.properties"
|
||||
*/
|
||||
public static final String TEST_PROPERTY_FILE_PATH = "src/test/java/io/hetu/plugin/clickhouse/test-clickhouse-dev.properties";
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.prestosql.spi.Plugin;
|
||||
import io.prestosql.spi.connector.ConnectorFactory;
|
||||
import io.prestosql.testing.TestingConnectorContext;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
|
||||
public class ClickHousePluginTest
|
||||
{
|
||||
@Test
|
||||
public void testCreateConnector()
|
||||
{
|
||||
Plugin plugin = new ClickHousePlugin();
|
||||
ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories());
|
||||
factory.create("test", ImmutableMap.of("connection-url", "jdbc:clickhouse://test"), new TestingConnectorContext());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,221 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
import io.airlift.tpch.TpchTable;
|
||||
|
||||
import javax.annotation.concurrent.GuardedBy;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
|
||||
|
||||
public final class ClickHouseServerTest
|
||||
{
|
||||
private static final Logger LOG = Logger.get(ClickHouseConstantsTest.class);
|
||||
|
||||
@GuardedBy("this")
|
||||
private static int referenceCount;
|
||||
|
||||
@GuardedBy("this")
|
||||
private static ClickHouseServerTest instance;
|
||||
|
||||
private final AtomicBoolean tpchLoaded = new AtomicBoolean(false);
|
||||
private final CountDownLatch tpchLoadComplete = new CountDownLatch(1);
|
||||
private static List<String> actualTables = new ArrayList<>();
|
||||
private final AtomicBoolean isServerUsable = new AtomicBoolean(false);
|
||||
private final AtomicInteger atomicIntegerTimes = new AtomicInteger(0);
|
||||
private String jdbcUrl;
|
||||
private String user;
|
||||
private String password;
|
||||
private String schema;
|
||||
|
||||
public static synchronized ClickHouseServerTest getInstance()
|
||||
{
|
||||
if (referenceCount == 0) {
|
||||
instance = new ClickHouseServerTest();
|
||||
instance.startup();
|
||||
}
|
||||
referenceCount++;
|
||||
return instance;
|
||||
}
|
||||
|
||||
public static synchronized void shutDown()
|
||||
throws SQLException
|
||||
{
|
||||
referenceCount--;
|
||||
if (referenceCount == 0) {
|
||||
instance.shutdown();
|
||||
instance = null;
|
||||
}
|
||||
}
|
||||
|
||||
private void startup()
|
||||
{
|
||||
for (TpchTable<?> table : TpchTable.getTables()) {
|
||||
String newTable = generateNewTableName(table.getTableName());
|
||||
actualTables.add(newTable);
|
||||
}
|
||||
}
|
||||
|
||||
private void shutdown()
|
||||
throws SQLException
|
||||
{
|
||||
if (isClickHouseServerAvailable() && tpchLoaded.get()) {
|
||||
for (String table : actualTables) {
|
||||
String sql = "DROP TABLE " + schema + "." + table;
|
||||
executeInClickHouse(sql);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void executeInClickHouse(String sql)
|
||||
throws SQLException
|
||||
{
|
||||
try (Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
|
||||
Statement statement = connection.createStatement()) {
|
||||
statement.execute(sql);
|
||||
}
|
||||
}
|
||||
|
||||
private ClickHouseServerTest()
|
||||
{
|
||||
File file = new File(ClickHouseConstantsTest.TEST_PROPERTY_FILE_PATH);
|
||||
try {
|
||||
Map<String, String> properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
|
||||
LOG.info("test-clickhouse properties: %s", properties);
|
||||
|
||||
String connectionUrl = properties.get("connection.url");
|
||||
String connectionUser = properties.get("connection.user");
|
||||
String connectionPass = properties.get("connection.password");
|
||||
String connectionSchema = properties.get("connection.schema");
|
||||
|
||||
if (connectionUrl != null) {
|
||||
this.jdbcUrl = connectionUrl;
|
||||
}
|
||||
|
||||
if (connectionUser != null) {
|
||||
this.user = connectionUser;
|
||||
}
|
||||
|
||||
if (connectionPass != null) {
|
||||
this.password = connectionPass;
|
||||
}
|
||||
|
||||
if (connectionSchema != null) {
|
||||
this.schema = connectionSchema;
|
||||
}
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.warn("Failed to load properties for file %s", file);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isClickHouseServerAvailable()
|
||||
{
|
||||
if (atomicIntegerTimes.getAndIncrement() > 0) {
|
||||
return isServerUsable.get();
|
||||
}
|
||||
|
||||
try {
|
||||
Connection connection = DriverManager.getConnection(this.jdbcUrl, this.user, this.password);
|
||||
isServerUsable.set(true);
|
||||
return isServerUsable.get();
|
||||
}
|
||||
catch (SQLException e) {
|
||||
isServerUsable.set(false);
|
||||
return isServerUsable.get();
|
||||
}
|
||||
}
|
||||
|
||||
public String getJdbcUrl()
|
||||
{
|
||||
return jdbcUrl;
|
||||
}
|
||||
|
||||
public String getUser()
|
||||
{
|
||||
return user;
|
||||
}
|
||||
|
||||
public String getPassword()
|
||||
{
|
||||
return password;
|
||||
}
|
||||
|
||||
public String getSchema()
|
||||
{
|
||||
return schema;
|
||||
}
|
||||
|
||||
public boolean isTpchLoaded()
|
||||
{
|
||||
return tpchLoaded.getAndSet(true);
|
||||
}
|
||||
|
||||
public void setTpchLoaded()
|
||||
{
|
||||
tpchLoadComplete.countDown();
|
||||
}
|
||||
|
||||
public void waitTpchLoaded()
|
||||
throws InterruptedException
|
||||
{
|
||||
tpchLoadComplete.await(2, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
public static String generateNewTableName(String tableName)
|
||||
{
|
||||
return tableName + "_" + UUID.randomUUID().toString().replace("-", "");
|
||||
}
|
||||
|
||||
public static String getActualTable(String tablePattern)
|
||||
{
|
||||
return getActualTable(actualTables, tablePattern);
|
||||
}
|
||||
|
||||
public static String getActualTable(List<String> tables, String tablePattern)
|
||||
{
|
||||
String actualTable = tablePattern;
|
||||
|
||||
for (String table : tables) {
|
||||
int lastIndex = table.lastIndexOf("_");
|
||||
if (lastIndex == -1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (table.substring(0, lastIndex).equalsIgnoreCase(tablePattern)) {
|
||||
actualTable = tablePattern + table.substring(lastIndex);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return actualTable;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,145 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.plugin.clickhouse;
|
||||
|
||||
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
|
||||
import io.prestosql.plugin.jdbc.ConnectionFactory;
|
||||
import io.prestosql.plugin.jdbc.DriverConnectionFactory;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
|
||||
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.Driver;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
|
||||
import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
|
||||
|
||||
public class DataBaseTest
|
||||
implements AutoCloseable
|
||||
{
|
||||
private final Connection connection;
|
||||
|
||||
private final ClickHouseClient clickHouseClient;
|
||||
|
||||
private ClickHouseServerTest clickHouseServerTest;
|
||||
|
||||
private List<String> tables = new ArrayList<>();
|
||||
|
||||
/**
|
||||
* constructor
|
||||
*/
|
||||
public DataBaseTest(ClickHouseServerTest clickHouseServer)
|
||||
throws SQLException
|
||||
{
|
||||
this.clickHouseServerTest = clickHouseServer;
|
||||
|
||||
BaseJdbcConfig jdbcConfig = new BaseJdbcConfig();
|
||||
ClickHouseConfig clickHouseConfig = new ClickHouseConfig();
|
||||
|
||||
jdbcConfig.setConnectionUrl(clickHouseServer.getJdbcUrl());
|
||||
jdbcConfig.setConnectionUser(clickHouseServer.getUser());
|
||||
jdbcConfig.setConnectionPassword(clickHouseServer.getPassword());
|
||||
|
||||
clickHouseConfig.setTableTypes("TABLE,VIEW");
|
||||
clickHouseConfig.setSchemaPattern(clickHouseServer.getSchema());
|
||||
clickHouseConfig.setQueryPushDownEnabled(false);
|
||||
|
||||
Driver driver = null;
|
||||
try {
|
||||
driver = (Driver) Class.forName(ClickHouseConstants.CLICKHOUSE_JDBC_DRIVER_CLASS_NAME).getConstructor(((Class<?>[]) null)).newInstance();
|
||||
}
|
||||
catch (InstantiationException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
catch (IllegalAccessException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
catch (ClassNotFoundException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
catch (InvocationTargetException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
catch (NoSuchMethodException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
ConnectionFactory connectionFactory = new DriverConnectionFactory(driver, jdbcConfig.getConnectionUrl(), Optional.ofNullable(jdbcConfig.getUserCredentialName()), Optional.ofNullable(jdbcConfig.getPasswordCredentialName()), basicConnectionProperties(jdbcConfig));
|
||||
|
||||
clickHouseClient = new ClickHouseClient(jdbcConfig, clickHouseConfig, connectionFactory, new JdbcExternalFunctionHub());
|
||||
connection =
|
||||
DriverManager.getConnection(clickHouseServer.getJdbcUrl(), clickHouseServer.getUser(), clickHouseServer.getPassword());
|
||||
|
||||
connection.createStatement()
|
||||
.execute(buildCreateTableSql("example", "(text varchar,text_short varchar(32), value bigint)"));
|
||||
connection.createStatement().execute(buildCreateTableSql("student", "(id varchar)"));
|
||||
connection.createStatement()
|
||||
.execute(buildCreateTableSql("table_with_float_col", "(col1 bigint, col2 double, col3 float, col4 real)"));
|
||||
connection.createStatement()
|
||||
.execute(buildCreateTableSql("number", "(text varchar, text_short varchar(32), value bigint)"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
throws SQLException
|
||||
{
|
||||
for (String table : tables) {
|
||||
String sql = "DROP TABLE " + clickHouseServerTest.getSchema() + "." + table;
|
||||
connection.createStatement().execute(sql);
|
||||
}
|
||||
|
||||
ClickHouseServerTest.shutDown();
|
||||
}
|
||||
|
||||
public Connection getConnection()
|
||||
{
|
||||
return connection;
|
||||
}
|
||||
|
||||
public ClickHouseClient getClickHouseClient()
|
||||
{
|
||||
return clickHouseClient;
|
||||
}
|
||||
|
||||
public List<String> getTables()
|
||||
{
|
||||
return tables;
|
||||
}
|
||||
|
||||
public String getSchema()
|
||||
{
|
||||
return clickHouseServerTest.getSchema();
|
||||
}
|
||||
|
||||
public String getActualTable(String tablePattern)
|
||||
{
|
||||
return ClickHouseServerTest.getActualTable(tables, tablePattern);
|
||||
}
|
||||
|
||||
private String buildCreateTableSql(String tableName, String columnInfo)
|
||||
{
|
||||
String newTableName = ClickHouseServerTest.generateNewTableName(tableName);
|
||||
tables.add(newTableName);
|
||||
String sql = "CREATE TABLE " + clickHouseServerTest.getSchema() + "." + newTableName + columnInfo + "engine=MergeTree() order by tuple()";
|
||||
System.out.println(sql);
|
||||
|
||||
return sql;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
#
|
||||
# WARNING
|
||||
# ^^^^^^^
|
||||
# This configuration file is for development only and should NOT be used
|
||||
# in production. For example configuration, see the Hetu documentation.
|
||||
# please set your data source info here
|
||||
connection.url=jdbc:clickhouse://localhost:8123
|
||||
connection.user=default
|
||||
connection.password=yourpassword
|
||||
connection.schema=test
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
|
||||
<suite name="TestGroupSuite">
|
||||
<test name="GroupTest">
|
||||
<classes>
|
||||
<class name="io.hetu.core.plugin.clickhouse.ClickHouseConfigTest"/>
|
||||
<class name="io.hetu.core.plugin.clickhouse.ClickHouseClientTest"/>
|
||||
<class name="io.hetu.core.plugin.clickhouse.ClickHousePluginTest"/>
|
||||
</classes>
|
||||
</test>
|
||||
</suite>
|
||||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-common</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -18,14 +18,15 @@ package io.hetu.core.common.algorithm;
|
|||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
/**
|
||||
* Algorithms for ordered iterator operations (merge, union, intersect)
|
||||
* Algorithms for sequence operations (merge, union, intersect)
|
||||
*
|
||||
* @author Han
|
||||
*/
|
||||
public class SequenceUtils
|
||||
{
|
||||
|
|
@ -45,12 +46,6 @@ public class SequenceUtils
|
|||
return merge(iterators, false);
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
public static <T extends Comparable<T>> Iterator<T> union(Iterator<T>... iterators)
|
||||
{
|
||||
return union(Arrays.asList(iterators));
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge k sorted iterators into one by keeping the order and only one copy of each element.
|
||||
*
|
||||
|
|
@ -73,57 +68,34 @@ public class SequenceUtils
|
|||
|
||||
return new Iterator<T>()
|
||||
{
|
||||
T next;
|
||||
T prev;
|
||||
|
||||
public boolean hasNext()
|
||||
{
|
||||
if (pq.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
else if (keepDuplication) {
|
||||
PeekingIterator<T> curr = pq.poll();
|
||||
next = curr.next();
|
||||
if (curr.hasNext()) {
|
||||
pq.add(curr);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
PeekingIterator<T> curr = pq.poll();
|
||||
next = curr.next();
|
||||
if (curr.hasNext()) {
|
||||
pq.add(curr);
|
||||
}
|
||||
while (!pq.isEmpty()) {
|
||||
PeekingIterator<T> it = pq.poll();
|
||||
if (it.peek().equals(next)) {
|
||||
it.next();
|
||||
if (it.hasNext()) {
|
||||
pq.add(it);
|
||||
}
|
||||
}
|
||||
else {
|
||||
pq.add(it);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return !pq.isEmpty();
|
||||
}
|
||||
|
||||
public T next()
|
||||
{
|
||||
return next;
|
||||
PeekingIterator<T> curr = pq.poll();
|
||||
T val = curr.hasNext() ? curr.next() : null;
|
||||
if (curr.hasNext()) {
|
||||
pq.add(curr);
|
||||
}
|
||||
if (keepDuplication) {
|
||||
return val;
|
||||
}
|
||||
else {
|
||||
if (prev == null || !prev.equals(val)) {
|
||||
prev = val;
|
||||
return val;
|
||||
}
|
||||
return next();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
public static <T extends Comparable<T>> Iterator<T> merge(boolean keepDuplication, Iterator<T>... iterators)
|
||||
{
|
||||
return merge(Arrays.asList(iterators), keepDuplication);
|
||||
}
|
||||
|
||||
/**
|
||||
* Intersect k sorted iterators to find the common values in them.
|
||||
*
|
||||
|
|
@ -199,12 +171,6 @@ public class SequenceUtils
|
|||
};
|
||||
}
|
||||
|
||||
@SafeVarargs
|
||||
public static <T extends Comparable<T>> Iterator<T> intersect(Iterator<T>... iterators)
|
||||
{
|
||||
return intersect(Arrays.asList(iterators));
|
||||
}
|
||||
|
||||
private static <T> T allEquals(T[] arr)
|
||||
{
|
||||
if (arr.length < 1) {
|
||||
|
|
|
|||
|
|
@ -13,13 +13,9 @@
|
|||
*/
|
||||
package io.hetu.core.common.filesystem;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Comparator;
|
||||
import java.util.UUID;
|
||||
|
||||
/**
|
||||
|
|
@ -28,24 +24,14 @@ import java.util.UUID;
|
|||
public class TempFolder
|
||||
implements AutoCloseable
|
||||
{
|
||||
private static final Logger LOG = Logger.get(TempFolder.class);
|
||||
private final String prefix;
|
||||
private File root;
|
||||
|
||||
public TempFolder()
|
||||
{
|
||||
this("");
|
||||
}
|
||||
|
||||
public TempFolder(String user)
|
||||
{
|
||||
this.prefix = "hetu-tmp-folder-" + user;
|
||||
}
|
||||
public TempFolder() {}
|
||||
|
||||
public TempFolder create()
|
||||
throws IOException
|
||||
{
|
||||
root = Files.createTempDirectory(prefix).toFile();
|
||||
root = Files.createTempDirectory("hetu-tmp-folder-").toFile();
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -86,36 +72,6 @@ public class TempFolder
|
|||
throw new IOException("Not able to create folder " + relativePath);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
if (root != null && root.exists()) {
|
||||
// retry deletion for 3 times
|
||||
for (int i = 0; i < 3; i++) {
|
||||
if (deleteRecursively(root)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!root.exists()) {
|
||||
return;
|
||||
}
|
||||
|
||||
LOG.warn("Temporary folder can not be deleted. Shutdown hook added: " + root.toPath().toAbsolutePath());
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
|
||||
try {
|
||||
Files.walk(root.toPath())
|
||||
.sorted(Comparator.reverseOrder())
|
||||
.map(Path::toFile)
|
||||
.forEach(File::delete);
|
||||
}
|
||||
catch (IOException e) {
|
||||
LOG.warn("Temporary folder not deleted. Manual deletion required: " + root.toPath().toAbsolutePath());
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
private boolean deleteRecursively(File fileToDelete)
|
||||
{
|
||||
if (fileToDelete.delete()) {
|
||||
|
|
@ -131,4 +87,14 @@ public class TempFolder
|
|||
}
|
||||
return fileToDelete.delete();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close()
|
||||
{
|
||||
if (root != null && root.exists()) {
|
||||
if (!deleteRecursively(root)) {
|
||||
throw new RuntimeException("Temporary folder not deleted. Manual deletion required.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.common.heuristicindex;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public class IndexCacheKey
|
||||
{
|
||||
private String path;
|
||||
private long lastModifiedTime;
|
||||
|
||||
/**
|
||||
* @param path path to the file the index files should be read for
|
||||
* @param lastModifiedTime lastModifiedTime of the file, used to validate the indexes
|
||||
*/
|
||||
public IndexCacheKey(String path, long lastModifiedTime)
|
||||
{
|
||||
this.path = path;
|
||||
this.lastModifiedTime = lastModifiedTime;
|
||||
}
|
||||
|
||||
public String getPath()
|
||||
{
|
||||
return path;
|
||||
}
|
||||
|
||||
public long getLastModifiedTime()
|
||||
{
|
||||
return lastModifiedTime;
|
||||
}
|
||||
|
||||
// only the path should be used as the key
|
||||
// the lastModifiedTime time is only used to check if index is valid
|
||||
@Override
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
IndexCacheKey that = (IndexCacheKey) o;
|
||||
return Objects.equals(path, that.path);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "IndexCacheKey{" +
|
||||
"path='" + path + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(path);
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,6 @@ package io.hetu.core.common.util;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
|
@ -37,7 +36,6 @@ public class SecurePathWhiteList
|
|||
|
||||
/**
|
||||
* Due to security concerns, all data must be read from one of the whitelisted paths
|
||||
*
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
|
|
@ -55,11 +53,6 @@ public class SecurePathWhiteList
|
|||
return securePathwhiteList;
|
||||
}
|
||||
|
||||
public static boolean isSecurePath(Path absolutePath) throws IOException
|
||||
{
|
||||
return isSecurePath(absolutePath.toString());
|
||||
}
|
||||
|
||||
public static boolean isSecurePath(String absolutePath) throws IOException
|
||||
{
|
||||
// absolutePath
|
||||
|
|
|
|||
|
|
@ -47,6 +47,16 @@ public class SslSocketUtil
|
|||
if (!tlsEnabled) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
// https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
|
||||
// as per link above, the default SSLContext will be constructed using the default KeyManager and
|
||||
// default TrustManager. Those can be configured using the following system properties:
|
||||
// javax.net.ssl.keyStore
|
||||
// javax.net.ssl.keyStorePassword
|
||||
// javax.net.ssl.keyStoreType
|
||||
// javax.net.ssl.trustStore
|
||||
// javax.net.ssl.trustStorePassword
|
||||
// see link above for more details
|
||||
return Optional.of(SSLContext.getDefault());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@
|
|||
*/
|
||||
package io.hetu.core.common.util;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
import io.airlift.security.pem.PemReader;
|
||||
|
||||
import javax.security.auth.x500.X500Principal;
|
||||
|
|
@ -30,8 +29,6 @@ import java.util.Optional;
|
|||
|
||||
public class TrustStore
|
||||
{
|
||||
private static final Logger LOGGER = Logger.get(TrustStore.class);
|
||||
|
||||
private TrustStore() {}
|
||||
|
||||
public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
|
||||
|
|
@ -51,7 +48,6 @@ public class TrustStore
|
|||
}
|
||||
}
|
||||
catch (IOException | GeneralSecurityException ignored) {
|
||||
LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
|
||||
}
|
||||
|
||||
try (InputStream in = new FileInputStream(trustStorePath)) {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -18,7 +18,6 @@ package io.hetu.core.common.algorithm;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
|
|
@ -34,16 +33,15 @@ public class TestSequenceUtils
|
|||
List<Integer> l3 = ImmutableList.of(5);
|
||||
Iterator<Integer> emptyIterator = Collections.emptyIterator();
|
||||
|
||||
private static <T1, T2> void assertIteratorSame(Iterator<T1> iterator1, Iterator<T2> iterator2)
|
||||
private static void assertIteratorSame(Iterator iterator1, Iterator iterator2)
|
||||
{
|
||||
while (iterator1.hasNext()) {
|
||||
assertTrue(iterator2.hasNext());
|
||||
T1 v1 = iterator1.next();
|
||||
T2 v2 = iterator2.next();
|
||||
System.out.println(v1 + "---" + v2);
|
||||
assertEquals(v1, v2);
|
||||
assertEquals(iterator2.next(), iterator1.next());
|
||||
}
|
||||
if (iterator2.hasNext()) {
|
||||
System.out.println(iterator2.next());
|
||||
}
|
||||
System.out.println();
|
||||
assertFalse(iterator2.hasNext());
|
||||
}
|
||||
|
||||
|
|
@ -75,32 +73,4 @@ public class TestSequenceUtils
|
|||
assertIteratorSame(SequenceUtils.intersect(ImmutableList.of(l1.iterator(), l2.iterator(), l3.iterator(), emptyIterator)),
|
||||
ImmutableList.of().iterator());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRepeatingValues()
|
||||
{
|
||||
List<String> l = new ArrayList<>(10000);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
l.add(String.valueOf(10000));
|
||||
}
|
||||
Iterator<String> it = SequenceUtils.union(ImmutableList.of(l.iterator()));
|
||||
// detect stack overflow
|
||||
while (it.hasNext()) {
|
||||
it.next();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDupAsEnd()
|
||||
{
|
||||
List<Integer> i1 = ImmutableList.of(1, 4, 5, 5, 7, 7, 7);
|
||||
List<Integer> i2 = ImmutableList.of(2, 2, 2, 3, 5, 5, 5);
|
||||
List<Integer> i3 = ImmutableList.of(6, 6, 6, 6);
|
||||
List<Integer> i4 = ImmutableList.of(3);
|
||||
|
||||
assertIteratorSame(SequenceUtils.union(ImmutableList.of(i1.iterator(), i2.iterator(), i3.iterator(), i4.iterator())),
|
||||
ImmutableList.of(1, 2, 3, 4, 5, 6, 7).iterator());
|
||||
assertIteratorSame(SequenceUtils.merge(ImmutableList.of(i1.iterator(), i2.iterator(), i3.iterator(), i4.iterator()), true),
|
||||
ImmutableList.of(1, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7).iterator());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,9 +34,9 @@ public class TestTempFolder
|
|||
root = folder.getRoot();
|
||||
assertTrue(root.exists());
|
||||
File newFile = folder.newFile("aNewFile");
|
||||
assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
|
||||
assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
|
||||
File newFolder = folder.newFile("aNewFolder");
|
||||
assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
|
||||
assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
|
||||
}
|
||||
assertFalse(root.exists());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-cube</artifactId>
|
||||
<name>hetu-cube</name>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-annotations</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public enum CubeAggregateFunction
|
||||
{
|
||||
SUM("sum"),
|
||||
COUNT("count"),
|
||||
AVG("avg"),
|
||||
MIN("min"),
|
||||
MAX("max");
|
||||
|
||||
private final String name;
|
||||
|
||||
public static final Set<String> SUPPORTED_FUNCTIONS = Stream.of(CubeAggregateFunction.values()).map(CubeAggregateFunction::getName).collect(Collectors.toSet());
|
||||
|
||||
CubeAggregateFunction(String name)
|
||||
{
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return this.name;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public class CubeFilter
|
||||
{
|
||||
private final String sourceTablePredicate;
|
||||
private final String cubePredicate;
|
||||
|
||||
@JsonCreator
|
||||
public CubeFilter(
|
||||
@JsonProperty("sourceTablePredicate") String sourceTablePredicate,
|
||||
@JsonProperty("cubePredicate") String cubePredicate)
|
||||
{
|
||||
this.sourceTablePredicate = sourceTablePredicate;
|
||||
this.cubePredicate = cubePredicate;
|
||||
}
|
||||
|
||||
public CubeFilter(String sourceTablePredicate)
|
||||
{
|
||||
this(sourceTablePredicate, null);
|
||||
}
|
||||
|
||||
public String getSourceTablePredicate()
|
||||
{
|
||||
return sourceTablePredicate;
|
||||
}
|
||||
|
||||
public String getCubePredicate()
|
||||
{
|
||||
return cubePredicate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
CubeFilter that = (CubeFilter) o;
|
||||
return Objects.equals(sourceTablePredicate, that.sourceTablePredicate)
|
||||
&& Objects.equals(cubePredicate, that.cubePredicate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(sourceTablePredicate, cubePredicate);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "CubeFilter{" +
|
||||
"sourceTablePredicate='" + sourceTablePredicate + '\'' +
|
||||
", cubePredicate='" + cubePredicate + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,117 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
import io.hetu.core.spi.cube.aggregator.AggregationSignature;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public interface CubeMetadata
|
||||
extends Serializable
|
||||
{
|
||||
/**
|
||||
* Returns name of the cube
|
||||
*/
|
||||
String getCubeName();
|
||||
|
||||
/**
|
||||
* Returns the name of the source table
|
||||
*/
|
||||
String getSourceTableName();
|
||||
|
||||
/**
|
||||
* Returns the last updated time of the cube
|
||||
*/
|
||||
long getLastUpdatedTime();
|
||||
|
||||
/**
|
||||
* Returns the last updated time of the source table
|
||||
*/
|
||||
long getSourceTableLastUpdatedTime();
|
||||
|
||||
/**
|
||||
* Return the names of the dimension columns
|
||||
*/
|
||||
List<String> getDimensions();
|
||||
|
||||
/**
|
||||
* Return the names of the aggregation columns
|
||||
*/
|
||||
List<String> getAggregations();
|
||||
|
||||
/**
|
||||
* Cube selection filter
|
||||
*/
|
||||
CubeFilter getCubeFilter();
|
||||
|
||||
/**
|
||||
* Return the group by columns
|
||||
*/
|
||||
Set<String> getGroup();
|
||||
|
||||
/**
|
||||
* Checks if metadata matches the CubeStatement
|
||||
* @param statement cube statement
|
||||
* @return true - if metadata matches CubeStatement
|
||||
* false - otherwise
|
||||
*/
|
||||
boolean matches(CubeStatement statement);
|
||||
|
||||
/**
|
||||
* Filters all metadata that matches the cube statement
|
||||
* @param metadataList metadata list
|
||||
* @param statement cube statement
|
||||
* @return all metadata that is matching the cube statement
|
||||
*/
|
||||
static List<CubeMetadata> filter(List<CubeMetadata> metadataList, CubeStatement statement)
|
||||
{
|
||||
return metadataList.stream()
|
||||
.filter(metadata -> metadata.matches(statement))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the cube column matching the given aggregation signature
|
||||
* @return name of the aggregation column if found
|
||||
*/
|
||||
Optional<String> getColumn(AggregationSignature aggSignature);
|
||||
|
||||
/**
|
||||
* Return the aggregation function associated with given cube column
|
||||
* @return name of the aggregation function
|
||||
*/
|
||||
Optional<String> getAggregationFunction(String column);
|
||||
|
||||
/**
|
||||
* Get the aggregation information of the given cube column
|
||||
* @param column name of the cube column
|
||||
*/
|
||||
Optional<AggregationSignature> getAggregationSignature(String column);
|
||||
|
||||
/**
|
||||
* Return all aggregation column information
|
||||
*/
|
||||
List<AggregationSignature> getAggregationSignatures();
|
||||
|
||||
/**
|
||||
* Return the status of the cube
|
||||
*/
|
||||
CubeStatus getCubeStatus();
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
import java.util.Set;
|
||||
|
||||
public interface CubeMetadataBuilder
|
||||
{
|
||||
void addDimensionColumn(String name, String originalColumn);
|
||||
|
||||
void addAggregationColumn(String name, String aggregationFunction, String originalColumn, boolean distinct);
|
||||
|
||||
void addGroup(Set<String> group);
|
||||
|
||||
void withCubeFilter(CubeFilter cubeFilter);
|
||||
|
||||
void setCubeStatus(CubeStatus cubeStatus);
|
||||
|
||||
void setTableLastUpdatedTime(long tableLastUpdatedTime);
|
||||
|
||||
void setCubeLastUpdatedTime(long cubeLastUpdatedTime);
|
||||
|
||||
CubeMetadata build();
|
||||
}
|
||||
|
|
@ -1,163 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
import io.hetu.core.spi.cube.aggregator.AggregationSignature;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.StringJoiner;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public class CubeStatement
|
||||
{
|
||||
private final String from;
|
||||
private final Set<String> groupBy;
|
||||
private final Set<String> selection;
|
||||
private final List<AggregationSignature> aggregations;
|
||||
|
||||
private CubeStatement(
|
||||
Set<String> selection,
|
||||
String from,
|
||||
Set<String> groupBy,
|
||||
List<AggregationSignature> aggregations)
|
||||
{
|
||||
this.selection = requireNonNull(selection, "selection is null");
|
||||
this.from = requireNonNull(from, "from is null");
|
||||
this.groupBy = requireNonNull(groupBy, "groupBy is null");
|
||||
this.aggregations = requireNonNull(aggregations, "aggregations is null");
|
||||
}
|
||||
|
||||
public static Builder newBuilder()
|
||||
{
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public Set<String> getSelection()
|
||||
{
|
||||
return selection;
|
||||
}
|
||||
|
||||
public String getFrom()
|
||||
{
|
||||
return from;
|
||||
}
|
||||
|
||||
public List<AggregationSignature> getAggregations()
|
||||
{
|
||||
return aggregations;
|
||||
}
|
||||
|
||||
public Set<String> getGroupBy()
|
||||
{
|
||||
return groupBy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
CubeStatement that = (CubeStatement) o;
|
||||
return Objects.equals(selection, that.selection) &&
|
||||
Objects.equals(from, that.from) &&
|
||||
Objects.equals(groupBy, that.groupBy) &&
|
||||
Objects.equals(aggregations, that.aggregations);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(selection, from, groupBy, aggregations);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
StringJoiner columns = new StringJoiner(", ");
|
||||
selection.forEach(columns::add);
|
||||
aggregations.forEach(agg -> columns.add(agg.toString()));
|
||||
|
||||
StringJoiner groupingColumns = new StringJoiner(", ");
|
||||
groupBy.forEach(groupingColumns::add);
|
||||
|
||||
return "SELECT " + columns +
|
||||
" FROM " + from +
|
||||
(groupBy.isEmpty() ? "" : " GROUP BY " + groupingColumns);
|
||||
}
|
||||
|
||||
public static class Builder
|
||||
{
|
||||
private String from;
|
||||
private final Set<String> groupBy = new HashSet<>();
|
||||
private final Set<String> selection = new HashSet<>();
|
||||
private final List<AggregationSignature> aggregations = new ArrayList<>();
|
||||
|
||||
private Builder()
|
||||
{
|
||||
// Do nothing
|
||||
}
|
||||
|
||||
public Builder select(String column, String... columns)
|
||||
{
|
||||
this.selection.add(column);
|
||||
this.selection.addAll(Arrays.asList(columns));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder aggregate(AggregationSignature signature)
|
||||
{
|
||||
this.aggregations.add(signature);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder from(String from)
|
||||
{
|
||||
this.from = from;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder groupByAddString(String column)
|
||||
{
|
||||
this.groupBy.add(column);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder groupByAddStringList(String... columns)
|
||||
{
|
||||
this.groupBy.addAll(Arrays.asList(columns));
|
||||
return this;
|
||||
}
|
||||
|
||||
public CubeStatement build()
|
||||
{
|
||||
if (this.aggregations.isEmpty() && this.selection.isEmpty()) {
|
||||
throw new UnsupportedOperationException("Cannot construct a cube statement without selection and aggregation");
|
||||
}
|
||||
return new CubeStatement(Collections.unmodifiableSet(selection), from, Collections.unmodifiableSet(groupBy), Collections.unmodifiableList(aggregations));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
public enum CubeStatus
|
||||
{
|
||||
INACTIVE(0),
|
||||
READY(1);
|
||||
|
||||
private final int value;
|
||||
|
||||
CubeStatus(int value)
|
||||
{
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public int getValue()
|
||||
{
|
||||
return this.value;
|
||||
}
|
||||
|
||||
public static CubeStatus forValue(int value)
|
||||
{
|
||||
if (value == 0) {
|
||||
return INACTIVE;
|
||||
}
|
||||
else if (value == 1) {
|
||||
return READY;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube.aggregator;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.AVG;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.COUNT;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.MAX;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.MIN;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.SUM;
|
||||
|
||||
public class AggregationSignature
|
||||
implements Serializable, Comparable<AggregationSignature>
|
||||
{
|
||||
private static final AggregationSignature COUNT_SIGNATURE = new AggregationSignature(COUNT.getName(), "*", false);
|
||||
|
||||
private String function;
|
||||
private String dimension;
|
||||
private boolean distinct;
|
||||
|
||||
@JsonCreator
|
||||
public AggregationSignature(
|
||||
@JsonProperty("function") String function,
|
||||
@JsonProperty("dimension") String dimension,
|
||||
@JsonProperty("distinct") boolean distinct)
|
||||
{
|
||||
this.function = function;
|
||||
this.dimension = dimension;
|
||||
this.distinct = distinct;
|
||||
}
|
||||
|
||||
public static AggregationSignature count()
|
||||
{
|
||||
return COUNT_SIGNATURE;
|
||||
}
|
||||
|
||||
public static AggregationSignature count(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(COUNT.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature sum(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(SUM.toString(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature avg(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(AVG.toString(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature min(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(MIN.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature max(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(MAX.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFunction()
|
||||
{
|
||||
return function;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getDimension()
|
||||
{
|
||||
return dimension;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isDistinct()
|
||||
{
|
||||
return distinct;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(function, dimension);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj)
|
||||
{
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
if (obj == null || getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
AggregationSignature other = (AggregationSignature) obj;
|
||||
return Objects.equals(this.function, other.function) &&
|
||||
Objects.equals(this.dimension, other.dimension) &&
|
||||
Objects.equals(this.distinct, other.distinct);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return this.function + "(" + (this.distinct ? "distinct " : "") + this.dimension + ")";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int compareTo(AggregationSignature aggregationSignature)
|
||||
{
|
||||
int nameComparison = function.compareTo(aggregationSignature.function);
|
||||
if (0 != nameComparison) {
|
||||
return nameComparison;
|
||||
}
|
||||
else {
|
||||
int dimensionComparison = dimension.compareTo(aggregationSignature.dimension);
|
||||
if (0 != dimensionComparison) {
|
||||
return dimensionComparison;
|
||||
}
|
||||
else {
|
||||
return Boolean.compare(distinct, aggregationSignature.distinct);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube.io;
|
||||
|
||||
import io.hetu.core.spi.cube.CubeMetadata;
|
||||
import io.hetu.core.spi.cube.CubeMetadataBuilder;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* CubeMetaStore provides APIs to retrieve, update the Cube metadata
|
||||
* from the underlying metastore.
|
||||
*/
|
||||
public interface CubeMetaStore
|
||||
{
|
||||
/**
|
||||
* Persist cube metadata in the underlying metastore
|
||||
* @param cubeMetadata cube metadata
|
||||
*/
|
||||
void persist(CubeMetadata cubeMetadata);
|
||||
|
||||
/**
|
||||
* Create a new Metadata builder
|
||||
* @param cubeName Name of the cube
|
||||
* @param sourceTableName Name of the table from which cube was created
|
||||
* @return a metadata builder
|
||||
*/
|
||||
CubeMetadataBuilder getBuilder(String cubeName, String sourceTableName);
|
||||
|
||||
/**
|
||||
* Create new metadata builder from the existing metadata
|
||||
* @param existingMetadata existing metadata
|
||||
* @return a metadata builder
|
||||
*/
|
||||
CubeMetadataBuilder getBuilder(CubeMetadata existingMetadata);
|
||||
|
||||
/**
|
||||
* Returns the list of cube metadata associated with the given table.
|
||||
* @param tableName fully qualified name of the table
|
||||
* @return list of cube metadata
|
||||
*/
|
||||
List<CubeMetadata> getMetadataList(String tableName);
|
||||
|
||||
/**
|
||||
* Find a cube metadata associated with the given name
|
||||
* @param cubeName name of the cube
|
||||
* @return optional cube metadata
|
||||
*/
|
||||
Optional<CubeMetadata> getMetadataFromCubeName(String cubeName);
|
||||
|
||||
/**
|
||||
* Return all cube information
|
||||
* @return list of cube metadata
|
||||
*/
|
||||
List<CubeMetadata> getAllCubes();
|
||||
|
||||
/**
|
||||
* Remove cube metadata
|
||||
*/
|
||||
void removeCube(CubeMetadata cubeMetadata);
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.spi.cube;
|
||||
|
||||
import io.hetu.core.spi.cube.aggregator.AggregationSignature;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotEquals;
|
||||
|
||||
public class TestCubeStatement
|
||||
{
|
||||
@Test
|
||||
public void testCubeStatement()
|
||||
{
|
||||
CubeStatement statement = CubeStatement.newBuilder()
|
||||
.select("name", "address", "nationkey")
|
||||
.aggregate(AggregationSignature.count())
|
||||
.from("tpch.tiny.customer")
|
||||
.groupByAddString("address")
|
||||
.groupByAddStringList("name", "nationkey")
|
||||
.build();
|
||||
|
||||
assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");
|
||||
assertEquals(statement.getSelection(), new HashSet<>(Arrays.asList("name", "address", "nationkey")), "incorrect selection");
|
||||
assertEquals(statement.getGroupBy(), new HashSet<>(Arrays.asList("name", "address", "nationkey")), "incorrect address");
|
||||
assertEquals(statement.getAggregations(), Collections.singletonList(AggregationSignature.count()), "incorrect aggregations");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCubeEquality()
|
||||
{
|
||||
CubeStatement statement1 = CubeStatement.newBuilder()
|
||||
.select("name", "address", "nationkey")
|
||||
.aggregate(AggregationSignature.count())
|
||||
.from("tpch.tiny.customer")
|
||||
.build();
|
||||
|
||||
CubeStatement statement2 = CubeStatement.newBuilder()
|
||||
.select("name", "address", "nationkey")
|
||||
.aggregate(AggregationSignature.count())
|
||||
.from("tpch.tiny.customer")
|
||||
.build();
|
||||
|
||||
CubeStatement statement3 = CubeStatement.newBuilder()
|
||||
.select("name", "address")
|
||||
.aggregate(AggregationSignature.count())
|
||||
.from("tpch.tiny.customer")
|
||||
.build();
|
||||
|
||||
assertEquals(statement1, statement2, "statements are not equal");
|
||||
assertNotEquals(statement1, statement3, "statements are not equal");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
<?xml version="1.0"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.0.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-datacenter</artifactId>
|
||||
|
|
@ -73,11 +74,6 @@
|
|||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-expressions</artifactId>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.airlift</groupId>
|
||||
<artifactId>slice</artifactId>
|
||||
|
|
@ -109,12 +105,6 @@
|
|||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>hetu-state-store</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.testng</groupId>
|
||||
<artifactId>testng</artifactId>
|
||||
|
|
|
|||
|
|
@ -55,7 +55,6 @@ public final class DataCenterColumnHandle
|
|||
}
|
||||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public String getColumnName()
|
||||
{
|
||||
return columnName;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -20,8 +20,6 @@ import io.airlift.configuration.ConfigDescription;
|
|||
import io.airlift.configuration.ConfigSecuritySensitive;
|
||||
import io.airlift.units.DataSize;
|
||||
import io.airlift.units.Duration;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
|
||||
import io.prestosql.spi.function.Mandatory;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
|
@ -50,7 +48,7 @@ public class DataCenterConfig
|
|||
|
||||
private URI connectionUrl;
|
||||
|
||||
private String connectionUser = System.getProperty("user.name");
|
||||
private String connectionUser;
|
||||
|
||||
private String connectionPassword;
|
||||
|
||||
|
|
@ -104,8 +102,6 @@ public class DataCenterConfig
|
|||
|
||||
private boolean isQueryPushDownEnabled = true;
|
||||
|
||||
private JdbcPushDownModule queryPushDownModule = JdbcPushDownModule.DEFAULT;
|
||||
|
||||
private Duration metadataCacheTtl = new Duration(1, TimeUnit.SECONDS); // DataCenter metadata cache eviction time
|
||||
|
||||
private long metadataCacheMaximumSize = DEFAULT_METADATA_CACHE_MAX_SIZE; // DataCenter metadata cache max size
|
||||
|
|
@ -147,10 +143,6 @@ public class DataCenterConfig
|
|||
* @param connectionUrl the connection url of data center.
|
||||
* @return DataCenterConfig object.
|
||||
*/
|
||||
@Mandatory(name = "connection-url",
|
||||
description = "The connection URL of remote OpenLooKeng data center",
|
||||
defaultValue = "http://host:port",
|
||||
required = true)
|
||||
@Config("connection-url")
|
||||
public DataCenterConfig setConnectionUrl(URI connectionUrl)
|
||||
{
|
||||
|
|
@ -690,25 +682,6 @@ public class DataCenterConfig
|
|||
return this;
|
||||
}
|
||||
|
||||
public JdbcPushDownModule getQueryPushDownModule()
|
||||
{
|
||||
return queryPushDownModule;
|
||||
}
|
||||
|
||||
/**
|
||||
* set queryPushDownEnabled
|
||||
*
|
||||
* @param queryPushDownModule Push Down Module
|
||||
* @return DataCenterConfig object
|
||||
*/
|
||||
@Config("dc.query.pushdown.module")
|
||||
@ConfigDescription("query push down module [FULL_PUSHDOWN/BASE_PUSHDOWN]")
|
||||
public DataCenterConfig setQueryPushDownModule(JdbcPushDownModule queryPushDownModule)
|
||||
{
|
||||
this.queryPushDownModule = queryPushDownModule;
|
||||
return this;
|
||||
}
|
||||
|
||||
public DataSize getRemoteHttpServerMaxRequestHeaderSize()
|
||||
{
|
||||
return remoteHeaderSize;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -15,19 +15,15 @@
|
|||
|
||||
package io.hetu.core.plugin.datacenter;
|
||||
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.airlift.bootstrap.LifeCycleManager;
|
||||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.plugin.datacenter.client.DataCenterClient;
|
||||
import io.hetu.core.plugin.datacenter.client.DataCenterStatementClientFactory;
|
||||
import io.hetu.core.plugin.datacenter.optimization.DataCenterPlanOptimizer;
|
||||
import io.hetu.core.plugin.datacenter.pagesource.DataCenterPageSourceProvider;
|
||||
import io.prestosql.spi.ConnectorPlanOptimizer;
|
||||
import io.prestosql.spi.connector.CachedConnectorMetadata;
|
||||
import io.prestosql.spi.connector.Connector;
|
||||
import io.prestosql.spi.connector.ConnectorMetadata;
|
||||
import io.prestosql.spi.connector.ConnectorPageSourceProvider;
|
||||
import io.prestosql.spi.connector.ConnectorPlanOptimizerProvider;
|
||||
import io.prestosql.spi.connector.ConnectorSplitManager;
|
||||
import io.prestosql.spi.connector.ConnectorTransactionHandle;
|
||||
import io.prestosql.spi.transaction.IsolationLevel;
|
||||
|
|
@ -38,7 +34,6 @@ import javax.inject.Inject;
|
|||
|
||||
import java.util.Collection;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static io.hetu.core.plugin.datacenter.DataCenterTransactionHandle.INSTANCE;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
|
@ -65,8 +60,6 @@ public class DataCenterConnector
|
|||
|
||||
private final OkHttpClient httpClient;
|
||||
|
||||
private final ConnectorPlanOptimizer planOptimizer;
|
||||
|
||||
/**
|
||||
* Constructor of data center connector.
|
||||
*
|
||||
|
|
@ -75,18 +68,14 @@ public class DataCenterConnector
|
|||
* @param typeManager the type manager.
|
||||
*/
|
||||
@Inject
|
||||
public DataCenterConnector(
|
||||
LifeCycleManager lifeCycleManager,
|
||||
DataCenterConfig dataCenterConfig,
|
||||
TypeManager typeManager,
|
||||
DataCenterPlanOptimizer planOptimizer)
|
||||
public DataCenterConnector(LifeCycleManager lifeCycleManager, DataCenterConfig dataCenterConfig,
|
||||
TypeManager typeManager)
|
||||
{
|
||||
this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null");
|
||||
this.httpClient = DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
|
||||
this.dataCenterClient = new DataCenterClient(dataCenterConfig, this.httpClient, typeManager);
|
||||
this.splitManager = new DataCenterSplitManager(dataCenterConfig, this.dataCenterClient);
|
||||
this.pageSourceProvider = new DataCenterPageSourceProvider(dataCenterConfig, this.httpClient, typeManager);
|
||||
this.planOptimizer = planOptimizer;
|
||||
if (dataCenterConfig.isMetadataCacheEnabled()) {
|
||||
this.metadata = new CachedConnectorMetadata(new DataCenterMetadata(dataCenterClient, dataCenterConfig),
|
||||
dataCenterConfig.getMetadataCacheTtl(), dataCenterConfig.getMetadataCacheMaximumSize());
|
||||
|
|
@ -96,25 +85,6 @@ public class DataCenterConnector
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider()
|
||||
{
|
||||
return new ConnectorPlanOptimizerProvider()
|
||||
{
|
||||
@Override
|
||||
public Set<ConnectorPlanOptimizer> getLogicalPlanOptimizers()
|
||||
{
|
||||
return ImmutableSet.of(planOptimizer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<ConnectorPlanOptimizer> getPhysicalPlanOptimizers()
|
||||
{
|
||||
return ImmutableSet.of();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean isReadOnly)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -22,10 +22,6 @@ import io.prestosql.spi.connector.Connector;
|
|||
import io.prestosql.spi.connector.ConnectorContext;
|
||||
import io.prestosql.spi.connector.ConnectorFactory;
|
||||
import io.prestosql.spi.connector.ConnectorHandleResolver;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpressionService;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
|
@ -58,15 +54,7 @@ public class DataCenterConnectorFactory
|
|||
requireNonNull(requiredConfig, "requiredConfig is null");
|
||||
try {
|
||||
// A plugin is not required to use Guice; it is just very convenient
|
||||
Bootstrap app = new Bootstrap(
|
||||
binder -> {
|
||||
binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager());
|
||||
binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution());
|
||||
binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService());
|
||||
binder.bind(DeterminismEvaluator.class).toInstance(context.getRowExpressionService().getDeterminismEvaluator());
|
||||
},
|
||||
new JsonModule(),
|
||||
new DataCenterModule(context.getTypeManager()));
|
||||
Bootstrap app = new Bootstrap(new JsonModule(), new DataCenterModule(context.getTypeManager()));
|
||||
|
||||
Injector injector = app.strictConfig()
|
||||
.doNotInitializeLogging()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -32,9 +32,13 @@ import io.prestosql.spi.connector.LimitApplicationResult;
|
|||
import io.prestosql.spi.connector.SchemaNotFoundException;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.connector.SchemaTablePrefix;
|
||||
import io.prestosql.spi.connector.SubQueryApplicationResult;
|
||||
import io.prestosql.spi.connector.TableNotFoundException;
|
||||
import io.prestosql.spi.sql.SqlQueryWriter;
|
||||
import io.prestosql.spi.statistics.TableStatistics;
|
||||
import io.prestosql.spi.type.Type;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
|
@ -242,7 +246,50 @@ public class DataCenterMetadata
|
|||
}
|
||||
|
||||
@Override
|
||||
public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, boolean includeColumnStatistics)
|
||||
public Optional<SubQueryApplicationResult<ConnectorTableHandle>> applySubQuery(ConnectorSession session,
|
||||
ConnectorTableHandle handle, String subQuery, Map<String, Type> types)
|
||||
{
|
||||
if (!isQueryPushDownEnabled || subQuery.getBytes(StandardCharsets.ISO_8859_1).length >= maxRemoteHeaderSize) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
// If the subQuery pushed down to the connector, table name, limit or predicate push downs are not necessary
|
||||
// Therefore, either of the table name can be used for the new TableHandle as long as the subQuery is valid
|
||||
requireNonNull(subQuery, "cannot apply null sub-query");
|
||||
DataCenterTableHandle tableHandle = (DataCenterTableHandle) handle;
|
||||
|
||||
// If we can get the columns from the sub-query, it should be able to push sub-query down
|
||||
List<DataCenterColumn> columns = dataCenterClient.getColumns(subQuery);
|
||||
if (columns.isEmpty()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
DataCenterTableHandle newTableHandle = new DataCenterTableHandle(tableHandle.getCatalogName(),
|
||||
tableHandle.getSchemaName(), tableHandle.getTableName(), OptionalLong.empty(), subQuery);
|
||||
|
||||
ImmutableMap.Builder<String, ColumnHandle> columnHandleBuilder = new ImmutableMap.Builder<>();
|
||||
ImmutableMap.Builder<String, Type> typesBuilder = new ImmutableMap.Builder<>();
|
||||
|
||||
columns.forEach(column -> {
|
||||
columnHandleBuilder.put(column.getName(),
|
||||
new DataCenterColumnHandle(column.getName(), column.getType(), 0));
|
||||
typesBuilder.put(column.getName(), column.getType());
|
||||
});
|
||||
|
||||
return Optional.of(
|
||||
new SubQueryApplicationResult<>(newTableHandle, columnHandleBuilder.build(), typesBuilder.build()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<SqlQueryWriter> getSqlQueryWriter()
|
||||
{
|
||||
if (!isQueryPushDownEnabled) {
|
||||
return Optional.empty();
|
||||
}
|
||||
return Optional.of(new DataCenterSqlQueryWriter());
|
||||
}
|
||||
|
||||
@Override
|
||||
public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint)
|
||||
{
|
||||
Map<String, ColumnHandle> columnHandles = getColumnHandles(session, tableHandle);
|
||||
String tableFullName = tableHandle.getSchemaPrefixedTableName();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -18,8 +18,6 @@ package io.hetu.core.plugin.datacenter;
|
|||
import com.google.inject.Binder;
|
||||
import com.google.inject.Module;
|
||||
import com.google.inject.Scopes;
|
||||
import io.hetu.core.plugin.datacenter.optimization.DataCenterPlanOptimizer;
|
||||
import io.hetu.core.plugin.datacenter.optimization.DataCenterQueryGenerator;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
|
||||
import static io.airlift.configuration.ConfigBinder.configBinder;
|
||||
|
|
@ -50,8 +48,6 @@ public class DataCenterModule
|
|||
{
|
||||
binder.bind(TypeManager.class).toInstance(typeManager);
|
||||
binder.bind(DataCenterConnector.class).in(Scopes.SINGLETON);
|
||||
binder.bind(DataCenterPlanOptimizer.class).in(Scopes.SINGLETON);
|
||||
binder.bind(DataCenterQueryGenerator.class).in(Scopes.SINGLETON);
|
||||
configBinder(binder).bindConfig(DataCenterConfig.class);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -18,24 +18,12 @@ package io.hetu.core.plugin.datacenter;
|
|||
import com.google.common.collect.ImmutableList;
|
||||
import io.prestosql.spi.Plugin;
|
||||
import io.prestosql.spi.connector.ConnectorFactory;
|
||||
import io.prestosql.spi.function.ConnectorConfig;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorUtil;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
|
||||
/**
|
||||
* Data center plugin.
|
||||
*
|
||||
* @since 2020-02-11
|
||||
*/
|
||||
@ConnectorConfig(connectorLabel = "DataCenter: Query data on remote OpenLooKeng data center",
|
||||
propertiesEnabled = true,
|
||||
catalogConfigFilesEnabled = true,
|
||||
globalConfigFilesEnabled = true,
|
||||
docLink = "https://openlookeng.io/docs/docs/connector/datacenter.html",
|
||||
configLink = "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration")
|
||||
public class DataCenterPlugin
|
||||
implements Plugin
|
||||
{
|
||||
|
|
@ -44,12 +32,4 @@ public class DataCenterPlugin
|
|||
{
|
||||
return ImmutableList.of(new DataCenterConnectorFactory());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ConnectorWithProperties> getConnectorWithProperties()
|
||||
{
|
||||
ConnectorConfig connectorConfig = DataCenterPlugin.class.getAnnotation(ConnectorConfig.class);
|
||||
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
|
||||
Arrays.asList(DataCenterConfig.class.getDeclaredMethods()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,23 +13,27 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.prestosql.catalog.showcatalog;
|
||||
package io.hetu.core.plugin.datacenter;
|
||||
|
||||
import com.google.inject.Binder;
|
||||
import com.google.inject.Scopes;
|
||||
import io.airlift.configuration.AbstractConfigurationAwareModule;
|
||||
import org.glassfish.jersey.media.multipart.MultiPartFeature;
|
||||
import io.prestosql.spi.sql.expression.Selection;
|
||||
import io.prestosql.sql.builder.BaseSqlQueryWriter;
|
||||
|
||||
import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
public class ShowCatalogModule
|
||||
extends AbstractConfigurationAwareModule
|
||||
/**
|
||||
* Implementation of BaseSqlQueryWriter. It knows how to write
|
||||
* Hetu SQL for the logical plan.
|
||||
*/
|
||||
public class DataCenterSqlQueryWriter
|
||||
extends BaseSqlQueryWriter
|
||||
{
|
||||
@Override
|
||||
protected void setup(Binder binder)
|
||||
public String formatIdentifier(Optional<Map<String, Selection>> qualifiedNames, String identifier)
|
||||
{
|
||||
jaxrsBinder(binder).bind(ShowCatalogResource.class);
|
||||
jaxrsBinder(binder).bind(MultiPartFeature.class);
|
||||
binder.bind(ShowCatalogService.class).in(Scopes.SINGLETON);
|
||||
if (qualifiedNames.isPresent()) {
|
||||
return qualifiedNames.get().get(identifier).getExpression();
|
||||
}
|
||||
return '"' + identifier.replace("\"", "\"\"") + '"';
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -17,7 +17,6 @@ package io.hetu.core.plugin.datacenter;
|
|||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.base.Joiner;
|
||||
import io.prestosql.spi.connector.ConnectorTableHandle;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
|
||||
|
|
@ -44,7 +43,7 @@ public final class DataCenterTableHandle
|
|||
|
||||
private final OptionalLong limit;
|
||||
|
||||
private final String pushDownSql;
|
||||
private final String subQuery;
|
||||
|
||||
/**
|
||||
* Constructor of data center table handle.
|
||||
|
|
@ -56,11 +55,11 @@ public final class DataCenterTableHandle
|
|||
*/
|
||||
public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
|
||||
{
|
||||
this(catalogName,
|
||||
requireNonNull(schemaName, "schemaName is null"),
|
||||
requireNonNull(tableName, "tableName is null"),
|
||||
requireNonNull(limit, "limit is null"),
|
||||
"");
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = requireNonNull(schemaName, "schemaName is null");
|
||||
this.tableName = requireNonNull(tableName, "tableName is null");
|
||||
this.limit = requireNonNull(limit, "limit is null");
|
||||
this.subQuery = "";
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -70,25 +69,25 @@ public final class DataCenterTableHandle
|
|||
* @param schemaName schema name.
|
||||
* @param tableName table name.
|
||||
* @param limit the limit number of this query need.
|
||||
* @param pushDownSql the sub query statement that want to be pushed down to remote data center.
|
||||
* @param subQuery the sub query statement that want to be pushed down to remote data center.
|
||||
*/
|
||||
@JsonCreator
|
||||
public DataCenterTableHandle(@JsonProperty("catalogName") String catalogName,
|
||||
@JsonProperty("schemaName") String schemaName, @JsonProperty("tableName") String tableName,
|
||||
@JsonProperty("limit") OptionalLong limit, @JsonProperty("subQuery") String pushDownSql)
|
||||
@JsonProperty("limit") OptionalLong limit, @JsonProperty("subQuery") String subQuery)
|
||||
{
|
||||
this.catalogName = catalogName;
|
||||
this.schemaName = requireNonNull(schemaName, "schemaName is null");
|
||||
this.tableName = requireNonNull(tableName, "tableName is null");
|
||||
this.limit = requireNonNull(limit, "limit is null");
|
||||
this.pushDownSql = pushDownSql;
|
||||
this.subQuery = subQuery;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorTableHandle createFrom(ConnectorTableHandle connectorTableHandle)
|
||||
{
|
||||
DataCenterTableHandle dataCenterTableHandle = (DataCenterTableHandle) connectorTableHandle;
|
||||
return new DataCenterTableHandle(catalogName, schemaName, dataCenterTableHandle.tableName, dataCenterTableHandle.getLimit(), dataCenterTableHandle.getPushDownSql());
|
||||
return new DataCenterTableHandle(catalogName, schemaName, dataCenterTableHandle.tableName, dataCenterTableHandle.getLimit(), dataCenterTableHandle.getSubQuery());
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -125,16 +124,15 @@ public final class DataCenterTableHandle
|
|||
return new SchemaTableName(schemaName, tableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getSchemaPrefixedTableName()
|
||||
{
|
||||
return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getPushDownSql()
|
||||
public String getSubQuery()
|
||||
{
|
||||
return pushDownSql;
|
||||
return subQuery;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -161,14 +159,6 @@ public final class DataCenterTableHandle
|
|||
@Override
|
||||
public String toString()
|
||||
{
|
||||
StringBuilder builder = new StringBuilder();
|
||||
if (!pushDownSql.isEmpty()) {
|
||||
Joiner.on(SPLIT_DOT).skipNulls().appendTo(builder, catalogName, "{" + pushDownSql + "}");
|
||||
}
|
||||
else {
|
||||
Joiner.on(SPLIT_DOT).skipNulls().appendTo(builder, catalogName, schemaName, tableName);
|
||||
}
|
||||
limit.ifPresent(value -> builder.append(" limit=").append(value));
|
||||
return builder.toString();
|
||||
return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,324 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.plugin.datacenter.optimization;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterColumn;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterColumnHandle;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterConfig;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterTableHandle;
|
||||
import io.hetu.core.plugin.datacenter.client.DataCenterClient;
|
||||
import io.hetu.core.plugin.datacenter.client.DataCenterStatementClientFactory;
|
||||
import io.prestosql.expressions.LogicalRowExpressions;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorContext;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorResult;
|
||||
import io.prestosql.spi.ConnectorPlanOptimizer;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.SymbolAllocator;
|
||||
import io.prestosql.spi.connector.CatalogName;
|
||||
import io.prestosql.spi.connector.ColumnHandle;
|
||||
import io.prestosql.spi.connector.ConnectorSession;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.metadata.TableHandle;
|
||||
import io.prestosql.spi.operator.ReuseExchangeOperator;
|
||||
import io.prestosql.spi.plan.Assignments;
|
||||
import io.prestosql.spi.plan.FilterNode;
|
||||
import io.prestosql.spi.plan.GroupIdNode;
|
||||
import io.prestosql.spi.plan.MarkDistinctNode;
|
||||
import io.prestosql.spi.plan.PlanNode;
|
||||
import io.prestosql.spi.plan.PlanNodeIdAllocator;
|
||||
import io.prestosql.spi.plan.PlanVisitor;
|
||||
import io.prestosql.spi.plan.ProjectNode;
|
||||
import io.prestosql.spi.plan.Symbol;
|
||||
import io.prestosql.spi.plan.TableScanNode;
|
||||
import io.prestosql.spi.predicate.TupleDomain;
|
||||
import io.prestosql.spi.relation.CallExpression;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpression;
|
||||
import io.prestosql.spi.relation.VariableReferenceExpression;
|
||||
import io.prestosql.spi.type.Type;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
import io.prestosql.spi.type.UnknownType;
|
||||
import okhttp3.OkHttpClient;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.IntStream;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.collect.ImmutableList.toImmutableList;
|
||||
import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.getGroupingSetColumn;
|
||||
import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.replaceGroupingSetColumns;
|
||||
import static io.prestosql.spi.function.OperatorType.CAST;
|
||||
|
||||
public class DataCenterPlanOptimizer
|
||||
implements ConnectorPlanOptimizer
|
||||
{
|
||||
private static final Logger log = Logger.get(DataCenterPlanOptimizer.class);
|
||||
|
||||
private static final String DATACENTER_CATALOG_PREFIX = "dc.";
|
||||
private static final Set<Class<? extends PlanNode>> UNSUPPORTED_ROOT_NODE = ImmutableSet.of(GroupIdNode.class, MarkDistinctNode.class);
|
||||
|
||||
private final DataCenterClient client;
|
||||
private final DataCenterConfig config;
|
||||
private final TypeManager typeManager;
|
||||
private final DataCenterQueryGenerator queryGenerator;
|
||||
private final StandardFunctionResolution functionResolution;
|
||||
private final LogicalRowExpressions logicalRowExpressions;
|
||||
|
||||
@Inject
|
||||
public DataCenterPlanOptimizer(
|
||||
TypeManager typeManager,
|
||||
DataCenterConfig config,
|
||||
DataCenterQueryGenerator query,
|
||||
DeterminismEvaluator determinismEvaluator,
|
||||
FunctionMetadataManager functionManager,
|
||||
StandardFunctionResolution functionResolution)
|
||||
{
|
||||
OkHttpClient httpClient = DataCenterStatementClientFactory.newHttpClient(config);
|
||||
this.client = new DataCenterClient(config, httpClient, typeManager);
|
||||
this.config = config;
|
||||
this.typeManager = typeManager;
|
||||
this.queryGenerator = query;
|
||||
this.functionResolution = functionResolution;
|
||||
this.logicalRowExpressions = new LogicalRowExpressions(
|
||||
determinismEvaluator,
|
||||
functionResolution,
|
||||
functionManager);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlanNode optimize(PlanNode maxSubPlan, ConnectorSession session, Map<String, Type> types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator)
|
||||
{
|
||||
if (!config.isQueryPushDownEnabled()) {
|
||||
return maxSubPlan;
|
||||
}
|
||||
// Some node cannot be push down root node.
|
||||
if (UNSUPPORTED_ROOT_NODE.contains(maxSubPlan.getClass())) {
|
||||
return maxSubPlan;
|
||||
}
|
||||
return maxSubPlan.accept(new Visitor(idAllocator, types, session, symbolAllocator), null);
|
||||
}
|
||||
|
||||
private static PlanNode replaceChildren(PlanNode node, List<PlanNode> children)
|
||||
{
|
||||
for (int i = 0; i < node.getSources().size(); i++) {
|
||||
if (children.get(i) != node.getSources().get(i)) {
|
||||
return node.replaceChildren(children);
|
||||
}
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
private class Visitor
|
||||
extends PlanVisitor<PlanNode, Void>
|
||||
{
|
||||
private final PlanNodeIdAllocator idAllocator;
|
||||
private final ConnectorSession session;
|
||||
private final Map<String, Type> types;
|
||||
private final SymbolAllocator symbolAllocator;
|
||||
private final IdentityHashMap<FilterNode, Void> filtersSplitUp = new IdentityHashMap<>();
|
||||
|
||||
public Visitor(
|
||||
PlanNodeIdAllocator idAllocator,
|
||||
Map<String, Type> types,
|
||||
ConnectorSession session,
|
||||
SymbolAllocator symbolAllocator)
|
||||
{
|
||||
this.idAllocator = idAllocator;
|
||||
this.types = types;
|
||||
this.session = session;
|
||||
this.symbolAllocator = symbolAllocator;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlanNode visitPlan(PlanNode node, Void context)
|
||||
{
|
||||
Optional<PlanNode> pushDownPlan = tryCreatingNewScanNode(node);
|
||||
return pushDownPlan.orElseGet(() -> replaceChildren(
|
||||
node, node.getSources().stream().map(source -> source.accept(this, null)).collect(toImmutableList())));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PlanNode visitFilter(FilterNode node, Void context)
|
||||
{
|
||||
if (filtersSplitUp.containsKey(node)) {
|
||||
return this.visitPlan(node, context);
|
||||
}
|
||||
filtersSplitUp.put(node, null);
|
||||
FilterNode nodeToRecurseInto = node;
|
||||
List<RowExpression> pushable = new ArrayList<>();
|
||||
List<RowExpression> nonPushable = new ArrayList<>();
|
||||
|
||||
for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
|
||||
try {
|
||||
conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
|
||||
pushable.add(conjunct);
|
||||
}
|
||||
catch (PrestoException pe) {
|
||||
nonPushable.add(conjunct);
|
||||
}
|
||||
}
|
||||
if (!pushable.isEmpty()) {
|
||||
FilterNode pushableFilter = new FilterNode(idAllocator.getNextId(), node.getSource(), logicalRowExpressions.combineConjuncts(pushable));
|
||||
Optional<FilterNode> nonPushableFilter = nonPushable.isEmpty() ? Optional.empty() : Optional.of(new FilterNode(idAllocator.getNextId(), pushableFilter, logicalRowExpressions.combineConjuncts(nonPushable)));
|
||||
|
||||
filtersSplitUp.put(pushableFilter, null);
|
||||
if (nonPushableFilter.isPresent()) {
|
||||
FilterNode nonPushableFilterNode = nonPushableFilter.get();
|
||||
filtersSplitUp.put(nonPushableFilterNode, null);
|
||||
nodeToRecurseInto = nonPushableFilterNode;
|
||||
}
|
||||
else {
|
||||
nodeToRecurseInto = pushableFilter;
|
||||
}
|
||||
}
|
||||
return this.visitFilter(nodeToRecurseInto, context);
|
||||
}
|
||||
|
||||
private Optional<PlanNode> tryCreatingNewScanNode(PlanNode node)
|
||||
{
|
||||
Optional<JdbcQueryGeneratorResult> result = queryGenerator.generate(node, typeManager);
|
||||
if (!result.isPresent()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
JdbcQueryGeneratorContext context = result.get().getContext();
|
||||
JdbcQueryGeneratorResult.GeneratedSql generatedSql = result.get().getGeneratedSql();
|
||||
if (!generatedSql.isPushDown()) {
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
JdbcQueryGeneratorContext.GroupIdNodeInfo groupIdNodeInfo = context.getGroupIdNodeInfo();
|
||||
String sql = generatedSql.getSql();
|
||||
// replace grouping sets column
|
||||
if (groupIdNodeInfo.isGroupByComplexOperation()) {
|
||||
sql = replaceGroupingSetColumns(sql);
|
||||
}
|
||||
|
||||
if (sql.getBytes(StandardCharsets.ISO_8859_1).length >= config.getRemoteHttpServerMaxRequestHeaderSize().toBytes()) {
|
||||
log.debug("Generated sql is too long, push down failed.");
|
||||
return Optional.empty();
|
||||
}
|
||||
List<DataCenterColumn> columnsList;
|
||||
try {
|
||||
columnsList = client.getColumns(sql);
|
||||
}
|
||||
catch (PrestoException e) {
|
||||
log.warn("query push down failed for [%s]", e.getMessage());
|
||||
return Optional.empty();
|
||||
}
|
||||
if (columnsList.isEmpty()) {
|
||||
log.debug("Get columns from generated sql failed.");
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
Map<String, ColumnHandle> columns = new HashMap<>();
|
||||
IntStream.range(0, columnsList.size()).forEach(i -> {
|
||||
DataCenterColumn column = columnsList.get(i);
|
||||
columns.put(column.getName(), new DataCenterColumnHandle(column.getName(), column.getType(), i));
|
||||
});
|
||||
ImmutableList.Builder<Symbol> scanOutputs = new ImmutableList.Builder<>();
|
||||
ImmutableMap.Builder<Symbol, ColumnHandle> columnHandles = new ImmutableMap.Builder<>();
|
||||
ImmutableMap.Builder<Symbol, RowExpression> assignments = new ImmutableMap.Builder<>();
|
||||
|
||||
for (Symbol symbol : node.getOutputSymbols()) {
|
||||
String name = symbol.getName().toLowerCase(Locale.ENGLISH);
|
||||
String aliasName = groupIdNodeInfo.isGroupByComplexOperation()
|
||||
? getGroupingSetColumn(name)
|
||||
: name;
|
||||
if (!types.containsKey(name) || !columns.containsKey(aliasName)) {
|
||||
log.debug("Get type of column [%s] failed", name);
|
||||
return Optional.empty();
|
||||
}
|
||||
Type prestoType = types.get(name);
|
||||
Type dcType = ((DataCenterColumnHandle) columns.get(aliasName)).getColumnType();
|
||||
|
||||
if (prestoType.equals(dcType)) {
|
||||
scanOutputs.add(symbol);
|
||||
columnHandles.put(symbol, columns.get(aliasName));
|
||||
assignments.put(symbol, new VariableReferenceExpression(symbol.getName(), prestoType));
|
||||
}
|
||||
else {
|
||||
if (prestoType instanceof UnknownType) {
|
||||
log.debug("Can't cast from type[%s] to type[%s]", dcType.getDisplayName(), prestoType.getDisplayName());
|
||||
return Optional.empty();
|
||||
}
|
||||
// If Jdbc return a different type from Presto's expected type, add a CAST expression
|
||||
Symbol scanSymbol = symbolAllocator.newSymbol(symbol.getName(), dcType);
|
||||
scanOutputs.add(scanSymbol);
|
||||
columnHandles.put(scanSymbol, columns.get(aliasName));
|
||||
assignments.put(symbol, new CallExpression(
|
||||
CAST.name(),
|
||||
functionResolution.castFunction(prestoType.getTypeSignature(), dcType.getTypeSignature()),
|
||||
prestoType,
|
||||
ImmutableList.of(new VariableReferenceExpression(scanSymbol.getName(), dcType)),
|
||||
Optional.empty()));
|
||||
}
|
||||
}
|
||||
|
||||
checkState(context.getCatalogName().isPresent(), "CatalogName is null");
|
||||
checkState(context.getSchemaTableName().isPresent(), "schemaTableName is null");
|
||||
checkState(context.getTransaction().isPresent(), "transaction is null");
|
||||
CatalogName catalogName = context.getCatalogName().get();
|
||||
String tableCatalogName = catalogName.getCatalogName().startsWith(DATACENTER_CATALOG_PREFIX)
|
||||
? catalogName.getCatalogName().substring(DATACENTER_CATALOG_PREFIX.length())
|
||||
: catalogName.getCatalogName();
|
||||
|
||||
TableHandle newTableHandle = new TableHandle(
|
||||
catalogName,
|
||||
new DataCenterTableHandle(
|
||||
tableCatalogName,
|
||||
context.getSchemaTableName().get().getSchemaName(),
|
||||
context.getSchemaTableName().get().getTableName(),
|
||||
OptionalLong.empty(),
|
||||
sql),
|
||||
context.getTransaction().get(),
|
||||
Optional.empty());
|
||||
return Optional.of(
|
||||
new ProjectNode(
|
||||
this.idAllocator.getNextId(),
|
||||
new TableScanNode(
|
||||
idAllocator.getNextId(),
|
||||
newTableHandle,
|
||||
scanOutputs.build(),
|
||||
columnHandles.build(),
|
||||
TupleDomain.all(),
|
||||
Optional.empty(),
|
||||
ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT,
|
||||
new UUID(0, 0),
|
||||
0,
|
||||
false),
|
||||
new Assignments(assignments.build())));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.hetu.core.plugin.datacenter.optimization;
|
||||
|
||||
import io.hetu.core.plugin.datacenter.DataCenterColumnHandle;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterConfig;
|
||||
import io.hetu.core.plugin.datacenter.DataCenterTableHandle;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcQueryGenerator;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
|
||||
import io.prestosql.plugin.jdbc.optimization.BaseJdbcSqlStatementWriter;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
|
||||
import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorContext;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.connector.ColumnHandle;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.function.FunctionMetadataManager;
|
||||
import io.prestosql.spi.function.StandardFunctionResolution;
|
||||
import io.prestosql.spi.metadata.TableHandle;
|
||||
import io.prestosql.spi.plan.PlanNode;
|
||||
import io.prestosql.spi.plan.PlanVisitor;
|
||||
import io.prestosql.spi.plan.TableScanNode;
|
||||
import io.prestosql.spi.predicate.TupleDomain;
|
||||
import io.prestosql.spi.relation.DeterminismEvaluator;
|
||||
import io.prestosql.spi.relation.RowExpressionService;
|
||||
import io.prestosql.spi.sql.expression.Selection;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Optional;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_QUERY_GENERATOR_FAILURE;
|
||||
import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.quote;
|
||||
|
||||
public class DataCenterQueryGenerator
|
||||
extends BaseJdbcQueryGenerator
|
||||
{
|
||||
@Inject
|
||||
public DataCenterQueryGenerator(DataCenterConfig config, RowExpressionService rowExpressionService, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator)
|
||||
{
|
||||
super(new JdbcPushDownParameter("\"", false, config.getQueryPushDownModule(), functionResolution),
|
||||
new BaseJdbcRowExpressionConverter(functionManager, functionResolution, rowExpressionService, determinismEvaluator),
|
||||
new BaseJdbcSqlStatementWriter(new JdbcPushDownParameter("\"", false, config.getQueryPushDownModule(), functionResolution)));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected PlanVisitor<Optional<JdbcQueryGeneratorContext>, Void> getVisitor(TypeManager typeManager)
|
||||
{
|
||||
return new DataCenterPlanVisitor(typeManager);
|
||||
}
|
||||
|
||||
protected class DataCenterPlanVisitor
|
||||
extends BaseJdbcPlanVisitor
|
||||
{
|
||||
public DataCenterPlanVisitor(TypeManager typeManager)
|
||||
{
|
||||
super(typeManager);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<JdbcQueryGeneratorContext> visitPlan(PlanNode node, Void contextIn)
|
||||
{
|
||||
log.debug(GENERATE_FAILED_LOG, "Don't know how to handle plan node of type " + node);
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<JdbcQueryGeneratorContext> visitTableScan(TableScanNode node, Void contextIn)
|
||||
{
|
||||
checkAvailable(node);
|
||||
checkArgument(node.getTable().getConnectorHandle() instanceof DataCenterTableHandle,
|
||||
"Expected to find Data Center table handle for the scan node");
|
||||
TupleDomain<ColumnHandle> constraint = node.getEnforcedConstraint();
|
||||
if (constraint != null && constraint.getDomains().isPresent()) {
|
||||
if (!constraint.getDomains().get().isEmpty()) {
|
||||
// Predicate is pushed down
|
||||
throw new PrestoException(JDBC_QUERY_GENERATOR_FAILURE, "Cannot push down table scan with predicates pushed down");
|
||||
}
|
||||
}
|
||||
TableHandle tableHandle = node.getTable();
|
||||
DataCenterTableHandle dcTableHandle = (DataCenterTableHandle) node.getTable().getConnectorHandle();
|
||||
checkArgument(dcTableHandle.getPushDownSql().isEmpty(), "Data center should not have sql before pushdown");
|
||||
LinkedHashMap<String, Selection> selections = new LinkedHashMap<>();
|
||||
node.getOutputSymbols().forEach(outputColumn -> {
|
||||
DataCenterColumnHandle dcColumn = (DataCenterColumnHandle) node.getAssignments().get(outputColumn);
|
||||
selections.put(outputColumn.getName(), new Selection(dcColumn.getColumnName(), outputColumn.getName()));
|
||||
});
|
||||
StringBuilder table = new StringBuilder();
|
||||
if (!isNullOrEmpty(dcTableHandle.getCatalogName())) {
|
||||
table.append(quote(quote, dcTableHandle.getCatalogName())).append('.');
|
||||
}
|
||||
if (!isNullOrEmpty(dcTableHandle.getSchemaName())) {
|
||||
table.append(quote(quote, dcTableHandle.getSchemaName())).append('.');
|
||||
}
|
||||
table.append(quote(quote, dcTableHandle.getTableName()));
|
||||
|
||||
JdbcQueryGeneratorContext.Builder contextBuilder = new JdbcQueryGeneratorContext.Builder()
|
||||
.setCatalogName(Optional.of(tableHandle.getCatalogName()))
|
||||
.setTransaction(Optional.of(tableHandle.getTransaction()))
|
||||
.setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
|
||||
.setSelections(selections)
|
||||
.setFrom(Optional.of(table.toString()));
|
||||
|
||||
String catalogName = dcTableHandle.getCatalogName();
|
||||
if (catalogName != null) {
|
||||
contextBuilder.setRemoteCatalogName(catalogName);
|
||||
}
|
||||
String schemaName = dcTableHandle.getSchemaName();
|
||||
if (schemaName != null) {
|
||||
contextBuilder.setRemoteSchemaName(schemaName);
|
||||
}
|
||||
String tableName = dcTableHandle.getTableName();
|
||||
if (tableName != null) {
|
||||
contextBuilder.setRemoteTablename(tableName);
|
||||
}
|
||||
|
||||
// If LIMIT has been push down, add it to context
|
||||
if (dcTableHandle.getLimit().isPresent()) {
|
||||
contextBuilder.setLimit(dcTableHandle.getLimit());
|
||||
contextBuilder.setHasPushDown(true);
|
||||
}
|
||||
|
||||
return Optional.of(contextBuilder.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -23,9 +23,7 @@ import io.prestosql.spi.Page;
|
|||
import io.prestosql.spi.connector.ColumnHandle;
|
||||
import io.prestosql.spi.connector.ConnectorPageSource;
|
||||
import io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter;
|
||||
import io.prestosql.spi.dynamicfilter.CombinedDynamicFilter;
|
||||
import io.prestosql.spi.dynamicfilter.DynamicFilter;
|
||||
import io.prestosql.spi.dynamicfilter.DynamicFilterSupplier;
|
||||
import io.prestosql.spi.dynamicfilter.HashSetDynamicFilter;
|
||||
import okhttp3.OkHttpClient;
|
||||
|
||||
|
|
@ -34,9 +32,9 @@ import java.util.HashSet;
|
|||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Queue;
|
||||
import java.util.Set;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/**
|
||||
* Data center page source.
|
||||
|
|
@ -54,7 +52,7 @@ public class DataCenterPageSource
|
|||
private long readBytes;
|
||||
private long lastMemoryUsage;
|
||||
private Queue<Page> pages = new LinkedList<>();
|
||||
private final Optional<DynamicFilterSupplier> dynamicFilterSupplier;
|
||||
private final Supplier<Map<ColumnHandle, DynamicFilter>> dynamicFilterSupplier;
|
||||
private final Set<String> appliedDynamicFilters = new HashSet<>();
|
||||
|
||||
/**
|
||||
|
|
@ -73,7 +71,7 @@ public class DataCenterPageSource
|
|||
}
|
||||
|
||||
public DataCenterPageSource(OkHttpClient httpClient, DataCenterClientSession clientSession, String sql,
|
||||
String queryId, List<ColumnHandle> columns, Optional<DynamicFilterSupplier> dynamicFilterSupplier)
|
||||
String queryId, List<ColumnHandle> columns, Supplier<Map<ColumnHandle, DynamicFilter>> dynamicFilterSupplier)
|
||||
{
|
||||
this.startTime = System.nanoTime();
|
||||
this.client = DataCenterStatementClient.newStatementClient(httpClient, clientSession, sql, queryId);
|
||||
|
|
@ -102,10 +100,8 @@ public class DataCenterPageSource
|
|||
@Override
|
||||
public Page getNextPage()
|
||||
{
|
||||
if (dynamicFilterSupplier.isPresent() && !dynamicFilterSupplier.get().getDynamicFilters().isEmpty()) {
|
||||
/* applying only for the first map in the dynamic filter since we do not have
|
||||
more than one element as we do not expect disjuncts in this connector */
|
||||
applyDynamicFilters(dynamicFilterSupplier.get().getDynamicFilters().get(0));
|
||||
if (dynamicFilterSupplier != null) {
|
||||
applyDynamicFilters(dynamicFilterSupplier.get());
|
||||
}
|
||||
|
||||
if (!this.pages.isEmpty()) {
|
||||
|
|
@ -134,17 +130,8 @@ public class DataCenterPageSource
|
|||
BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromHashSetDynamicFilter((HashSetDynamicFilter) df);
|
||||
builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
|
||||
}
|
||||
else if (df instanceof CombinedDynamicFilter) {
|
||||
BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromCombinedDynamicFilter((CombinedDynamicFilter) df);
|
||||
if (bloomFilterDynamicFilter != null) {
|
||||
builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
|
||||
}
|
||||
}
|
||||
else if (df instanceof BloomFilterDynamicFilter) {
|
||||
builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Dynamic Filter (type: " + df.getClass().getSimpleName() + ") skipped for DC connector");
|
||||
builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
|
|
@ -27,15 +27,15 @@ import io.prestosql.spi.connector.ConnectorSession;
|
|||
import io.prestosql.spi.connector.ConnectorSplit;
|
||||
import io.prestosql.spi.connector.ConnectorTableHandle;
|
||||
import io.prestosql.spi.connector.ConnectorTransactionHandle;
|
||||
import io.prestosql.spi.dynamicfilter.DynamicFilterSupplier;
|
||||
import io.prestosql.spi.dynamicfilter.DynamicFilter;
|
||||
import io.prestosql.spi.type.TypeManager;
|
||||
import okhttp3.OkHttpClient;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.OptionalLong;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static java.util.stream.Collectors.joining;
|
||||
|
|
@ -49,14 +49,14 @@ public class DataCenterPageSourceProvider
|
|||
implements ConnectorPageSourceProvider
|
||||
{
|
||||
private static final int STRING_CAPACITY = 16;
|
||||
private static final String ENABLE_CROSS_REGION_DYNAMIC_FILTER = "cross_region_dynamic_filter_enabled";
|
||||
private static final String ENABLE_CROSS_REGION_DYNAMIC_FILTER = "cross-region-dynamic-filter-enabled";
|
||||
private static final String EXCHANGE_COMPRESSION = "exchange_compression";
|
||||
|
||||
private final DataCenterConfig config;
|
||||
|
||||
private final OkHttpClient httpClient;
|
||||
|
||||
private final TypeManager typeManager;
|
||||
private TypeManager typeManager;
|
||||
|
||||
/**
|
||||
* Constructor of data center page source provider.
|
||||
|
|
@ -90,7 +90,7 @@ public class DataCenterPageSourceProvider
|
|||
|
||||
sql.append(" FROM ");
|
||||
|
||||
if (tableHandler.getPushDownSql() == null || "".equals(tableHandler.getPushDownSql())) {
|
||||
if (tableHandler.getSubQuery() == null || "".equals(tableHandler.getSubQuery())) {
|
||||
if (!isNullOrEmpty(catalog)) {
|
||||
sql.append(catalog).append('.');
|
||||
}
|
||||
|
|
@ -101,7 +101,7 @@ public class DataCenterPageSourceProvider
|
|||
sql.append(table);
|
||||
}
|
||||
else {
|
||||
sql.append("(").append(tableHandler.getPushDownSql()).append(") pushdown");
|
||||
sql.append(tableHandler.getSubQuery());
|
||||
}
|
||||
|
||||
if (limit.isPresent()) {
|
||||
|
|
@ -114,13 +114,13 @@ public class DataCenterPageSourceProvider
|
|||
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session,
|
||||
ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns)
|
||||
{
|
||||
return createPageSource(transaction, session, split, table, columns, Optional.empty());
|
||||
return createPageSource(transaction, session, split, table, columns, null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session,
|
||||
ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns,
|
||||
Optional<DynamicFilterSupplier> dynamicFilterSupplier)
|
||||
Supplier<Map<ColumnHandle, DynamicFilter>> dynamicFilterSupplier)
|
||||
{
|
||||
// Build the sql
|
||||
String query = buildSql((DataCenterTableHandle) table, columns, ((DataCenterTableHandle) table).getLimit());
|
||||
|
|
@ -128,7 +128,7 @@ public class DataCenterPageSourceProvider
|
|||
Map<String, String> properties = new HashMap<>();
|
||||
|
||||
// Only set the session if there is any dynamic filter for this page source
|
||||
if (dynamicFilterSupplier != null && dynamicFilterSupplier.isPresent()) {
|
||||
if (dynamicFilterSupplier != null) {
|
||||
properties.put(ENABLE_CROSS_REGION_DYNAMIC_FILTER, "true");
|
||||
}
|
||||
if (config.isCompressionEnabled()) {
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue