!605 fix orc cache unit test

Merge pull request !605 from Jiahao/fix-orc-cache-ut

.gitignore

@@ -1,7 +1,6 @@
 *.iml
 *.ipr
 *.iws
-*.icloud
 target/
 /var
 /*/var/
@@ -28,4 +27,4 @@ node_modules
 */*.iml
 */.idea
 */.gitignore
-*/dependency-reduced-pom.xml
+*/dependency-reduced-pom.xml

OWNERS

@@ -7,4 +7,3 @@ approvers:
 - farhan3
 - fbird2020
 - lizheng920625
-- giteezhangjingfang

docker/container-test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved.
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

docker/default/etc/catalog/memory.properties

@@ -0,0 +1,14 @@
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+connector.name=memory

hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.7.0-SNAPSHOT</version>
+    <version>1.2.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>
@@ -34,7 +34,7 @@
     <httpcore.version>4.4.9</httpcore.version>
     <jacoco.append>true</jacoco.append>
     <carbon.version>2.0.1</carbon.version>
-    <scala.version>2.11.12</scala.version>
+    <scala.version>2.11.8</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <hadoop.version>3.2.0</hadoop.version>
     <spark.version>2.4.5</spark.version>
@@ -361,17 +361,13 @@
           <artifactId>bootstrap</artifactId>
           <groupId>io.airlift</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>jetty-util</artifactId>
-          <groupId>org.eclipse.jetty</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
 
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-compress</artifactId>
-      <version>1.21</version>
+      <version>1.19</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>
@@ -687,10 +683,6 @@
           <artifactId>parquet-hadoop-bundle</artifactId>
           <groupId>org.apache.parquet</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>groovy-all</artifactId>
-          <groupId>org.codehaus.groovy</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,7 +108,8 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this(metastore, null);
+            this.metastore = metastore;
+            this.schemaName = null;
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -232,6 +233,7 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
+        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,17 +73,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putShorts(inputRowId, count, value);
+            columnVector.putShorts(rowId, count, value);
         }
     }
 
@@ -98,17 +97,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putInts(inputRowId, count, value);
+            columnVector.putInts(rowId, count, value);
         }
     }
 
@@ -123,17 +121,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putLongs(inputRowId, count, value);
+            columnVector.putLongs(rowId, count, value);
         }
     }
 
@@ -148,12 +145,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -168,17 +164,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putDoubles(inputRowId, count, value);
+            columnVector.putDoubles(rowId, count, value);
         }
     }
 
@@ -201,12 +196,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -229,17 +223,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putNull(counter++);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putNulls(inputRowId, count);
+            columnVector.putNulls(rowId, count);
         }
     }
 
@@ -326,72 +319,66 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -18,7 +18,7 @@ import com.google.gson.Gson;
 import io.prestosql.plugin.hive.HiveACIDWriteType;
 import io.prestosql.plugin.hive.HiveFileWriter;
 import io.prestosql.plugin.hive.HiveType;
-import io.prestosql.plugin.hive.util.FieldSetterFactory;
+import io.prestosql.plugin.hive.HiveWriteUtils;
 import io.prestosql.spi.Page;
 import io.prestosql.spi.PrestoException;
 import io.prestosql.spi.block.Block;
@@ -64,7 +64,6 @@ import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.log4j.Logger;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -94,7 +93,6 @@ public class CarbondataFileWriter
 {
     private static final Logger LOG =
             LogServiceFactory.getLogService(CarbondataFileWriter.class.getName());
-    private static final io.airlift.log.Logger AIR_LOG = io.airlift.log.Logger.get(CarbondataFileWriter.class);
 
     private static final String LOAD_MODEL = "mapreduce.carbontable.load.model";
 
@@ -105,7 +103,7 @@ public class CarbondataFileWriter
     private final Object row;
     private final SettableStructObjectInspector tableInspector;
     private final List<StructField> structFields;
-    private final FieldSetterFactory.FieldSetter[] setters;
+    private final HiveWriteUtils.FieldSetter[] setters;
     private final Properties properties;
     private final Optional<AcidOutputFormat.Options> acidOptions;
     private final HiveACIDWriteType acidWriteType;
@@ -127,16 +125,15 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        Path localOutPutPath = paramOutPutPath;
-        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
+        this.outPutPath = requireNonNull(outPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            localOutPutPath = new Path(properties.getProperty("location"));
+            outPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -146,7 +143,6 @@ public class CarbondataFileWriter
                 Long.toString(System.currentTimeMillis()));
         this.taskId = taskId.orElseGet(() -> 0);
 
-        AIR_LOG.debug("[carbonWriterTask] taskId: " + this.taskId + ", outputPath: " + this.outPutPath);
         try {
             if (HiveACIDWriteType.isUpdateOrDelete(this.acidWriteType)) {
                 String encodedCarbonTable = configuration.get(CarbondataConstants.CarbonTable);
@@ -185,16 +181,13 @@ public class CarbondataFileWriter
 
         row = tableInspector.create();
 
-        setters = new FieldSetterFactory.FieldSetter[structFields.size()];
-
-        FieldSetterFactory fieldSetterFactory = new FieldSetterFactory(DateTimeZone.UTC);
-
+        setters = new HiveWriteUtils.FieldSetter[structFields.size()];
         for (int i = 0; i < setters.length; i++) {
-            setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i),
+            setters[i] = HiveWriteUtils.createFieldSetter(tableInspector, row, structFields.get(i),
                     fileColumnTypes.get(structFields.get(i).getFieldID()));
         }
 
-        if (this.acidWriteType == HiveACIDWriteType.INSERT || this.acidWriteType == HiveACIDWriteType.INSERT_OVERWRITE) {
+        if (this.acidWriteType == HiveACIDWriteType.INSERT | this.acidWriteType == HiveACIDWriteType.INSERT_OVERWRITE) {
             try {
                 boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
 
@@ -212,7 +205,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -227,25 +220,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path finalOutPutPath = this.outPutPath;
-        Properties finalProperties = this.properties;
-        JobConf finalConfiguration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
+        Path outPutPath = this.outPutPath;
+        Properties properties = this.properties;
+        JobConf configuration = this.configuration;
+        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
-                        finalProperties, Reporter.NULL);
+                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
+                        properties, Reporter.NULL);
     }
 
     @Override
@@ -279,14 +272,14 @@ public class CarbondataFileWriter
     private String getUpdateTupleIdFromRec(Page dataPage, int position)
     {
         Block tupleIdBlock = null;
-        tupleIdBlock = dataPage.getBlock(dataPage.getChannelCount() - 1).getLoadedBlock();
+        tupleIdBlock = dataPage.getBlock(fieldCount).getLoadedBlock();
 
         return tupleIdBlock.getString(position, 0, 0);
     }
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter finalRecordWriter = null;
+        FileSinkOperator.RecordWriter recordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -335,7 +328,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -352,7 +345,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            finalRecordWriter = this.recordWriter;
+            recordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -366,8 +359,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (finalRecordWriter != null) {
-                finalRecordWriter.write(serDe.serialize(row, tableInspector));
+            if (recordWriter != null) {
+                recordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriterFactory.java

@@ -30,7 +30,6 @@ import org.apache.hadoop.mapred.JobConf;
 
 import java.util.List;
 import java.util.Optional;
-import java.util.OptionalInt;
 import java.util.Properties;
 
 import static java.util.Objects.requireNonNull;
@@ -68,19 +67,10 @@ public class CarbondataFileWriterFactory
                                                      Optional<HiveACIDWriteType> acidWriteType)
     {
         try {
-            int taskId = session.getTaskId().getAsInt();
-            int driverId = session.getDriverId().getAsInt();
-            int taskWriterCount = session.getTaskWriterCount();
-            //taskId starts from 0.
-            //driverId starts from 0 and will be < taskWriterCount.
-            //taskWriterCount starts from 1
-            // for taskId n, buckets will be between n*taskWriterCount (inclusive) and (n+1)*taskWriterCount (exclusive)
-            int bucketNumber = taskId * taskWriterCount + driverId;
-
             /* Create a DeleteDeltaFileWriter */
             return Optional
                     .of(new CarbondataFileWriter(path, inputColumnNames, schema, configuration,
-                            typeManager, acidOptions, acidWriteType, OptionalInt.of(bucketNumber)));
+                            typeManager, acidOptions, acidWriteType, session.getTaskId()));
         }
         catch (SerDeException e) {
             throw new RuntimeException("Error while creating carbon file writer", e);

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,7 +48,6 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
-    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuFilterUtil.java

@@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
             if (rawData instanceof Slice) {
                 String value = ((Slice) rawData).toStringUtf8();
                 if (type.getTypeInfo() instanceof CharTypeInfo) {
-                    StringBuilder padding = new StringBuilder();
+                    String padding = "";
                     int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
                     int truncatedLength = value.length();
                     for (int i = 0; i < paddedLength - truncatedLength; i++) {
-                        padding.append(" ");
+                        padding += " ";
                     }
                     return value + padding;
                 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuOutputFormat.java

@@ -103,6 +103,7 @@ public class CarbondataHetuOutputFormat<T>
         OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
         JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
         carbonOutputCommitter.setupJob(jobContext);
+        CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
         org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
         return new FileSinkOperator.RecordWriter()
         {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataLocationService.java

@@ -80,6 +80,8 @@ public class CarbondataLocationService
     {
         // TODO: check and make it compatible for cloud scenario
 
+        HdfsEnvironment.HdfsContext context =
+                new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
         Path targetPath = new Path(table.getStorage().getLocation());
 
         return new LocationHandle(targetPath, targetPath, true,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -21,7 +21,6 @@ import io.airlift.log.Logger;
 import io.airlift.units.Duration;
 import io.hetu.core.plugin.carbondata.impl.CarbondataTableReader;
 import io.prestosql.plugin.hive.ForHive;
-import io.prestosql.plugin.hive.ForHiveMetastore;
 import io.prestosql.plugin.hive.ForHiveTransactionHeartbeats;
 import io.prestosql.plugin.hive.HdfsEnvironment;
 import io.prestosql.plugin.hive.HiveMetadata;
@@ -37,6 +36,7 @@ import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
 import io.prestosql.plugin.hive.security.AccessControlMetadataFactory;
 import io.prestosql.plugin.hive.statistics.MetastoreHiveStatisticsProvider;
 import io.prestosql.spi.type.TypeManager;
+import org.joda.time.DateTimeZone;
 
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
@@ -49,6 +49,7 @@ public class CarbondataMetadataFactory
         extends HiveMetadataFactory
 {
     private static final Logger log = Logger.get(HiveMetadataFactory.class);
+    private final boolean allowCorruptWritesForTesting;
     private final boolean skipDeletionForAlter;
     private final boolean skipTargetCleanupOnRollback;
     private final boolean writesToNonManagedTablesEnabled;
@@ -58,11 +59,11 @@ public class CarbondataMetadataFactory
     private final HiveMetastore metastore;
     private final HdfsEnvironment hdfsEnvironment;
     private final HivePartitionManager partitionManager;
+    private final DateTimeZone timeZone;
     private final TypeManager typeManager;
     private final LocationService locationService;
     private final BoundedExecutor renameExecution;
     private final ScheduledExecutorService vacuumExecutorService;
-    private final ScheduledExecutorService hiveMetastoreClientService;
     private final TypeTranslator typeTranslator;
     private final String hetuVersion;
     private final AccessControlMetadataFactory accessControlMetadataFactory;
@@ -82,7 +83,6 @@ public class CarbondataMetadataFactory
                                      @ForHive ExecutorService executorService,
                                      @ForCarbonVacuum ScheduledExecutorService vacuumExecutorService,
                                      @ForHiveTransactionHeartbeats ScheduledExecutorService heartbeatService,
-                                     @ForHiveMetastore ScheduledExecutorService hiveMetastoreClientService,
                                      TypeManager typeManager, LocationService locationService,
                                      JsonCodec<PartitionUpdate> partitionUpdateCodec,
                                      JsonCodec<CarbondataSegmentInfoUtil> segmentInfoCodec,
@@ -90,8 +90,9 @@ public class CarbondataMetadataFactory
                                      AccessControlMetadataFactory accessControlMetadataFactory,
                                      CarbondataTableReader carbondataTableReader)
     {
-        this(metastore, hdfsEnvironment, partitionManager,
+        this(metastore, hdfsEnvironment, partitionManager, carbondataConfig.getDateTimeZone(),
                 carbondataConfig.getMaxConcurrentFileRenames(),
+                carbondataConfig.getAllowCorruptWritesForTesting(),
                 carbondataConfig.isSkipDeletionForAlter(),
                 carbondataConfig.isSkipTargetCleanupOnRollback(),
                 true,
@@ -101,16 +102,16 @@ public class CarbondataMetadataFactory
                 carbondataConfig.getHiveTransactionHeartbeatInterval(),
                 carbondataConfig.getVacuumCleanupRecheckInterval(), typeManager, locationService,
                 partitionUpdateCodec, segmentInfoCodec, executorService,
-                vacuumExecutorService, heartbeatService, hiveMetastoreClientService, typeTranslator, nodeVersion.toString(),
+                vacuumExecutorService, heartbeatService, typeTranslator, nodeVersion.toString(),
                 accessControlMetadataFactory, carbondataTableReader, carbondataConfig.getStoreLocation(),
                 carbondataConfig.getMajorVacuumSegSize(), carbondataConfig.getMinorVacuumSegCount(),
-                carbondataConfig.getAutoVacuumEnable(), carbondataConfig.getMetastoreWriteBatchSize());
+                carbondataConfig.getAutoVacuumEnable());
     }
 
     public CarbondataMetadataFactory(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment,
-                                     HivePartitionManager partitionManager,
+                                     HivePartitionManager partitionManager, DateTimeZone timeZone,
                                      int maxConcurrentFileRenames,
-                                     boolean skipDeletionForAlter,
+                                     boolean allowCorruptWritesForTesting, boolean skipDeletionForAlter,
                                      boolean skipTargetCleanupOnRollback, boolean writesToNonManagedTablesEnabled,
                                      boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
                                      long perTransactionCacheMaximumSize,
@@ -120,16 +121,17 @@ public class CarbondataMetadataFactory
                                      JsonCodec<PartitionUpdate> partitionUpdateCodec,
                                      JsonCodec<CarbondataSegmentInfoUtil> segmentInfoCodec,
                                      ExecutorService executorService, ScheduledExecutorService vacuumExecutorService, ScheduledExecutorService heartbeatService,
-                                     ScheduledExecutorService hiveMetastoreClientService,
                                      TypeTranslator typeTranslator, String hetuVersion,
                                      AccessControlMetadataFactory accessControlMetadataFactory,
                                      CarbondataTableReader carbondataTableReader, String storeLocation, long majorVacuumSegSize, long minorVacuumSegCount,
-                                     boolean autoVacuumEnable, int hmsWriteBatchSize)
+                                     boolean autoVacuumEnable)
     {
         super(metastore,
                 hdfsEnvironment,
                 partitionManager,
+                timeZone,
                 maxConcurrentFileRenames,
+                allowCorruptWritesForTesting,
                 skipDeletionForAlter,
                 skipTargetCleanupOnRollback,
                 writesToNonManagedTablesEnabled,
@@ -144,13 +146,11 @@ public class CarbondataMetadataFactory
                 executorService,
                 vacuumExecutorService,
                 heartbeatService,
-                hiveMetastoreClientService,
                 typeTranslator,
                 hetuVersion,
                 accessControlMetadataFactory,
-                2, 0.0, false,
-                Optional.of(new Duration(5, TimeUnit.MINUTES)),
-                hmsWriteBatchSize);
+                2, 0.0, false, Optional.of(new Duration(5, TimeUnit.MINUTES)));
+        this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
         this.skipDeletionForAlter = skipDeletionForAlter;
         this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
         this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled;
@@ -160,6 +160,7 @@ public class CarbondataMetadataFactory
         this.metastore = requireNonNull(metastore, "metastore is null");
         this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
         this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
+        this.timeZone = requireNonNull(timeZone, "timeZone is null");
         this.typeManager = requireNonNull(typeManager, "typeManager is null");
         this.locationService = requireNonNull(locationService, "locationService is null");
         this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
@@ -168,9 +169,15 @@ public class CarbondataMetadataFactory
         this.hetuVersion = requireNonNull(hetuVersion, "hetuVersion is null");
         this.accessControlMetadataFactory = requireNonNull(accessControlMetadataFactory,
                 "accessControlMetadataFactory is null");
+        if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
+            log.warn(
+                    "Hive writes are disabled. To write data to Hive, your JVM timezone must match the " +
+                            "Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments",
+                    timeZone.getID());
+        }
+
         this.renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
         this.vacuumExecutorService = requireNonNull(vacuumExecutorService, "vacuumExecutorService is null");
-        this.hiveMetastoreClientService = requireNonNull(hiveMetastoreClientService, "hiveMetastoreClientService is null");
         this.vacuumCleanupInterval = requireNonNull(vacuumCleanupInterval,
                 "vacuumCleanupInterval is null");
         this.hiveTransactionHeartbeatInterval = requireNonNull(hiveTransactionHeartbeatInterval,
@@ -191,18 +198,20 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
+        SemiTransactionalHiveMetastore metastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
                         vacuumExecutorService, this.vacuumCleanupInterval, this.skipDeletionForAlter,
                         this.skipTargetCleanupOnRollback,
                         this.hiveTransactionHeartbeatInterval,
-                        this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
+                        this.heartbeatService);
 
-        return new CarbondataMetadata(semiTransactionalHiveMetastore,
+        return new CarbondataMetadata(metastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
+                this.timeZone,
+                this.allowCorruptWritesForTesting,
                 this.writesToNonManagedTablesEnabled,
                 this.createsOfNonManagedTablesEnabled,
                 this.tableCreatesWithLocationAllowed,
@@ -212,13 +221,12 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
+                new MetastoreHiveStatisticsProvider(metastore),
+                this.accessControlMetadataFactory.create(metastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,
                 this.carbondataMinorVacuumSegmentCount,
-                vacuumExecutorService,
-                hiveMetastoreClientService);
+                vacuumExecutorService);
     }
 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        finalHdfsEnvironment.doAs(session.getUser(), () -> {
+        hdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,7 +163,6 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
-    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;
@@ -184,7 +183,6 @@ public class CarbondataPageSinkProvider
                 vacuumTableHandle.getTableStorageFormat(),
                 vacuumTableHandle.getPartitionStorageFormat(),
                 vacuumTableHandle.isFullVacuum(),
-                vacuumTableHandle.isUnifyVacuum(),
                 null);
         return createPageSink(hiveVacuumTableHandle, session, HiveACIDWriteType.VACUUM, ImmutableMap.of(), vacuumTableHandle.getAdditionalConf(), false);
     }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int columnBatchSize = 0;
+        int batchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    columnBatchSize = columnarBatch.numRows();
-                    if (columnBatchSize == 0) {
+                    batchSize = columnarBatch.numRows();
+                    if (batchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(columnBatchSize, blocks);
+            Page page = new Page(batchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataSplitManager.java

@@ -14,7 +14,6 @@
 package io.hetu.core.plugin.carbondata;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.impl.CarbondataLocalInputSplit;
 import io.hetu.core.plugin.carbondata.impl.CarbondataLocalMultiBlockSplit;
@@ -140,7 +139,7 @@ public class CarbondataSplitManager
     @Override
     public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle,
             ConnectorSession session, ConnectorTableHandle tableHandle,
-            SplitSchedulingStrategy splitSchedulingStrategy, Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier,
+            SplitSchedulingStrategy splitSchedulingStrategy, Supplier<Set<DynamicFilter>> dynamicFilterSupplier,
             Optional<QueryType> queryType, Map<String, Object> queryProperties,
             Set<TupleDomain<ColumnMetadata>> userDefinedCachePredicates,
             boolean partOfReuse)
@@ -201,7 +200,7 @@ public class CarbondataSplitManager
                             0, 0, 0, 0,
                             properties, new ArrayList(), getHostAddresses(split.getLocations()),
                             OptionalInt.empty(), false, new HashMap<>(),
-                            Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
+                            Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
                     /* Todo: Make this part aligned with rest of the HiveSlipt loading flow...
                      *   and figure out how to pass valid transaction Ids to CarbonData? */
                 }
@@ -310,7 +309,7 @@ public class CarbondataSplitManager
                         schemaTableName.getTableName(), tablePath, 0L, 0L, 0L, 0L,
                         properties, new ArrayList(), getHostAddresses(currSplit.getLocations()),
                         OptionalInt.empty(), false, new HashMap<>(),
-                        Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
+                        Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
             }
         }
         LOGGER.info("Splits for compaction built and ready");

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableHandle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -42,10 +42,4 @@ public class CarbondataTableHandle
     {
         return false;
     }
-
-    @Override
-    public boolean isUpdateAsInsertSupported()
-    {
-        return true;
-    }
 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,17 +90,16 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
-        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            finalName = name.substring(0, name.length() - 4).trim();
+            name = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            finalName = name.substring(0, name.length() - 5).trim();
+            name = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(finalName, order);
+        return new SortingColumn(name, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataVacuumTableHandle.java

@@ -61,7 +61,6 @@ public class CarbondataVacuumTableHandle
                 tableStorageFormat,
                 partitionStorageFormat,
                 full,
-                false,
                 null);
         // Additional conf is used to store the encoded load model
         this.additionalConf = ImmutableMap.copyOf(requireNonNull(additionalConf, "additional conf map is null"));

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -31,6 +31,7 @@ import io.prestosql.plugin.hive.OrcFileWriterFactory;
 import io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider;
 import io.prestosql.plugin.hive.metastore.SortingColumn;
 import io.prestosql.spi.NodeManager;
+import io.prestosql.spi.Page;
 import io.prestosql.spi.PageSorter;
 import io.prestosql.spi.connector.ConnectorSession;
 import io.prestosql.spi.type.TypeManager;
@@ -47,7 +48,6 @@ import java.util.Set;
 
 import static java.util.Objects.requireNonNull;
 import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION;
-import static org.joda.time.DateTimeZone.UTC;
 
 public class CarbondataWriterFactory
         extends HiveWriterFactory
@@ -88,7 +88,7 @@ public class CarbondataWriterFactory
                 additionalTableParameters, bucketCount, sortedBy, locationHandle,
                 locationService, queryId, pageSinkMetadataProvider,
                 typeManager, hdfsEnvironment, pageSorter, sortBufferSize,
-                maxOpenSortFiles, immutablePartitions, UTC, session, nodeManager,
+                maxOpenSortFiles, immutablePartitions, session, nodeManager,
                 eventClient, hiveSessionProperties, hiveWriterStats, orcFileWriterFactory);
 
         this.additionalJobConf = requireNonNull(additionalJobConf, "Additional JobConf is null");
@@ -120,20 +120,19 @@ public class CarbondataWriterFactory
     }
 
     @Override
-    public HiveWriter createWriter(List<String> partitionValues, OptionalInt bucketNumber, Optional<Options> vacuumOptions)
+    public HiveWriter createWriter(Page partitionColumns, int position, OptionalInt bucketNumber, Optional<Options> vacuumOptions)
     {
         /* set Additional JobConf */
         JobConf jobConf = getSuperJobConf();
 
         additionalJobConf.forEach((k, v) -> jobConf.set(k, v));
-        return super.createWriter(partitionValues, bucketNumber, vacuumOptions);
+        return super.createWriter(partitionColumns, position, bucketNumber, vacuumOptions);
     }
 
     protected void checkWriteMode(LocationService.WriteInfo writeInfo)
     {
     }
 
-    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,15 +143,14 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDecimal(inputRowId, value, precision);
+                columnVector.putDecimal(rowId, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -186,9 +185,8 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(inputRowId++, value);
+            columnVector.putByteArray(rowId++, value);
         }
     }
 
@@ -305,90 +303,84 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putFloat(inputRowId, src[i]);
+                columnVector.putFloat(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putShort(inputRowId, src[i]);
+                columnVector.putShort(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putInt(inputRowId, src[i]);
+                columnVector.putInt(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putLong(inputRowId, src[i]);
+                columnVector.putLong(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDouble(inputRowId, src[i]);
+                columnVector.putDouble(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putByte(inputRowId, src[i]);
+                columnVector.putByte(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/impl/CarbondataTableReader.java

@@ -56,10 +56,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.log4j.Logger;
 import org.apache.thrift.TBase;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -142,19 +139,10 @@ public class CarbondataTableReader
                 carbonTableCacheModel.getCarbonTable().isTransactionalTable()) {
             CarbonTable carbonTable = carbonTableCacheModel.getCarbonTable();
             try {
-                String schemaFilePath = CarbonTablePath
-                        .getSchemaFilePath(
-                                carbonTable.getTablePath());
-                FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
-
-                CarbonFile carbonFile = FileFactory.getCarbonFile(schemaFilePath,
-                        config);
-
-                long latestTime = carbonFile.getLastModifiedTime();
-                // Workaround for cache updation error due to round off of latestTime inside LocalCarbonFile
-                if (fileType == FileFactory.FileType.LOCAL) {
-                    latestTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
-                }
+                long latestTime = FileFactory.getCarbonFile(CarbonTablePath
+                                .getSchemaFilePath(
+                                        carbonTable.getTablePath()),
+                        config).getLastModifiedTime();
                 carbonTableCacheModel.setCurrentSchemaTime(latestTime);
                 if (!carbonTableCacheModel.isValid()) {
                     // Invalidate datamaps
@@ -162,7 +150,7 @@ public class CarbondataTableReader
                             .clearIndex(carbonTableCacheModel.getCarbonTable().getAbsoluteTableIdentifier());
                 }
             }
-            catch (CarbonFileException | IOException e) {
+            catch (CarbonFileException e) {
                 carbonCache.get().remove(schemaTableName);
             }
         }
@@ -192,7 +180,7 @@ public class CarbondataTableReader
                 return cache;
             }
             // multiple tasks can be launched in a worker concurrently. Hence need to synchronize this.
-            synchronized (CarbondataTableReader.class) {
+            synchronized (this) {
                 // cache might be filled by another thread, so if filled use that cache.
                 CarbondataTableCacheModel cacheModel = getValidCacheBySchemaTableName(table);
                 if (cacheModel != null) {
@@ -223,11 +211,6 @@ public class CarbondataTableReader
                     tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
                     thriftReader.close();
                     modifiedTime = schemaFile.getLastModifiedTime();
-                    FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
-                    // Workaround for cache updation error due to round off of latesttime inside LocalCarbonFile
-                    if (fileType == FileFactory.FileType.LOCAL) {
-                        modifiedTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
-                    }
                 }
                 else {
                     tableInfo = CarbonUtil.inferSchema(tablePath, table.getTableName(), false, config);
@@ -340,7 +323,7 @@ public class CarbondataTableReader
                             inputSplits.get(j).stream().flatMap(f -> Arrays.stream(getLocations(f))).distinct()
                                     .toArray(String[]::new)));
                 }
-                LOGGER.error("Size of MultiblockList   " + multiBlockSplitList.size());
+                LOGGER.error("Size fo MultiblockList   " + multiBlockSplitList.size());
             }
         }
         catch (IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,9 +58,8 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIdx++] == 1);
+            type.writeBoolean(builder, src[srcIndex++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,9 +76,8 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(id++, value, precision);
+            putDecimal(rowId++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,9 +58,8 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(id++, value);
+            putInt(rowId++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -40,7 +40,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
 
 public class TestCarbonAutoVacuum
 {
@@ -100,6 +99,7 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
+        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -145,12 +145,11 @@ public class TestCarbonAutoVacuum
 
         try {
             CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
-            assertNotNull(catalog);
             connector = catalog.getConnector(catalog.getConnectorCatalogName());
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -216,7 +215,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -42,8 +42,8 @@ import java.util.Map;
 
 import static org.testng.Assert.assertEquals;
 
-public class TestCarbondataAutoCleanup
-{
+public class TestCarbondataAutoCleanup {
+
     private final Logger logger = LogServiceFactory.getLogService(TestCarbondataAutoCleanup.class.getCanonicalName());
 
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
@@ -62,6 +62,7 @@ public class TestCarbondataAutoCleanup
     public void setup() throws Exception
     {
         logger.info("Setup begin: " + this.getClass().getSimpleName());
+        String dataPath = rootPath + "/src/test/resources/alldatatype.csv";
 
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
@@ -76,8 +77,8 @@ public class TestCarbondataAutoCleanup
         map.put("carbondata.minor-vacuum-seg-count", "4");
         map.put("carbondata.major-vacuum-seg-size", "1");
 
-        if (!FileFactory.isFileExist(storePath + "/carbon.store")) {
-            FileFactory.mkdirs(storePath + "/carbon.store");
+        if (!FileFactory.isFileExist( storePath + "/carbon.store")) {
+            FileFactory.mkdirs( storePath + "/carbon.store");
         }
 
         hetuServer.startServer("testdb", map);
@@ -89,7 +90,6 @@ public class TestCarbondataAutoCleanup
         hetuServer.execute("drop table if exists testdb.testtableautocleanup6");
         hetuServer.execute("drop table if exists testdb.testtableautocleanup7");
         hetuServer.execute("drop table if exists testdb.testtableautocleanup8");
-        hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
         hetuServer.execute("drop schema if exists testdb");
         hetuServer.execute("drop schema if exists default");
         hetuServer.execute("create schema testdb");
@@ -130,7 +130,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -159,7 +159,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -189,7 +189,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -201,32 +201,32 @@ public class TestCarbondataAutoCleanup
     {
         try {
             hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = true");
-            hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
-            hetuServer.execute("CREATE TABLE testdb.testtableautocleanupwithpushdown (a int, b int)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (10, 11)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (110, 211)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (120, 311)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 411)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 511)");
-            hetuServer.execute("vacuum table testdb.testtableautocleanupwithpushdown AND WAIT");
+            hetuServer.execute("drop table if exists testdb.testtableautocleanup3");
+            hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)");
+            hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT");
 
-            reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Metadata");
+            reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata");
             CarbondataMetadata.enableTracingCleanupTask(true);
 
-            hetuServer.execute("DELETE FROM testdb.testtableautocleanupwithpushdown WHERE a=130");
+            hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130");
             try {
                 CarbondataMetadata.waitForSubmittedTasksFinish();
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_0", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_1", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-                logger.debug(exception.getMessage());
+
             }
         }
         finally {
             CarbondataMetadata.enableTracingCleanupTask(false);
-            hetuServer.execute("drop table testdb.testtableautocleanupwithpushdown");
+            hetuServer.execute("drop table testdb.testtableautocleanup3");
             hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = false");
         }
     }
@@ -254,7 +254,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -285,7 +285,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -314,7 +314,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -344,7 +344,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -374,7 +374,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -403,7 +403,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -15,7 +15,6 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -31,13 +30,11 @@ import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
-    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -66,9 +63,7 @@ public class TestsWithHiveConnector
         }
 
         hetuServer.startServer("default", map);
-        hetuServer.execute("drop schema if exists default");
         hetuServer.execute("create schema default");
-        hetuServer.addHiveCatalogToQueryRunner(createHiveProperties());
     }
 
     @AfterClass
@@ -82,6 +77,7 @@ public class TestsWithHiveConnector
     public void block_Hive_Table_from_Carbondata()
             throws SQLException
     {
+        hetuServer.addHiveCatalogToQueryRunner(createHiveProperties());
         hetuServer.execute("CREATE TABLE hive.default.demotable (c1 int)");
 
         hetuServer.execute("use carbondata.default");
@@ -101,134 +97,24 @@ public class TestsWithHiveConnector
         hetuServer.execute("DROP TABLE hive.default.demotable");
     }
 
-    @Test
-   public void deleteTransactionTableDirc()
-            throws SQLException
-    {
-        hetuServer.execute("drop table if exists hive.default.parttable");
-        hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = true");
-        hetuServer.execute("create table hive.default.parttable (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable values (3,2013)");
-        hetuServer.execute("delete from hive.default.parttable where year =2013");
-        try {
-        assertEquals(FileFactory.isFileExist(storePath +
-                "hive.store/default/parttable/year=2013", false), false);
-        } catch (IOException exception) {
-            log.error(exception.getMessage());
-        }
-        hetuServer.execute("DROP TABLE hive.default.parttable");
-    }
-
-    @Test
-    public void deleteTransactionTableDirUsing2tables()
-            throws SQLException
-    {
-        hetuServer.execute("drop table if exists hive.default.parttable2");
-        hetuServer.execute("drop table if exists hive.default.parttable1");
-        hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = true");
-        hetuServer.execute("create table hive.default.parttable2 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable2 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable2 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable2 values (3,2013)");
-
-        hetuServer.execute("create table hive.default.parttable1 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable1 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable1 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable1 values (3,2013)");
-
-        hetuServer.execute("delete from hive.default.parttable2 where year >= (select max(year) from hive.default.parttable1)  ");
-        try {
-            assertEquals(FileFactory.isFileExist(storePath +
-                    "hive.store/default/parttable2/year=2013", false), false);
-        } catch (IOException exception) {
-            log.error(exception.getMessage());
-        }
-
-        hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
-        hetuServer.execute("delete from hive.default.parttable2 where year >= (select year from hive.default.parttable1 where orderkey=4 )  ");
-        try {
-            assertEquals(FileFactory.isFileExist(storePath +
-                    "hive.store/default/parttable2/year=2014", false), false);
-        } catch (IOException exception) {
-            log.error(exception.getMessage());
-        }
-
-        hetuServer.execute("DROP TABLE hive.default.parttable2");
-        hetuServer.execute("DROP TABLE hive.default.parttable1");
-    }
-
-    @Test
-    public void deleteTransactionTableDirDisable()
-            throws SQLException
-    {
-        hetuServer.execute("drop table if exists hive.default.parttable3");
-        hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = false");
-        hetuServer.execute("create table hive.default.parttable3 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable3 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable3 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable3 values (3,2013)");
-
-        hetuServer.execute("create table hive.default.parttable4 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable4 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable4 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable4 values (3,2013)");
-
-        hetuServer.execute("delete from hive.default.parttable3 where year >= (select max(year) from hive.default.parttable4)  ");
-        try {
-            assertEquals(FileFactory.isFileExist(storePath +
-                    "/hive.store/default/parttable3/year=2013", false), true);
-        } catch (IOException exception) {
-            log.error(exception.getMessage());
-        }
-        hetuServer.execute("DROP TABLE hive.default.parttable3");
-    }
-
     @Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
     public void block_Carbondata_Table_from_Hive()
             throws SQLException
     {
-        hetuServer.execute("CREATE TABLE carbondata.default.demotable1 (c1 int)");
+        hetuServer.execute("CREATE TABLE carbondata.default.demotable (c1 int)");
 
         hetuServer.execute("use hive.default");
 
-        runQueryAndAssertErrorMessage("SELECT * FROM demotable1",
+        runQueryAndAssertErrorMessage("SELECT * FROM demotable",
                 "Hive connector can't read carbondata tables");
 
-        runQueryAndAssertErrorMessage("INSERT INTO demotable1 VALUES(1)",
+        runQueryAndAssertErrorMessage("INSERT INTO demotable VALUES(1)",
                 "Tables with MapredCarbonInputFormat are not supported by Hive connector");
 
-        runQueryAndAssertErrorMessage("UPDATE demotable1 SET c1=1",
+        runQueryAndAssertErrorMessage("UPDATE demotable SET c1=1",
                 "Tables with MapredCarbonInputFormat are not supported by Hive connector");
 
-        hetuServer.execute("use carbondata.default");
-        hetuServer.execute("DROP TABLE carbondata.default.demotable1");
-    }
-
-    @Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
-    public void block_Carbondata_Table_change_from_Hive()
-            throws SQLException
-    {
-        hetuServer.execute("CREATE TABLE carbondata.default.changetable (a int, b int, c int)");
-        hetuServer.execute("use hive.default");
-        //rename column
-        runQueryAndAssertErrorMessage("ALTER TABLE default.changetable RENAME COLUMN a TO aa",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //add column
-        runQueryAndAssertErrorMessage("ALTER TABLE default.changetable ADD COLUMN e int COMMENT 'Hello World'",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //drop column
-        runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable DROP COLUMN c",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //change table name
-        runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable RENAME TO hive.default.changetable_N",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //drop table
-        runQueryAndAssertErrorMessage("DROP TABLE default.changetable",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        hetuServer.execute("use carbondata.default");
-        hetuServer.execute("DROP TABLE carbondata.default.changetable");
+        hetuServer.execute("DROP TABLE hive.default.demotable");
     }
 
     private Map<String, String> createHiveProperties()
@@ -237,11 +123,6 @@ public class TestsWithHiveConnector
         hiveProperties.put("hive.metastore", "file");
         hiveProperties.put("hive.allow-drop-table", "true");
         hiveProperties.put("hive.non-managed-table-writes-enabled", "true");
-        hiveProperties.put("hive.allow-add-column", "true");
-        hiveProperties.put("hive.allow-drop-column", "true");
-        hiveProperties.put("hive.allow-rename-table", "true");
-        hiveProperties.put("hive.allow-comment-table", "true");
-        hiveProperties.put("hive.allow-rename-column", "true");
         hiveProperties.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store");
         return hiveProperties;
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -111,8 +111,7 @@ public class HetuTestServer
         boolean result = false;
         try {
             result = statement.execute(query);
-        }
-        catch (SQLException e) {
+        } catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
@@ -123,20 +122,13 @@ public class HetuTestServer
     public List<Map<String, Object>> executeQuery(String query) throws SQLException
     {
         logger.info(">>>>> Executing Query: " + query);
-        ResultSet rs = null;
         try {
-            rs = statement.executeQuery(query);
+            ResultSet rs = statement.executeQuery(query);
             return convertResultSetToList(rs);
-        }
-        catch (SQLException e) {
+        } catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
-        finally {
-            if (rs != null) {
-                rs.close();
-            }
-        }
     }
 
     private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
@@ -175,8 +167,7 @@ public class HetuTestServer
 
         if (StringUtils.isEmpty(dbName)) {
             url = "jdbc:presto://localhost:" + port + "/carbondata/default";
-        }
-        else {
+        } else {
             url = "jdbc:presto://localhost:" + port + "/carbondata/" + dbName;
         }
 
@@ -192,21 +183,20 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
             Map<String, String> carbonPropertiesLocationDisabled = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
-                    .put("hive.table-creates-with-location-allowed", "false")
+                    .put("hive.table-creates-with-location-allowed","false")
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
-        }
-        catch (RuntimeException e) {
+        } catch (RuntimeException e) {
             queryRunner.close();
             throw e;
         }
@@ -220,8 +210,7 @@ public class HetuTestServer
         queryRunner.createCatalog("hive", "hive", hiveProperties);
     }
 
-    public CatalogManager getCatalog()
-    {
+    public CatalogManager getCatalog() {
         return queryRunner.getCatalogManager();
     }
 }

hetu-clickhouse/pom.xml

@@ -1,167 +0,0 @@
-<?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>io.hetu.core</groupId>
-        <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>hetu-clickhouse</artifactId>
-    <description>hetu - ClickHouse Connector</description>
-    <packaging>hetu-plugin</packaging>
-
-    <properties>
-        <air.main.basedir>${project.parent.basedir}</air.main.basedir>
-        <resources.dir>src/main/resources</resources.dir>
-        <version.clickhouse-jdbc>0.2.4</version.clickhouse-jdbc>
-        <version.maven-surefire-plugin>3.0.0-M2</version.maven-surefire-plugin>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.google.code.findbugs</groupId>
-            <artifactId>jsr305</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-base-jdbc</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>configuration</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>log</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>com.google.inject</groupId>
-            <artifactId>guice</artifactId>
-        </dependency>
-
-        <dependency>
-            <groupId>javax.inject</groupId>
-            <artifactId>javax.inject</artifactId>
-        </dependency>
-
-        <!-- Presto SPI -->
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-spi</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>slice</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>units</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
-        <!-- for testing -->
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-main</artifactId>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.bouncycastle</groupId>
-                    <artifactId>bcprov-jdk15on</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>testing</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-tpch</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift.tpch</groupId>
-            <artifactId>tpch</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-tests</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.airlift</groupId>
-            <artifactId>testing-mysql-server</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-testing-docker</artifactId>
-            <scope>test</scope>
-        </dependency>
-
-        <!-- https://mvnrepository.com/artifact/ru.yandex.clickhouse/clickhouse-jdbc -->
-        <dependency>
-            <groupId>ru.yandex.clickhouse</groupId>
-            <artifactId>clickhouse-jdbc</artifactId>
-            <version>${version.clickhouse-jdbc}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>javax.xml.bind</groupId>
-                    <artifactId>jaxb-api</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-
-    </dependencies>
-
-    <build>
-        <plugins>
-
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <!--                <version>${version.maven-surefire-plugin}</version>-->
-                <configuration>
-                    <useSystemClassLoader>false</useSystemClassLoader>
-                    <forkCount>1</forkCount>
-                    <parallel>classes</parallel>
-                    <threadCount>1</threadCount>
-                    <suiteXmlFiles>
-                        <file>src/test/testng.xml</file>
-                    </suiteXmlFiles>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -1,584 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.airlift.log.Logger;
-import io.hetu.core.plugin.clickhouse.optimization.ClickHousePushDownParameter;
-import io.hetu.core.plugin.clickhouse.optimization.ClickHouseQueryGenerator;
-import io.prestosql.plugin.jdbc.BaseJdbcClient;
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.ColumnMapping;
-import io.prestosql.plugin.jdbc.ConnectionFactory;
-import io.prestosql.plugin.jdbc.JdbcColumnHandle;
-import io.prestosql.plugin.jdbc.JdbcIdentity;
-import io.prestosql.plugin.jdbc.JdbcOutputTableHandle;
-import io.prestosql.plugin.jdbc.JdbcSplit;
-import io.prestosql.plugin.jdbc.JdbcTableHandle;
-import io.prestosql.plugin.jdbc.JdbcTypeHandle;
-import io.prestosql.plugin.jdbc.StatsCollecting;
-import io.prestosql.plugin.jdbc.WriteMapping;
-import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
-import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorResult;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.SuppressFBWarnings;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ColumnMetadata;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorTableMetadata;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.function.ExternalFunctionHub;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpressionService;
-import io.prestosql.spi.sql.QueryGenerator;
-import io.prestosql.spi.type.CharType;
-import io.prestosql.spi.type.DecimalType;
-import io.prestosql.spi.type.Type;
-
-import javax.inject.Inject;
-
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Types;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.function.BiFunction;
-
-import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.bigintColumnMapping;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.integerColumnMapping;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.smallintColumnMapping;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.varbinaryWriteFunction;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharColumnMapping;
-import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction;
-import static io.prestosql.spi.StandardErrorCode.NOT_FOUND;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
-import static io.prestosql.spi.type.BooleanType.BOOLEAN;
-import static io.prestosql.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE;
-import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE;
-import static io.prestosql.spi.type.VarbinaryType.VARBINARY;
-import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType;
-import static io.prestosql.spi.type.Varchars.isVarcharType;
-import static java.lang.String.format;
-import static java.lang.String.join;
-import static java.util.Collections.nCopies;
-import static java.util.Locale.ENGLISH;
-
-public class ClickHouseClient
-        extends BaseJdbcClient
-{
-    private static final Logger logger = Logger.get(ClickHouseClient.class);
-
-    private String[] tableTypes;
-
-    private ClickHouseConfig clickHouseConfig;
-
-    private BaseJdbcConfig baseJdbcConfig;
-
-    /**
-     * If disabled, do not accept sub-query push down.
-     */
-    private final JdbcPushDownModule pushDownModule;
-
-    /**
-     * constructor
-     *
-     * @param config Base config
-     * @param clickHouseConfig clickhouse config
-     * @param connectionFactory connectionFactory
-     */
-    @Inject
-    public ClickHouseClient(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig, @StatsCollecting ConnectionFactory connectionFactory, ExternalFunctionHub externalFunctionHub)
-    {
-        super(config, "", connectionFactory, externalFunctionHub);
-        tableTypes = clickHouseConfig.getTableTypes().split(",");
-        this.pushDownModule = config.getPushDownModule();
-        this.baseJdbcConfig = config;
-        this.clickHouseConfig = clickHouseConfig;
-    }
-
-    @Override
-    public Optional<ColumnMapping> toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle)
-    {
-        Optional<ColumnMapping> columnMapping;
-
-        String jdbcTypeName = "";
-        if (typeHandle.getJdbcTypeName().isPresent()) {
-            jdbcTypeName = typeHandle.getJdbcTypeName().get().toUpperCase(ENGLISH);
-        }
-        switch (typeHandle.getJdbcType()) {
-            case Types.TINYINT:
-                if (jdbcTypeName.equals("UINT8")) {
-                    columnMapping = Optional.of(smallintColumnMapping());
-                }
-                else {
-                    columnMapping = Optional.of(tinyintColumnMapping());
-                }
-                break;
-            case Types.INTEGER:
-                if (jdbcTypeName.equals("UINT32")) {
-                    columnMapping = Optional.of(bigintColumnMapping());
-                }
-                else {
-                    columnMapping = Optional.of(integerColumnMapping());
-                }
-                break;
-            case Types.SMALLINT:
-                if (jdbcTypeName.equals("UINT16")) {
-                    columnMapping = Optional.of(integerColumnMapping());
-                }
-                else {
-                    columnMapping = Optional.of(smallintColumnMapping());
-                }
-                break;
-            case Types.BIGINT:
-                if (jdbcTypeName.equals("UINT64")) {
-                    logger.warn("openLooKeng doesn't support UInt64, it will convert to Int64");
-                }
-                columnMapping = Optional.of(bigintColumnMapping());
-                break;
-            case Types.VARCHAR:
-                columnMapping = Optional.of(varcharColumnMapping(createUnboundedVarcharType()));
-                break;
-            case Types.DATE:
-            case Types.TIMESTAMP:
-            case Types.OTHER:
-            default:
-                columnMapping = super.toPrestoType(session, connection, typeHandle);
-        }
-        if (!columnMapping.isPresent()) {
-            throw new UnsupportedOperationException(
-                    "openLooKeng does not support the clickhouse type " + typeHandle.getJdbcTypeName().orElse("")
-                            + '(' + typeHandle.getColumnSize() + ", " + typeHandle.getDecimalDigits()
-                            + ')');
-        }
-        return columnMapping;
-    }
-
-    @Override
-    public WriteMapping toWriteMapping(ConnectorSession session, Type type)
-    {
-        if (isVarcharType(type)) {
-            return WriteMapping.sliceMapping("String", varcharWriteFunction());
-        }
-        else if (VARBINARY.equals(type)) {
-            return WriteMapping.sliceMapping("String", varbinaryWriteFunction());
-        }
-        else if (type instanceof CharType) {
-            return WriteMapping.sliceMapping("FixedString(" + ((CharType) type).getLength() + ')',
-                    charWriteFunction());
-        }
-        else if (type instanceof DecimalType) {
-            DecimalType decimalType = (DecimalType) type;
-            String dataType = format("decimal(%s, %s)", decimalType.getPrecision(), decimalType.getScale());
-            if (decimalType.isShort()) {
-                return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType));
-            }
-            return WriteMapping.sliceMapping(dataType, longDecimalWriteFunction(decimalType));
-        }
-        else if (BOOLEAN.equals(type)) {
-            throw new PrestoException(NOT_SUPPORTED, "There is no separate type for boolean values. Use UInt8 type, restricted to the values 0 or 1.");
-        }
-        else if (TIME_WITH_TIME_ZONE.equals(type) || TIMESTAMP_WITH_TIME_ZONE.equals(type)) {
-            throw new PrestoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName());
-        }
-
-        return super.toWriteMapping(session, type);
-    }
-
-    @Override
-    protected Collection<String> listSchemas(Connection connection)
-    {
-        try (ResultSet resultSet = connection.getMetaData().getSchemas()) {
-            ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder();
-            while (resultSet.next()) {
-                String schemaName = resultSet.getString(1);
-                schemaNames.add(schemaName);
-            }
-            return schemaNames.build();
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to list schemas");
-        }
-    }
-
-    @Override
-    protected ResultSet getTables(Connection connection, Optional<String> schemaName, Optional<String> tableName)
-            throws SQLException
-    {
-        DatabaseMetaData metadata = connection.getMetaData();
-        Optional<String> escape = Optional.ofNullable(metadata.getSearchStringEscape());
-        return metadata.getTables(connection.getCatalog(), escapeNamePattern(schemaName, escape).orElse(null), escapeNamePattern(tableName, escape).orElse(null), tableTypes);
-    }
-
-    /**
-     * create table function should be disabled.
-     */
-    @Override
-    protected JdbcOutputTableHandle createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, String tableName)
-    {
-        throw new UnsupportedOperationException("ClickHouse Connector does not support create table.");
-    }
-
-    @Override
-    public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column)
-    {
-        try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session))) {
-            String columnName = column.getName();
-            if (connection.getMetaData().storesUpperCaseIdentifiers()) {
-                columnName = columnName.toUpperCase(ENGLISH);
-            }
-
-            String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-            String sql = format("ALTER TABLE %s ADD column %s", quoted(null, schema, handle.getTableName()), getColumnSql(session, column, columnName));
-            execute(connection, sql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to add column, check table engine and sql");
-        }
-    }
-
-    private String getColumnSql(ConnectorSession session, ColumnMetadata column, String columnName)
-    {
-        StringBuilder sb = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY)
-                .append(quoted(columnName))
-                .append(" ")
-                .append(toWriteMapping(session, column.getType()).getDataType());
-        return sb.toString();
-    }
-
-    @Override
-    public void dropColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle column)
-    {
-        try (Connection connection = connectionFactory.openConnection(identity)) {
-            String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-            String sql = format("ALTER TABLE %s DROP column %s", quoted(null, schema, handle.getTableName()), column.getColumnName());
-            execute(connection, sql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to drop column");
-        }
-    }
-
-    @Override
-    public long getTableModificationTime(ConnectorSession session, JdbcTableHandle handle)
-    {
-        String sql = format(
-                "SELECT max(modification_time) as modified_time FROM system.parts WHERE database='%s' and table='%s'",
-                handle.getSchemaTableName().getSchemaName(),
-                handle.getSchemaTableName().getTableName());
-
-        try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
-                PreparedStatement preparedStatement = getPreparedStatement(connection, sql)) {
-            ResultSet resultSet = preparedStatement.executeQuery();
-            if (resultSet.next()) {
-                String updateTime = resultSet.getString("modified_time");
-                SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
-                return parser.parse(updateTime).getTime();
-            }
-            else {
-                // In the case where for some reason the table doesn't exist anymore in system.parts
-                return -1L;
-            }
-        }
-        catch (Exception e) {
-            // We want to make sure the query doesn't fail because of star-tree not being able to get last modified time
-            logger.error("Exception thrown while trying to get modified time", e);
-            return -1L;
-        }
-    }
-
-    @Override
-    public boolean isPreAggregationSupported(ConnectorSession session)
-    {
-        return true;
-    }
-
-    @Override
-    protected void renameTable(JdbcIdentity identity, String catalogName, String schemaName, String tableName, SchemaTableName newTable)
-    {
-        try (Connection connection = connectionFactory.openConnection(identity)) {
-            String newSchemaName = newTable.getSchemaName();
-            String newTableName = newTable.getTableName();
-            if (connection.getMetaData().storesUpperCaseIdentifiers()) {
-                newSchemaName = newSchemaName.toUpperCase(ENGLISH);
-                newTableName = newTableName.toUpperCase(ENGLISH);
-            }
-
-            String schema = checkCatalog(catalogName, schemaName);
-            String newSchema = checkCatalog(catalogName, newSchemaName);
-            String sql = format("RENAME TABLE %s TO %s", quoted(null, schema, tableName), quoted(null, newSchema, newTableName));
-            execute(connection, sql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, "openLooKeng ClickHouse connector failed to rename table");
-        }
-    }
-
-    @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
-    {
-        String newColumnName = inputNewColumnName;
-        try (Connection connection = connectionFactory.openConnection(identity)) {
-            if (connection.getMetaData().storesUpperCaseIdentifiers()) {
-                newColumnName = newColumnName.toUpperCase(ENGLISH);
-            }
-            String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-            String sql = format(
-                    "ALTER TABLE %s RENAME COLUMN %s TO %s",
-                    quoted(null, schema, handle.getTableName()),
-                    jdbcColumn.getColumnName(),
-                    newColumnName);
-            execute(connection, sql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-    }
-
-    @Override
-    public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, ConnectorTableMetadata tableMetadata)
-    {
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-
-        JdbcIdentity identity = JdbcIdentity.from(session);
-        if (!getSchemaNames(identity).contains(schemaTableName.getSchemaName())) {
-            throw new PrestoException(NOT_FOUND, "Schema not found: " + schemaTableName.getSchemaName());
-        }
-
-        try (Connection connection = connectionFactory.openConnection(identity)) {
-            boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers();
-            String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName());
-            String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName());
-            String temporaryTableName = generateTemporaryTableName();
-            String catalog = connection.getCatalog();
-
-            ImmutableList.Builder<String> columnNames = ImmutableList.builder();
-            ImmutableList.Builder<Type> columnTypes = ImmutableList.builder();
-            ImmutableList.Builder<String> columnList = ImmutableList.builder();
-            for (ColumnMetadata column : tableMetadata.getColumns()) {
-                String columnName = column.getName();
-                if (uppercase) {
-                    columnName = columnName.toUpperCase(ENGLISH);
-                }
-                columnNames.add(columnName);
-                columnTypes.add(column.getType());
-                columnList.add(getColumnSql(session, column, columnName));
-            }
-
-            String schema = checkCatalog(catalog, remoteSchema);
-            String sql = format(
-                    "CREATE TABLE %s engine=Log AS SELECT * FROM %s WHERE 0 = 1",
-                    quoted(null, schema, temporaryTableName),
-                    quoted(null, schema, remoteTable));
-            execute(connection, sql);
-
-            return new JdbcOutputTableHandle(
-                    catalog,
-                    remoteSchema,
-                    remoteTable,
-                    columnNames.build(),
-                    columnTypes.build(),
-                    temporaryTableName);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-    }
-
-    @Override
-    public String buildInsertSql(JdbcOutputTableHandle handle)
-    {
-        String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-        String sql = format(
-                "INSERT INTO %s VALUES (%s)",
-                quoted(null, schema, handle.getTemporaryTableName()),
-                join(",", nCopies(handle.getColumnNames().size(), "?")));
-        return sql;
-    }
-
-    @Override
-    public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List<JdbcColumnHandle> columns)
-            throws SQLException
-    {
-        if (table.getGeneratedSql().isPresent()) {
-            // openLooKeng: If the sub-query is pushed down, use it as the table
-            return new ClickHouseQueryBuilder(identifierQuote, true).buildSql(
-                    this,
-                    session,
-                    connection,
-                    null,
-                    null,
-                    table.getGeneratedSql().get().getSql(),
-                    columns,
-                    table.getConstraint(),
-                    split.getAdditionalPredicate(),
-                    tryApplyLimit(table.getLimit()));
-        }
-        return new ClickHouseQueryBuilder(identifierQuote).buildSql(
-                this,
-                session,
-                connection,
-                table.getCatalogName(),
-                table.getSchemaName(),
-                table.getTableName(),
-                columns,
-                table.getConstraint(),
-                split.getAdditionalPredicate(),
-                tryApplyLimit(table.getLimit()));
-    }
-
-    @Override
-    public void finishInsertTable(JdbcIdentity identity, JdbcOutputTableHandle handle)
-    {
-        String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-        String temporaryTable = quoted(null, schema, handle.getTemporaryTableName());
-        String targetTable = quoted(null, schema, handle.getTableName());
-        String insertSql = format("INSERT INTO %s SELECT * FROM %s", targetTable, temporaryTable);
-        String cleanupSql = "DROP TABLE " + temporaryTable;
-
-        try (Connection connection = getConnection(identity, handle)) {
-            execute(connection, insertSql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-
-        try (Connection connection = getConnection(identity, handle)) {
-            execute(connection, cleanupSql);
-        }
-        catch (SQLException e) {
-            logger.warn(e, "Failed to cleanup temporary table: %s", temporaryTable);
-        }
-    }
-
-    @Override
-    public void dropTable(JdbcIdentity identity, JdbcTableHandle handle)
-    {
-        String schema = checkCatalog(handle.getCatalogName(), handle.getSchemaName());
-        String sql = "DROP TABLE " + quoted(null, schema, handle.getTableName());
-
-        try (Connection connection = connectionFactory.openConnection(identity)) {
-            execute(connection, sql);
-        }
-        catch (SQLException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-    }
-
-    @Override
-    protected Optional<BiFunction<String, Long, String>> limitFunction()
-    {
-        return Optional.of((sql, limit) -> sql + " LIMIT " + limit);
-    }
-
-    @Override
-    public boolean isLimitGuaranteed()
-    {
-        return true;
-    }
-
-    @Override
-    public Optional<QueryGenerator<JdbcQueryGeneratorResult, JdbcConverterContext>> getQueryGenerator(DeterminismEvaluator determinismEvaluator, RowExpressionService rowExpressionService, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution)
-    {
-        ClickHousePushDownParameter pushDownParameter = new ClickHousePushDownParameter(getIdentifierQuote(), this.caseInsensitiveNameMatching, pushDownModule, clickHouseConfig, functionResolution);
-        return Optional.of(new ClickHouseQueryGenerator(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig));
-    }
-
-    @SuppressFBWarnings("SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING")
-    @Override
-    public Map<String, ColumnHandle> getColumns(ConnectorSession session, String sql, Map<String, Type> types)
-    {
-        try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
-                PreparedStatement statement = connection.prepareStatement(sql)) {
-            ResultSetMetaData metadata = statement.getMetaData();
-            ImmutableMap.Builder<String, ColumnHandle> builder = new ImmutableMap.Builder<>();
-            for (int i = 1; i <= metadata.getColumnCount(); i++) {
-                String columnName = metadata.getColumnLabel(i);
-                String typeName = metadata.getColumnTypeName(i);
-                int precision = metadata.getPrecision(i);
-                int dataType = metadata.getColumnType(i);
-                int scale = metadata.getScale(i);
-
-                boolean isNullAble = metadata.isNullable(i) != ResultSetMetaData.columnNoNulls;
-                if (dataType == Types.DECIMAL) {
-                    String loweredColumnName = columnName.toLowerCase(ENGLISH);
-                    Type type = types.get(loweredColumnName);
-                    if (type instanceof DecimalType) {
-                        DecimalType decimalType = (DecimalType) type;
-                        precision = decimalType.getPrecision();
-                        scale = decimalType.getScale();
-                    }
-                }
-
-                JdbcTypeHandle typeHandle = new JdbcTypeHandle(dataType, Optional.ofNullable(typeName), precision, scale, Optional.empty());
-                Optional<ColumnMapping> columnMapping;
-                try {
-                    columnMapping = toPrestoType(session, connection, typeHandle);
-                }
-                catch (UnsupportedOperationException ex) {
-                    // User configured to fail the query if the data type is not supported
-                    return Collections.emptyMap();
-                }
-                // skip unsupported column types
-                if (columnMapping.isPresent()) {
-                    Type type = columnMapping.get().getType();
-                    JdbcColumnHandle handle = new JdbcColumnHandle(columnName, typeHandle, type, isNullAble);
-                    builder.put(columnName.toLowerCase(ENGLISH), handle);
-                }
-                else {
-                    return Collections.emptyMap();
-                }
-            }
-            return builder.build();
-        }
-        catch (SQLException | PrestoException e) {
-            logger.error("in clickhouse push down, clickhouse data source error msg[%s] rewrite sql[%s]", e.getMessage(), sql);
-            return Collections.emptyMap();
-        }
-    }
-
-    /**
-     * Clickhouse does not support operations on catalog.schema.table
-     */
-    private String checkCatalog(String catalog, String remoteSchema)
-    {
-        String schema;
-        if (catalog != null && remoteSchema != null) {
-            schema = remoteSchema;
-        }
-        else {
-            schema = catalog == null ? remoteSchema : catalog;
-        }
-        return schema;
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClientModule.java

@@ -1,61 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.inject.Binder;
-import com.google.inject.Provides;
-import com.google.inject.Scopes;
-import com.google.inject.Singleton;
-import io.airlift.configuration.AbstractConfigurationAwareModule;
-import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.ConnectionFactory;
-import io.prestosql.plugin.jdbc.DriverConnectionFactory;
-import io.prestosql.plugin.jdbc.JdbcClient;
-import io.prestosql.spi.function.ExternalFunctionHub;
-import ru.yandex.clickhouse.ClickHouseDriver;
-import ru.yandex.clickhouse.settings.ClickHouseConnectionSettings;
-
-import java.util.Optional;
-import java.util.Properties;
-
-import static io.airlift.configuration.ConfigBinder.configBinder;
-import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
-
-public class ClickHouseClientModule
-        extends AbstractConfigurationAwareModule
-{
-    @Override
-    protected void setup(Binder binder)
-    {
-        binder.bind(JdbcClient.class).to(ClickHouseClient.class).in(Scopes.SINGLETON);
-        configBinder(binder).bindConfig(BaseJdbcConfig.class);
-        binder.bind(ExternalFunctionHub.class).to(ClickHouseExternalFunctionHub.class).in(Scopes.SINGLETON);
-        configBinder(binder).bindConfig(ClickHouseConfig.class);
-    }
-
-    @Provides
-    @Singleton
-    public static ConnectionFactory createConnectionFactory(BaseJdbcConfig config, ClickHouseConfig clickHouseConfig)
-    {
-        Properties connectionProperties = basicConnectionProperties(config);
-        connectionProperties.setProperty(ClickHouseConnectionSettings.SOCKET_TIMEOUT.getKey(),
-                String.valueOf(clickHouseConfig.getSocketTimeout()));
-        return new DriverConnectionFactory(new ClickHouseDriver(), config.getConnectionUrl(),
-                Optional.ofNullable(config.getUserCredentialName()),
-                Optional.ofNullable(config.getPasswordCredentialName()),
-                connectionProperties);
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseConfig.java

@@ -1,100 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import io.airlift.configuration.Config;
-import io.airlift.configuration.ConfigDescription;
-
-import java.util.Locale;
-
-/**
- * To get the custom properties to connect to the database.
- */
-public class ClickHouseConfig
-{
-    private static final int DEFAULT_SOCKET_TIMEOUT = 120000;
-    private int socketTimeout = DEFAULT_SOCKET_TIMEOUT;
-    private String tableTypes = ClickHouseConstants.DEFAULT_TABLE_TYPES;
-    private String schemaPattern;
-    private boolean isQueryPushDownEnabled = true;
-    private String clickHouseSqlVersion = "DEFAULT";
-
-    public int getSocketTimeout()
-    {
-        return socketTimeout;
-    }
-
-    @Config("clickhouse.socket_timeout")
-    @ConfigDescription("Connection ClickHouse socket timeout ")
-    public ClickHouseConfig setSocketTimeout(int socketTimeout)
-    {
-        this.socketTimeout = socketTimeout;
-        return this;
-    }
-
-    @Config("clickhouse.query.pushdown.enabled")
-    @ConfigDescription("Enable sub-query push down to clickhouse. It's set by default")
-    public ClickHouseConfig setQueryPushDownEnabled(boolean isQueryPushDownEnabledParameter)
-    {
-        this.isQueryPushDownEnabled = isQueryPushDownEnabledParameter;
-        return this;
-    }
-
-    public boolean isQueryPushDownEnabled()
-    {
-        return this.isQueryPushDownEnabled;
-    }
-
-    public String getTableTypes()
-    {
-        return this.tableTypes;
-    }
-
-    public String getSchemaPattern()
-    {
-        return schemaPattern;
-    }
-
-    /**
-     * setTableTypes
-     *
-     * @param tableTypes the table types to set
-     * @return ClickHouseConfig
-     */
-    @Config("clickhouse.table-types")
-    public ClickHouseConfig setTableTypes(String tableTypes)
-    {
-        this.tableTypes = tableTypes.toUpperCase(Locale.ENGLISH);
-        return this;
-    }
-
-    public String getClickHouseSqlVersion()
-    {
-        return this.clickHouseSqlVersion;
-    }
-
-    /**
-     * setSchemaPattern
-     *
-     * @param schemaPattern the schema pattern to set
-     * @return ClickHouseConfig
-     */
-    @Config("clickhouse.schema-pattern")
-    public ClickHouseConfig setSchemaPattern(String schemaPattern)
-    {
-        this.schemaPattern = schemaPattern;
-        return this;
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHousePlugin.java

@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.JdbcPlugin;
-import io.prestosql.spi.function.ConnectorConfig;
-import io.prestosql.spi.queryeditorui.ConnectorUtil;
-import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
-
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Optional;
-
-@ConnectorConfig(connectorLabel = "Clickhouse: Query data in Clickhouse system",
-        propertiesEnabled = true,
-        catalogConfigFilesEnabled = true,
-        globalConfigFilesEnabled = true,
-        docLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html",
-        configLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html#configuration")
-public class ClickHousePlugin
-        extends JdbcPlugin
-{
-    public ClickHousePlugin()
-    {
-        super("clickhouse", new ClickHouseClientModule());
-    }
-
-    @Override
-    public Optional<ConnectorWithProperties> getConnectorWithProperties()
-    {
-        ConnectorConfig connectorConfig = ClickHousePlugin.class.getAnnotation(ConnectorConfig.class);
-        ArrayList<Method> methods = new ArrayList<>();
-        methods.addAll(Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
-        methods.addAll(Arrays.asList(ClickHouseConfig.class.getDeclaredMethods()));
-        return ConnectorUtil.assembleConnectorProperties(connectorConfig, methods);
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseQueryBuilder.java

@@ -1,346 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.VerifyException;
-import com.google.common.collect.ImmutableList;
-import io.airlift.slice.Slice;
-import io.prestosql.plugin.jdbc.BlockWriteFunction;
-import io.prestosql.plugin.jdbc.BooleanWriteFunction;
-import io.prestosql.plugin.jdbc.DoubleWriteFunction;
-import io.prestosql.plugin.jdbc.JdbcClient;
-import io.prestosql.plugin.jdbc.JdbcColumnHandle;
-import io.prestosql.plugin.jdbc.JdbcTypeHandle;
-import io.prestosql.plugin.jdbc.LongWriteFunction;
-import io.prestosql.plugin.jdbc.QueryBuilder;
-import io.prestosql.plugin.jdbc.SliceWriteFunction;
-import io.prestosql.plugin.jdbc.WriteFunction;
-import io.prestosql.spi.block.Block;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.predicate.Domain;
-import io.prestosql.spi.predicate.Range;
-import io.prestosql.spi.predicate.TupleDomain;
-import io.prestosql.spi.type.ArrayType;
-import io.prestosql.spi.type.BigintType;
-import io.prestosql.spi.type.BooleanType;
-import io.prestosql.spi.type.CharType;
-import io.prestosql.spi.type.DateType;
-import io.prestosql.spi.type.DoubleType;
-import io.prestosql.spi.type.IntegerType;
-import io.prestosql.spi.type.RealType;
-import io.prestosql.spi.type.SmallintType;
-import io.prestosql.spi.type.TimeType;
-import io.prestosql.spi.type.TimeWithTimeZoneType;
-import io.prestosql.spi.type.TimestampType;
-import io.prestosql.spi.type.TimestampWithTimeZoneType;
-import io.prestosql.spi.type.TinyintType;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.VarcharType;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Function;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static com.google.common.collect.Iterables.getOnlyElement;
-import static java.lang.String.format;
-import static java.util.Collections.nCopies;
-import static java.util.Objects.requireNonNull;
-import static java.util.stream.Collectors.joining;
-
-public class ClickHouseQueryBuilder
-        extends QueryBuilder
-{
-    private static final String ALWAYS_TRUE = "1=1";
-    private static final String ALWAYS_FALSE = "1=0";
-    private boolean isPushSubQueryDown;
-    private final String identifierQuote;
-
-    public ClickHouseQueryBuilder(String identifierQuote)
-    {
-        super(identifierQuote);
-        this.identifierQuote = requireNonNull(identifierQuote, "quote is null");
-    }
-
-    public ClickHouseQueryBuilder(String identifierQuote, boolean isPushSubQueryDown)
-    {
-        this(identifierQuote);
-        this.isPushSubQueryDown = isPushSubQueryDown;
-    }
-
-    private static class TypeAndValue
-    {
-        private final Type type;
-        private final JdbcTypeHandle typeHandle;
-        private final Object value;
-
-        public TypeAndValue(Type type, JdbcTypeHandle typeHandle, Object value)
-        {
-            this.type = requireNonNull(type, "type is null");
-            this.typeHandle = requireNonNull(typeHandle, "typeHandle is null");
-            this.value = requireNonNull(value, "value is null");
-        }
-
-        public Type getType()
-        {
-            return type;
-        }
-
-        public JdbcTypeHandle getTypeHandle()
-        {
-            return typeHandle;
-        }
-
-        public Object getValue()
-        {
-            return value;
-        }
-    }
-
-    private String quote(String name)
-    {
-        return identifierQuote + name.replace(identifierQuote, identifierQuote + identifierQuote) + identifierQuote;
-    }
-
-    private static Domain pushDownDomain(JdbcClient client, ConnectorSession session, Connection connection, JdbcColumnHandle column, Domain domain)
-    {
-        return client.toPrestoType(session, connection, column.getJdbcTypeHandle())
-                .orElseThrow(() -> new IllegalStateException(format("Unsupported type %s with handle %s", column.getColumnType(), column.getJdbcTypeHandle())))
-                .getPushdownConverter().apply(domain);
-    }
-
-    private List<String> toConjuncts(
-            JdbcClient client,
-            ConnectorSession session,
-            Connection connection,
-            List<JdbcColumnHandle> columns,
-            TupleDomain<ColumnHandle> tupleDomain,
-            List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
-    {
-        if (tupleDomain.isNone()) {
-            return ImmutableList.of(ALWAYS_FALSE);
-        }
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-        for (JdbcColumnHandle column : columns) {
-            Domain domain = tupleDomain.getDomains().get().get(column);
-            if (domain != null) {
-                domain = pushDownDomain(client, session, connection, column, domain);
-                builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
-            }
-        }
-        return builder.build();
-    }
-
-    private String toPredicate(String columnName, Domain domain, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
-    {
-        if (domain.getValues().isNone()) {
-            return domain.isNullAllowed() ? quote(columnName) + " IS NULL" : ALWAYS_FALSE;
-        }
-
-        if (domain.getValues().isAll()) {
-            return domain.isNullAllowed() ? ALWAYS_TRUE : quote(columnName) + " IS NOT NULL";
-        }
-
-        List<String> disjuncts = new ArrayList<>();
-        List<Object> singleValues = new ArrayList<>();
-        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
-            checkState(!range.isAll()); // Already checked
-            if (range.isSingleValue()) {
-                singleValues.add(range.getLow().getValue());
-            }
-            else {
-                List<String> rangeConjuncts = new ArrayList<>();
-                if (!range.getLow().isLowerUnbounded()) {
-                    switch (range.getLow().getBound()) {
-                        case ABOVE:
-                            rangeConjuncts.add(toPredicate(columnName, ">", range.getLow().getValue(), column, accumulator));
-                            break;
-                        case EXACTLY:
-                            rangeConjuncts.add(toPredicate(columnName, ">=", range.getLow().getValue(), column, accumulator));
-                            break;
-                        case BELOW:
-                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
-                        default:
-                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
-                    }
-                }
-                if (!range.getHigh().isUpperUnbounded()) {
-                    switch (range.getHigh().getBound()) {
-                        case ABOVE:
-                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
-                        case EXACTLY:
-                            rangeConjuncts.add(toPredicate(columnName, "<=", range.getHigh().getValue(), column, accumulator));
-                            break;
-                        case BELOW:
-                            rangeConjuncts.add(toPredicate(columnName, "<", range.getHigh().getValue(), column, accumulator));
-                            break;
-                        default:
-                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
-                    }
-                }
-                // If rangeConjuncts is null, then the range was ALL, which should already have been checked for
-                checkState(!rangeConjuncts.isEmpty());
-                disjuncts.add("(" + Joiner.on(" AND ").join(rangeConjuncts) + ")");
-            }
-        }
-
-        // Add back all of the possible single values either as an equality or an IN predicate
-        if (singleValues.size() == 1) {
-            disjuncts.add(toPredicate(columnName, "=", getOnlyElement(singleValues), column, accumulator));
-        }
-        else if (singleValues.size() > 1) {
-            for (Object value : singleValues) {
-                bindValue(value, column, accumulator);
-            }
-            String values = Joiner.on(",").join(nCopies(singleValues.size(), "?"));
-            disjuncts.add(quote(columnName) + " IN (" + values + ")");
-        }
-
-        // Add nullability disjuncts
-        checkState(!disjuncts.isEmpty());
-        if (domain.isNullAllowed()) {
-            disjuncts.add(quote(columnName) + " IS NULL");
-        }
-
-        return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
-    }
-
-    private String toPredicate(String columnName, String operator, Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
-    {
-        bindValue(value, column, accumulator);
-        return quote(columnName) + " " + operator + " ?";
-    }
-
-    private static void bindValue(Object value, JdbcColumnHandle column, List<ClickHouseQueryBuilder.TypeAndValue> accumulator)
-    {
-        Type type = column.getColumnType();
-        checkArgument(isAcceptedType(type), "Can't handle type: %s", type);
-        accumulator.add(new TypeAndValue(type, column.getJdbcTypeHandle(), value));
-    }
-
-    private static boolean isAcceptedType(Type type)
-    {
-        Type validType = requireNonNull(type, "type is null");
-        return validType.equals(BigintType.BIGINT) ||
-                validType.equals(TinyintType.TINYINT) ||
-                validType.equals(SmallintType.SMALLINT) ||
-                validType.equals(IntegerType.INTEGER) ||
-                validType.equals(DoubleType.DOUBLE) ||
-                validType.equals(RealType.REAL) ||
-                validType.equals(BooleanType.BOOLEAN) ||
-                validType.equals(DateType.DATE) ||
-                validType.equals(TimeType.TIME) ||
-                validType.equals(TimeWithTimeZoneType.TIME_WITH_TIME_ZONE) ||
-                validType.equals(TimestampType.TIMESTAMP) ||
-                validType.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE) ||
-                validType instanceof VarcharType ||
-                validType instanceof CharType ||
-                validType instanceof ArrayType;
-    }
-
-    @Override
-    public PreparedStatement buildSql(JdbcClient client,
-            ConnectorSession session,
-            Connection connection,
-            String catalog,
-            String schema,
-            String table,
-            List<JdbcColumnHandle> columns,
-            TupleDomain<ColumnHandle> tupleDomain,
-            Optional<String> additionalPredicate,
-            Function<String, String> sqlFunction)
-            throws SQLException
-    {
-        StringBuilder sql = new StringBuilder();
-
-        String columnNames = columns.stream()
-                .map(JdbcColumnHandle::getColumnName)
-                .map(this::quote)
-                .collect(joining(", "));
-
-        sql.append("SELECT ");
-        sql.append(columnNames);
-        if (columns.isEmpty()) {
-            sql.append("null");
-        }
-
-        sql.append(" FROM ");
-        if (!isNullOrEmpty(schema)) {
-            sql.append(quote(schema)).append('.');
-        }
-        if (isPushSubQueryDown) {
-            sql.append("(").append(table).append(") pushdown");
-        }
-        else {
-            sql.append(quote(table));
-        }
-
-        List<TypeAndValue> accumulator = new ArrayList<>();
-
-        List<String> clauses = toConjuncts(client, session, connection, columns, tupleDomain, accumulator);
-        if (additionalPredicate.isPresent()) {
-            clauses = ImmutableList.<String>builder()
-                    .addAll(clauses)
-                    .add(additionalPredicate.get())
-                    .build();
-        }
-        if (!clauses.isEmpty()) {
-            sql.append(" WHERE ")
-                    .append(Joiner.on(" AND ").join(clauses));
-        }
-
-        String query = sqlFunction.apply(sql.toString());
-        PreparedStatement statement = client.getPreparedStatement(connection, query);
-
-        for (int i = 0; i < accumulator.size(); i++) {
-            TypeAndValue typeAndValue = accumulator.get(i);
-            int parameterIndex = i + 1;
-            Type type = typeAndValue.getType();
-            WriteFunction writeFunction = client.toPrestoType(session, connection, typeAndValue.getTypeHandle())
-                    .orElseThrow(() -> new VerifyException(format("Unsupported type %s with handle %s", type, typeAndValue.getTypeHandle())))
-                    .getWriteFunction();
-            Class<?> javaType = type.getJavaType();
-            Object value = typeAndValue.getValue();
-            if (javaType == boolean.class) {
-                ((BooleanWriteFunction) writeFunction).set(statement, parameterIndex, (boolean) value);
-            }
-            else if (javaType == long.class) {
-                ((LongWriteFunction) writeFunction).set(statement, parameterIndex, (long) value);
-            }
-            else if (javaType == double.class) {
-                ((DoubleWriteFunction) writeFunction).set(statement, parameterIndex, (double) value);
-            }
-            else if (javaType == Slice.class) {
-                ((SliceWriteFunction) writeFunction).set(statement, parameterIndex, (Slice) value);
-            }
-            else if (javaType == Block.class) {
-                ((BlockWriteFunction) writeFunction).set(statement, parameterIndex, (Block) value);
-            }
-            else {
-                throw new VerifyException(format("Unexpected type %s with java type %s", type, javaType.getName()));
-            }
-        }
-
-        return statement;
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -1,63 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization;
-
-import com.google.common.base.Joiner;
-import io.hetu.core.plugin.clickhouse.optimization.externalfunc.ClickHouseExternalFunctionHub;
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
-import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
-import io.prestosql.spi.function.SqlFunctionHandle;
-import io.prestosql.spi.relation.CallExpression;
-import io.prestosql.sql.builder.functioncall.ApplyRemoteFunctionPushDown;
-
-import java.util.Map;
-import java.util.Optional;
-
-import static java.util.stream.Collectors.toList;
-
-public class ClickHouseApplyRemoteFunctionPushDown
-        extends ApplyRemoteFunctionPushDown
-{
-    private static Map<String, String> clickHousePrimalFunctionNameMap = ClickHouseExternalFunctionHub.getPrimalFunctionNameMap();
-
-    public ClickHouseApplyRemoteFunctionPushDown(BaseJdbcConfig baseJdbcConfig, String connectorName)
-    {
-        super(baseJdbcConfig, connectorName);
-    }
-
-    /**
-     * rewrite the remote function to a executable function in the data source.
-     */
-    @Override
-    public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
-    {
-        if (!isConnectorSupportedRemoteFunction(callExpression)) {
-            return Optional.empty();
-        }
-        jdbcConverterContext.setRemoteUdfVisited(true);
-        String displayName = ((SqlFunctionHandle) callExpression.getFunctionHandle()).getFunctionId().getFunctionName().getObjectName();
-        String args = Joiner.on(",").join(callExpression.getArguments().stream().map(expression -> expression.accept(rowExpressionConverter, jdbcConverterContext)).collect(toList()));
-        // The click house is case sensitive and presto only support the lower case function name,
-        // we need to handle this function into lower and store its primal name.
-        // The click house do not contain two different functions have the equal function name ignore case.
-        // If a data base contain two different functions have the equal function name ignore case,
-        // then we should register them into different function name space
-        if (!clickHousePrimalFunctionNameMap.containsKey(displayName)) {
-            return Optional.empty();
-        }
-        return Optional.of(String.format("%s(%s)", clickHousePrimalFunctionNameMap.get(displayName), args));
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHousePushDownParameter.java

@@ -1,40 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization;
-
-import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
-import io.prestosql.spi.function.StandardFunctionResolution;
-
-/**
- * Push Down parameter module
- */
-public class ClickHousePushDownParameter
-        extends JdbcPushDownParameter
-{
-    private final ClickHouseConfig clickHouseConfig;
-
-    public ClickHousePushDownParameter(String identifierQuote, boolean nameCaseInsensitive, JdbcPushDownModule pushDownModule, ClickHouseConfig clickHouseConfig, StandardFunctionResolution functionResolution)
-    {
-        super(identifierQuote, nameCaseInsensitive, pushDownModule, functionResolution);
-        this.clickHouseConfig = clickHouseConfig;
-    }
-
-    public ClickHouseConfig getClickHouseConfig()
-    {
-        return clickHouseConfig;
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseQueryGenerator.java

@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization;
-
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcQueryGenerator;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpressionService;
-
-public class ClickHouseQueryGenerator
-        extends BaseJdbcQueryGenerator
-{
-    public ClickHouseQueryGenerator(
-            DeterminismEvaluator determinismEvaluator,
-            RowExpressionService rowExpressionService,
-            FunctionMetadataManager functionManager,
-            StandardFunctionResolution functionResolution,
-            ClickHousePushDownParameter pushDownParameter,
-            BaseJdbcConfig baseJdbcConfig)
-    {
-        super(pushDownParameter, new ClickHouseRowExpressionConverter(determinismEvaluator, rowExpressionService, functionManager, functionResolution, pushDownParameter, baseJdbcConfig), new ClickHouseSqlStatementWriter(pushDownParameter));
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseRowExpressionConverter.java

@@ -1,257 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization;
-
-import com.google.common.base.Joiner;
-import io.hetu.core.plugin.clickhouse.ClickHouseConfig;
-import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
-import io.hetu.core.plugin.clickhouse.rewrite.BuildInDirectMapFunctionCallRewriter;
-import io.hetu.core.plugin.clickhouse.rewrite.ClickHouseUnsupportedFunctionCallRewriter;
-import io.hetu.core.plugin.clickhouse.rewrite.UdfFunctionRewriteConstants;
-import io.hetu.core.plugin.clickhouse.rewrite.functioncall.DateParseFunctionCallRewriter;
-import io.prestosql.configmanager.ConfigSupplier;
-import io.prestosql.configmanager.DefaultUdfRewriteConfigSupplier;
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
-import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.function.FunctionHandle;
-import io.prestosql.spi.function.FunctionMetadata;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.OperatorType;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.relation.CallExpression;
-import io.prestosql.spi.relation.ConstantExpression;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpression;
-import io.prestosql.spi.relation.RowExpressionService;
-import io.prestosql.spi.relation.SpecialForm;
-import io.prestosql.spi.sql.expression.QualifiedName;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.VarcharType;
-import io.prestosql.sql.builder.functioncall.FunctionWriterManager;
-import io.prestosql.sql.builder.functioncall.FunctionWriterManagerGroup;
-import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
-import io.prestosql.sql.builder.functioncall.functions.base.FromBase64CallRewriter;
-import io.prestosql.sql.builder.functioncall.functions.config.DefaultConnectorConfigFunctionRewriter;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-import static com.google.common.collect.ImmutableSet.toImmutableSet;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
-import static io.prestosql.sql.builder.functioncall.BaseFunctionUtil.isDefaultFunction;
-import static java.lang.String.format;
-import static java.util.Locale.ENGLISH;
-
-public class ClickHouseRowExpressionConverter
-        extends BaseJdbcRowExpressionConverter
-{
-    private static final Set<String> clickHouseNotSupportFunctions =
-            Stream.of("try", "try_cast", "at_timezone", "current_user", "current_path", "current_time").collect(toImmutableSet());
-    private static FunctionWriterManager clickHouseFunctionManager;
-    private final ClickHouseApplyRemoteFunctionPushDown clickHouseApplyRemoteFunctionPushDown;
-
-    /**
-     * ClickHouse sql query writer
-     *
-     * @param clickHouseConfig config
-     */
-    public ClickHouseRowExpressionConverter(
-            DeterminismEvaluator determinismEvaluator,
-            RowExpressionService rowExpressionService,
-            FunctionMetadataManager functionManager,
-            StandardFunctionResolution functionResolution,
-            ClickHousePushDownParameter clickHouseConfig,
-            BaseJdbcConfig baseJdbcConfig)
-    {
-        super(functionManager, functionResolution, rowExpressionService, determinismEvaluator);
-        clickHouseFunctionManager = initFunctionManager(clickHouseConfig.getClickHouseConfig());
-        clickHouseApplyRemoteFunctionPushDown = new ClickHouseApplyRemoteFunctionPushDown(baseJdbcConfig, ClickHouseConstants.CONNECTOR_NAME);
-    }
-
-    private FunctionWriterManager initFunctionManager(ClickHouseConfig clickHouseConfig)
-    {
-        ConfigSupplier configSupplier = new DefaultUdfRewriteConfigSupplier(UdfFunctionRewriteConstants.DEFAULT_VERSION_UDF_REWRITE_PATTERNS);
-        DefaultConnectorConfigFunctionRewriter connectorConfigFunctionRewriter =
-                new DefaultConnectorConfigFunctionRewriter(ClickHouseConstants.CONNECTOR_NAME, configSupplier);
-
-        return FunctionWriterManagerGroup.newFunctionWriterManagerInstance(ClickHouseConstants.CONNECTOR_NAME,
-                clickHouseConfig.getClickHouseSqlVersion(), getInjectFunctionCallRewritersDefault(clickHouseConfig), connectorConfigFunctionRewriter);
-    }
-
-    private Map<String, FunctionCallRewriter> getInjectFunctionCallRewritersDefault(ClickHouseConfig clickHouseConfig)
-    {
-        // add the user define function re-writer
-        Map<String, FunctionCallRewriter> functionCallRewriters = new HashMap<>(Collections.emptyMap());
-        // 1. the base function re-writers all connector can use
-        FromBase64CallRewriter fromBase64CallRewriter = new FromBase64CallRewriter();
-        functionCallRewriters.put(FromBase64CallRewriter.INNER_FUNC_FROM_BASE64, fromBase64CallRewriter);
-
-        // 2. the specific user define function re-writers
-        FunctionCallRewriter unSupportedFunctionCallRewriter = new ClickHouseUnsupportedFunctionCallRewriter(ClickHouseConstants.CONNECTOR_NAME);
-        functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_DAY2SEC, unSupportedFunctionCallRewriter);
-        functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH, unSupportedFunctionCallRewriter);
-        functionCallRewriters.put(ClickHouseUnsupportedFunctionCallRewriter.INNER_FUNC_TIME_WITH_TZ_LITERAL, unSupportedFunctionCallRewriter);
-
-        FunctionCallRewriter buildInDirectMapFunctionCallRewriter = new BuildInDirectMapFunctionCallRewriter();
-        functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUIDLIN_AGGR_FUNC_SUM, buildInDirectMapFunctionCallRewriter);
-        functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_AVG, buildInDirectMapFunctionCallRewriter);
-        functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_COUNT, buildInDirectMapFunctionCallRewriter);
-        functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MAX, buildInDirectMapFunctionCallRewriter);
-        functionCallRewriters.put(BuildInDirectMapFunctionCallRewriter.BUILDIN_AGGR_FUNC_MIN, buildInDirectMapFunctionCallRewriter);
-
-        FunctionCallRewriter dateParseFunctionCallRewriter = new DateParseFunctionCallRewriter();
-        functionCallRewriters.put(DateParseFunctionCallRewriter.BUILD_IN_FUNC_DATE_PARSE, dateParseFunctionCallRewriter);
-
-        return functionCallRewriters;
-    }
-
-    protected static String functionCall(QualifiedName name, boolean isDistinct, List<String> argumentsList, Optional<String> orderBy, Optional<String> filter, Optional<String> window)
-    {
-        if (clickHouseFunctionManager == null) {
-            throw new PrestoException(NOT_SUPPORTED, "Function manager is uninitialized");
-        }
-
-        try {
-            return clickHouseFunctionManager.getFunctionRewriteResult(name, isDistinct, argumentsList, orderBy, filter, window);
-        }
-        catch (UnsupportedOperationException e) {
-            throw new PrestoException(NOT_SUPPORTED, e.getMessage());
-        }
-    }
-
-    @Override
-    public String visitCall(CallExpression call, JdbcConverterContext context)
-    {
-        FunctionHandle functionHandle = call.getFunctionHandle();
-        // remote udf verify
-        if (!isDefaultFunction(call)) {
-            Optional<String> result = clickHouseApplyRemoteFunctionPushDown.rewriteRemoteFunction(call, this, context);
-            if (result.isPresent()) {
-                return result.get();
-            }
-            throw new PrestoException(NOT_SUPPORTED, String.format("ClickHouse connector does not support remote function: %s.%s", call.getDisplayName(), call.getFunctionHandle().getFunctionNamespace()));
-        }
-
-        FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(functionHandle);
-        String functionName = functionMetadata.getName().getObjectName();
-
-        if (clickHouseNotSupportFunctions.contains(functionName)) {
-            throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support " + functionName);
-        }
-
-        if (standardFunctionResolution.isOperator(functionHandle)) {
-            return handleOperatorFunction(call, functionMetadata, context);
-        }
-
-        List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
-        if (standardFunctionResolution.isNotFunction(functionHandle)) {
-            return format("(NOT %s)", argumentList.get(0));
-        }
-
-        if (standardFunctionResolution.isLikeFunction(functionHandle)) {
-            return format("(%s LIKE %s)", argumentList.get(0), argumentList.get(1));
-        }
-
-        /*
-         * Array needs to be tested
-         */
-        if (standardFunctionResolution.isArrayConstructor(functionHandle)) {
-            return format("ARRAY(%s)", Joiner.on(", ").join(argumentList));
-        }
-
-        return functionCall(new QualifiedName(Collections.singletonList(functionName)), false, argumentList, Optional.empty(), Optional.empty(), Optional.empty());
-    }
-
-    private String handleOperatorFunction(CallExpression call, FunctionMetadata functionMetadata, JdbcConverterContext context)
-    {
-        Optional<OperatorType> operatorTypeOptional = functionMetadata.getOperatorType();
-        OperatorType type = operatorTypeOptional.get();
-        if (type.equals(OperatorType.CAST)) {
-            return handleCastOperator(call.getArguments().get(0), call.getType(), context);
-        }
-
-        List<String> argumentList = call.getArguments().stream().map(expr -> expr.accept(this, context)).collect(Collectors.toList());
-        if (type.isArithmeticOperator()) {
-            return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
-        }
-
-        if (type.isComparisonOperator()) {
-            final String[] clickHouseCompareOperators = new String[] {"=", ">", "<", ">=", "<=", "!=", "<>"};
-            if (Arrays.asList(clickHouseCompareOperators).contains(type.getOperator())) {
-                return format("(%s %s %s)", argumentList.get(0), type.getOperator(), argumentList.get(1));
-            }
-            else {
-                String exceptionInfo = "ClickHouse Connector does not support comparison operator " + type.getOperator();
-                throw new PrestoException(NOT_SUPPORTED, exceptionInfo);
-            }
-        }
-
-        if (type.equals(OperatorType.SUBSCRIPT)) {
-            throw new PrestoException(NOT_SUPPORTED, "ClickHouse Connector does not support subscript now");
-        }
-
-        /*
-         * "Negative" needs to be tested
-         */
-        if (call.getArguments().size() == 1 && type.equals(OperatorType.NEGATION)) {
-            String value = argumentList.get(0);
-            String separator = value.startsWith("-") ? " " : "";
-            return format("-%s%s", separator, value);
-        }
-
-        throw new PrestoException(NOT_SUPPORTED, String.format("Unknown operator %s in push down", type.getOperator()));
-    }
-
-    private String handleCastOperator(RowExpression expression, Type dstType, JdbcConverterContext context)
-    {
-        /*
-         * In SqlToRowExpressionTranslator, it will translate GenericLiteral expression to a 'CONSTANT' rowExpression,
-         * so the 'CAST' operator is not needed.
-         * */
-        String value = expression.accept(this, context);
-        if (expression instanceof ConstantExpression && expression.getType() instanceof VarcharType) {
-            //dstType needs to be tested here.
-            //especially DateTime.
-            return value;
-        }
-
-        if (dstType.getDisplayName().equals(LIKE_PATTERN_NAME)) {
-            return value;
-        }
-
-        return format("CAST(%s AS %s)", value, dstType.getDisplayName().toLowerCase(ENGLISH));
-    }
-
-    @Override
-    public String visitSpecialForm(SpecialForm specialForm, JdbcConverterContext context)
-    {
-        if (specialForm.getForm().equals(SpecialForm.Form.DEREFERENCE)
-                || specialForm.getForm().equals(SpecialForm.Form.ROW_CONSTRUCTOR)
-                || specialForm.getForm().equals(SpecialForm.Form.BIND)) {
-            throw new PrestoException(NOT_SUPPORTED, "ClickHouse connector does not support" + specialForm.getForm().toString());
-        }
-
-        return super.visitSpecialForm(specialForm, context);
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -1,79 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization;
-
-import io.airlift.log.Logger;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcSqlStatementWriter;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
-import io.prestosql.spi.sql.expression.Types;
-
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-
-/**
- * It is not clear what is the difference between aggregation and the previous version.
- */
-public class ClickHouseSqlStatementWriter
-        extends BaseJdbcSqlStatementWriter
-{
-    protected static final Logger log = Logger.get(ClickHouseSqlStatementWriter.class);
-
-    public ClickHouseSqlStatementWriter(JdbcPushDownParameter pushDownParameter)
-    {
-        super(pushDownParameter);
-    }
-
-    @Override
-    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
-    {
-        String functionName = inputFunctionName;
-        if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
-            functionName = "varPop";
-        }
-        return super.aggregation(functionName, arguments, isDistinct);
-    }
-
-    @Override
-    public String windowFrame(Types.WindowFrameType type, String start, Optional<String> end)
-    {
-        throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
-    }
-
-    @Override
-    public String window(String functionName, List<String> functionArgs, List<String> partitionBy, Optional<String> orderBy, Optional<String> frame)
-    {
-        throw new UnsupportedOperationException("ClickHouse Connector does not support windows function");
-    }
-
-    @Override
-    public String from(String selections, String from)
-    {
-        String[] froms = from.split("\\.");
-        if (froms.length == 2) {
-            return selections + " FROM " + from;
-        }
-        else if (froms.length == 3) {
-            StringBuilder sb = new StringBuilder(froms[1]);
-            sb.append(".");
-            sb.append(froms[2]);
-            return selections + " FROM " + sb.toString();
-        }
-        else {
-            log.error("params more than 3 " + from);
-            return selections + " FROM " + from;
-        }
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/externalfunc/ClickHouseExternalDateTimeFunctions.java

@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.prestosql.spi.function.ExternalFunctionInfo;
-import io.prestosql.spi.type.StandardTypes;
-
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-
-public class ClickHouseExternalDateTimeFunctions
-{
-    public static Set<ExternalFunctionInfo> getFunctionsInfo()
-    {
-        return ImmutableSet.<ExternalFunctionInfo>builder()
-                .add(ClickHouse_toUnixTimeStamp_FUNCTION_INFO)
-                .add(ClickHouse_toDateTime_FUNCTION_INFO)
-                .build();
-    }
-
-    public static Map<String, String> getDateTimeHubFunctionStringMap()
-    {
-        // the click house is case sensitive and presto only support the lower case function name,
-        // we need to handle this function into lower and store its primal name.
-        // now the click house do not contain two different functions have the equal function name ignore case.
-        return ImmutableMap.<String, String>builder()
-                .put(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH), TO_UNIX_TIME_STAMP_PRIMAL)
-                .put(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH), TO_DATE_TIME_PRIMAL)
-                .build();
-    }
-
-    private static final String TO_UNIX_TIME_STAMP_PRIMAL = "toUnixTimestamp";
-
-    private static final String TO_DATE_TIME_PRIMAL = "toDateTime";
-
-    private static final ExternalFunctionInfo ClickHouse_toUnixTimeStamp_FUNCTION_INFO =
-            ExternalFunctionInfo.builder()
-                    .functionName(TO_UNIX_TIME_STAMP_PRIMAL.toLowerCase(Locale.ENGLISH))
-                    .inputArgs(StandardTypes.VARCHAR)
-                    .returnType(StandardTypes.BIGINT)
-                    .deterministic(true)
-                    .calledOnNullInput(false)
-                    .description("converts value to the number with type UInt32 -- Unix Timestamp")
-                    .build();
-
-    private static final ExternalFunctionInfo ClickHouse_toDateTime_FUNCTION_INFO =
-            ExternalFunctionInfo.builder()
-                    .functionName(TO_DATE_TIME_PRIMAL.toLowerCase(Locale.ENGLISH))
-                    .inputArgs(StandardTypes.VARCHAR)
-                    .returnType(StandardTypes.TIMESTAMP)
-                    .deterministic(true)
-                    .calledOnNullInput(false)
-                    .description("converts value to the number with type UInt32 -- Unix Timestamp")
-                    .build();
-
-    private ClickHouseExternalDateTimeFunctions()
-    {
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/externalfunc/ClickHouseExternalFunctionHub.java

@@ -1,68 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.optimization.externalfunc;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.spi.connector.CatalogSchemaName;
-import io.prestosql.spi.function.ExternalFunctionInfo;
-import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
-
-import javax.inject.Inject;
-
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-
-import static java.util.Objects.requireNonNull;
-
-public class ClickHouseExternalFunctionHub
-        extends JdbcExternalFunctionHub
-{
-    private final BaseJdbcConfig jdbcConfig;
-
-    @Inject
-    public ClickHouseExternalFunctionHub(BaseJdbcConfig jdbcConfig)
-    {
-        this.jdbcConfig = requireNonNull(jdbcConfig, "jdbcConfig is null");
-    }
-
-    @Override
-    public Optional<CatalogSchemaName> getExternalFunctionCatalogSchemaName()
-    {
-        return jdbcConfig.getConnectorRegistryFunctionNamespace();
-    }
-
-    @Override
-    public Set<ExternalFunctionInfo> getExternalFunctions()
-    {
-        return ImmutableSet.<ExternalFunctionInfo>builder()
-                .addAll(ClickHouseExternalDateTimeFunctions.getFunctionsInfo())
-                .build();
-    }
-
-    public static Map<String, String> getPrimalFunctionNameMap()
-    {
-        // The click house is case sensitive and presto only support the lower case function name,
-        // we need to handle this function into lower and store its primal name.
-        // The click house do not contain two different functions have the equal function name ignore case.
-        // If a data base contain two different functions have the equal function name ignore case,
-        // then we should register them into different function name space
-        return ImmutableMap.<String, String>builder()
-                .putAll(ClickHouseExternalDateTimeFunctions.getDateTimeHubFunctionStringMap())
-                .build();
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/BuildInDirectMapFunctionCallRewriter.java

@@ -1,72 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.rewrite;
-
-import io.hetu.core.plugin.clickhouse.ClickHouseConstants;
-import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
-import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
-
-public class BuildInDirectMapFunctionCallRewriter
-        implements FunctionCallRewriter
-{
-    // ========= for Aggregate Functions =========
-
-    /**
-     * functioncall name of build-in direct map aggregate function max
-     */
-    public static final String BUILDIN_AGGR_FUNC_MAX = "max";
-
-    /**
-     * functioncall name of build-in direct min aggregate function min
-     */
-    public static final String BUILDIN_AGGR_FUNC_MIN = "min";
-
-    /**
-     * functioncall name of build-in direct map aggregate function count
-     */
-    public static final String BUILDIN_AGGR_FUNC_COUNT = "count";
-
-    /**
-     * functioncall name of build-in direct map aggregate function avg
-     */
-    public static final String BUILDIN_AGGR_FUNC_AVG = "avg";
-
-    /**
-     * functioncall name of build-in direct map aggregate function sum
-     */
-    public static final String BUIDLIN_AGGR_FUNC_SUM = "sum";
-
-    @Override
-    public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
-    {
-        StringBuilder builder = new StringBuilder(ClickHouseConstants.DEAFULT_STRINGBUFFER_CAPACITY);
-
-        String arguments = RewriteUtil.joinExpressions(functionCallArgsPackage.getArgumentsList());
-        if (functionCallArgsPackage.getArgumentsList().isEmpty() && "count".equalsIgnoreCase(functionCallArgsPackage.getName().getSuffix())) {
-            arguments = "*";
-        }
-        if (functionCallArgsPackage.getIsDistinct()) {
-            arguments = "DISTINCT " + arguments;
-        }
-
-        builder.append(RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName())).append('(').append(arguments);
-        functionCallArgsPackage.getOrderBy().ifPresent(exp -> builder.append(' ').append(exp));
-        builder.append(')');
-        functionCallArgsPackage.getFilter().ifPresent(exp -> builder.append(" FILTER ").append(exp));
-        functionCallArgsPackage.getWindow().ifPresent(exp -> builder.append(" OVER ").append(exp));
-
-        return builder.toString();
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/ClickHouseUnsupportedFunctionCallRewriter.java

@@ -1,52 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.rewrite;
-
-import io.prestosql.spi.type.StandardTypes;
-import io.prestosql.sql.builder.functioncall.functions.base.UnsupportedFunctionCallRewriter;
-
-import static io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil.LITERAL_FUNCNAME_PREFIX;
-
-public class ClickHouseUnsupportedFunctionCallRewriter
-        extends UnsupportedFunctionCallRewriter
-{
-    /**
-     * functioncall name of INTERVAL_DAY_TO_SECOND literal in HeTu inner
-     */
-    public static final String INNER_FUNC_INTERVAL_LITERAL_DAY2SEC =
-            LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_DAY_TO_SECOND;
-
-    /**
-     * functioncall name of INTERVAL_YEAR_TO_MONTH literal in HeTu inner
-     */
-    public static final String INNER_FUNC_INTERVAL_LITERAL_YEAR2MONTH =
-            LITERAL_FUNCNAME_PREFIX + StandardTypes.INTERVAL_YEAR_TO_MONTH;
-
-    /**
-     * functioncall name of TIME_WITH_TIME_ZONE literal in HeTu inner
-     */
-    public static final String INNER_FUNC_TIME_WITH_TZ_LITERAL =
-            LITERAL_FUNCNAME_PREFIX + StandardTypes.TIME_WITH_TIME_ZONE;
-
-    /**
-     * the constructor of Unsupported Function Call Re-writer
-     *
-     * @param connectorName connector's name
-     */
-    public ClickHouseUnsupportedFunctionCallRewriter(String connectorName)
-    {
-        super(connectorName);
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/RewriteUtil.java

@@ -1,77 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.rewrite;
-
-import com.google.common.base.Joiner;
-import io.prestosql.spi.sql.expression.QualifiedName;
-import io.prestosql.spi.sql.expression.Selection;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static java.util.stream.Collectors.joining;
-
-/**
- * util helper class for rewrite handle.
- */
-public class RewriteUtil
-{
-    private RewriteUtil()
-    {
-    }
-
-    /**
-     * interal function prefix name
-     */
-    public static final String LITERAL_FUNCNAME_PREFIX = "$literal$";
-
-    /**
-     * expression list
-     *
-     * @param expressions expression lists
-     * @return sql statement
-     */
-    public static final String joinExpressions(List<String> expressions)
-    {
-        return Joiner.on(", ").join(expressions);
-    }
-
-    /**
-     * formate identifier
-     *
-     * @param qualifiedNames qualified names
-     * @param identifier identifier
-     * @return sql statement
-     */
-    public static final String formatIdentifier(Optional<Map<String, Selection>> qualifiedNames, String identifier)
-    {
-        if (qualifiedNames.isPresent()) {
-            return qualifiedNames.get().get(identifier).getExpression();
-        }
-        return identifier;
-    }
-
-    /**
-     * formate qualified name
-     *
-     * @param name qualified name
-     * @return sql statement
-     */
-    public static final String formatQualifiedName(QualifiedName name)
-    {
-        return name.getParts().stream().map(identifier -> formatIdentifier(Optional.empty(), identifier)).collect(joining("."));
-    }
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/UdfFunctionRewriteConstants.java

@@ -1,98 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.rewrite;
-
-import com.google.common.collect.ImmutableMap;
-
-import java.util.Map;
-
-public class UdfFunctionRewriteConstants
-{
-    private UdfFunctionRewriteConstants()
-    {
-    }
-
-    /**
-     * udf rewrite pattern map
-     */
-    public static final Map<String, String> DEFAULT_VERSION_UDF_REWRITE_PATTERNS =
-            new ImmutableMap.Builder<String, String>()
-                    // Statistical aggregate functions
-                    .put("CORR($1,$2)", "CORR($1, $2)")
-                    .put("STDDEV($1)", "stddevSamp($1)")
-                    .put("stddev_pop($1)", "stddev_pop($1)")
-                    .put("stddev_samp($1)", "stddevSamp($1)")
-                    .put("skewness($1)", "skewPop($1)")
-                    .put("kurtosis($1)", "kurtPop($1)")
-                    .put("VARIANCE($1)", "varSamp($1)")
-                    .put("var_samp($1)", "varSamp($1)")
-                    .put("APPROX_DISTINCT($1)", "uniq($1)")
-                    .put("APPROX_DISTINCT($1,$2)", "uniq($1)")
-                    // math functions
-                    .put("ABS($1)", "ABS($1)")
-                    .put("ACOS($1)", "ACOS($1)")
-                    .put("ASIN($1)", "ASIN($1)")
-                    .put("ATAN($1)", "ATAN($1)")
-                    .put("ATAN2($1,$2)", "ATAN2($1, $2)")
-                    .put("CEIL($1)", "CEIL($1)")
-                    .put("CEILING($1)", "CEIL($1)")
-                    .put("COS($1)", "COS($1)")
-                    .put("e()", "e()")
-                    .put("EXP($1)", "EXP($1)")
-                    .put("FLOOR($1)", "FLOOR($1)")
-                    .put("LN($1)", "LN($1)")
-                    .put("LOG10($1)", "log10($1)")
-                    .put("LOG2($1)", "log2($1)")
-                    .put("MOD($1,$2)", "MOD($1, $2)")
-                    .put("pi()", "pi()")
-                    .put("POW($1,$2)", "POW($1, $2)")
-                    .put("POWER($1,$2)", "POWER($1, $2)")
-                    .put("RAND()", "RAND()")
-                    .put("RANDOM()", "RAND()")
-                    .put("ROUND($1)", "ROUND($1)")
-                    .put("ROUND($1,$2)", "ROUND($1, $2)")
-                    .put("SIGN($1)", "SIGN($1)")
-                    .put("SIN($1)", "SIN($1)")
-                    .put("SQRT($1)", "SQRT($1)")
-                    .put("TAN($1)", "TAN($1)")
-                    //character functions
-                    .put("CONCAT($1,$2)", "CONCAT($1, $2)")
-                    .put("LENGTH($1)", "LENGTH($1)")
-                    .put("LOWER($1)", "LOWER($1)")
-                    .put("LTRIM($1)", "trimLeft($1)")
-                    .put("REPLACE($1,$2)", "replaceAll($1, $2, '')")
-                    .put("REPLACE($1,$2,$3)", "replaceAll($1, $2, $3)")
-                    .put("RTRIM($1)", "trimRight($1)")
-                    .put("STRPOS($1,$2)", "position($1, $2)")
-                    .put("SUBSTR($1,$2,$3)", "SUBSTR($1, $2, $3)")
-                    .put("POSITION($1,$2)", "position($2, $1)")
-                    .put("TRIM($1)", "trimBoth($1)")
-                    .put("UPPER($1)", "UPPER($1)")
-                    //date functions
-                    .put("YEAR($1)", "toYear($1)")
-                    .put("QUARTER($1)", "toQuarter($1)")
-                    .put("MONTH($1)", "toMonth($1)")
-                    .put("WEEK($1)", "toWeek($1)")
-                    .put("YEAR_OF_WEEK($1)", "toISOYear($1)")
-                    .put("DAY($1)", "toDayOfMonth($1)")
-                    .put("HOUR($1)", "toHour($1)")
-                    .put("MINUTE($1)", "toMinute($1)")
-                    .put("SECOND($1)", "toSecond($1)")
-                    .put("DAY_OF_WEEK($1)", "toDayOfWeek($1)")
-                    .put("DAY_OF_MONTH($1)", "toDayOfMonth($1)")
-                    .put("DAY_OF_YEAR($1)", "toDayOfYear($1)")
-                    .put("TO_UNIXTIME($1)", "toUnixTimestamp($1)")
-                    .build();
-}

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/functioncall/DateParseFunctionCallRewriter.java

@@ -1,51 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse.rewrite.functioncall;
-
-import io.hetu.core.plugin.clickhouse.rewrite.RewriteUtil;
-import io.prestosql.sql.builder.functioncall.FunctionCallArgsPackage;
-import io.prestosql.sql.builder.functioncall.functions.FunctionCallRewriter;
-
-import java.util.List;
-import java.util.Locale;
-
-/**
- * Clickhouse date time type conversion functions.
- * The rewrite of this function limits date_parse.
- */
-public class DateParseFunctionCallRewriter
-        implements FunctionCallRewriter
-{
-    public static final String BUILD_IN_FUNC_DATE_PARSE = "date_parse";
-
-    @Override
-    public String rewriteFunctionCall(FunctionCallArgsPackage functionCallArgsPackage)
-    {
-        String functionName = RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName());
-        if (!functionName.toLowerCase(Locale.ENGLISH).equals("date_parse") || functionCallArgsPackage.getArgumentsList().size() != 2) {
-            throw new UnsupportedOperationException("ClickHouse Connector does not support function call of " + RewriteUtil.formatQualifiedName(functionCallArgsPackage.getName()));
-        }
-        List<String> argsList = functionCallArgsPackage.getArgumentsList();
-        String args0 = argsList.get(0);
-        String args1 = argsList.get(1);
-        if (args1.equals("'%Y-%m-%d %H:%i:%s'")) {
-            return "toDateTime(" + args0 + ")";
-        }
-        else if (args1.equals("'%Y-%m-%d'")) {
-            return "toDate(" + args0 + ")";
-        }
-        return String.format("parseDateTimeBestEffort(%s)", args0);
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseClientTest.java

@@ -1,355 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.common.collect.ImmutableSet;
-import io.airlift.log.Logger;
-import io.prestosql.plugin.jdbc.JdbcColumnHandle;
-import io.prestosql.plugin.jdbc.JdbcIdentity;
-import io.prestosql.plugin.jdbc.JdbcTableHandle;
-import io.prestosql.spi.connector.ColumnMetadata;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.connector.ConnectorTableMetadata;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.type.IntegerType;
-import io.prestosql.spi.type.Type;
-import org.testng.SkipException;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import java.lang.reflect.Field;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-
-import static io.prestosql.spi.type.VarcharType.VARCHAR;
-import static io.prestosql.testing.TestingSession.testSessionBuilder;
-import static org.testng.Assert.assertEquals;
-
-/**
- * ClickHouseTest
- */
-public class ClickHouseClientTest
-{
-    private static final ConnectorSession SESSION = testSessionBuilder().build().toConnectorSession();
-
-    private static final Logger LOGGER = Logger.get(ClickHouseClientTest.class);
-
-    private DataBaseTest database;
-
-    private String catalogName;
-
-    private ClickHouseClient clickHouseClient;
-
-    private Connection connection;
-
-    private ClickHouseServerTest clickHouseServer;
-
-    private ClickHouseClientTest()
-    {
-    }
-
-    /**
-     * setUp
-     *
-     * @throws SQLException SQLException
-     */
-    @BeforeClass
-    public void setUp()
-            throws SQLException
-    {
-        this.clickHouseServer = ClickHouseServerTest.getInstance();
-        if (!clickHouseServer.isClickHouseServerAvailable()) {
-            LOGGER.info("please set correct clickhouse data base info!");
-            throw new SkipException("skip the test");
-        }
-        LOGGER.info("running TestClickHouseClient...");
-        database = new DataBaseTest(clickHouseServer);
-        connection = database.getConnection();
-        catalogName = connection.getCatalog();
-        clickHouseClient = database.getClickHouseClient();
-    }
-
-    /**
-     * tearDown
-     *
-     * @throws SQLException SQLException
-     */
-    @AfterClass(alwaysRun = true)
-    public void tearDown()
-            throws SQLException
-    {
-        if (this.clickHouseServer.isClickHouseServerAvailable()) {
-            database.close();
-        }
-    }
-
-    /**
-     * testCreateTable
-     */
-    @Test(enabled = false)
-    public void testCreateTable()
-            throws SQLException
-    {
-        String tableName = "testCreateTable";
-        String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
-        JdbcIdentity identity = JdbcIdentity.from(SESSION);
-        SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
-
-        Type intType = IntegerType.INTEGER;
-        ColumnMetadata columnMetadata = new ColumnMetadata("col1", intType);
-        List<ColumnMetadata> columns = new ArrayList<>();
-        columns.add(columnMetadata);
-
-        ConnectorTableMetadata connectorTableMetadata = new ConnectorTableMetadata(schemaTableName, columns);
-        clickHouseClient.createTable(SESSION, connectorTableMetadata, tableName);
-
-        JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
-        clickHouseClient.dropTable(identity, tableHandle);
-    }
-
-    /**
-     * testListSchemas
-     */
-    @Test
-    public void testListSchemas()
-    {
-        assertEquals(ImmutableSet.copyOf(clickHouseClient.listSchemas(connection)).contains("test"), true);
-    }
-
-    /**
-     * testGetTables
-     */
-    @Test
-    public void testGetTables()
-    {
-        List<String> expectedTables = database.getTables();
-        List<String> allTables = getAllTables(getNameByUpperCaseIdentifiers(database.getSchema()));
-        List<String> actualTables = getActualTables(allTables, expectedTables);
-        assertEquals(actualTables, expectedTables);
-    }
-
-    @Test
-    public void testRenameTable()
-    {
-        String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
-
-        List<String> expectedTables = database.getTables();
-        List<String> allTables = getAllTables(schemaName);
-        List<String> actualTables = getActualTables(allTables, expectedTables);
-        assertEquals(actualTables, expectedTables);
-
-        String actualTable = database.getActualTable("number"); // get actual table number_*
-        String tableNumber = getNameByUpperCaseIdentifiers(actualTable);
-        String tableNewNumber = getNameByUpperCaseIdentifiers(actualTable.replace("number", "new_number"));
-
-        /* rename table number_* to new_number_* */
-        JdbcIdentity identity = JdbcIdentity.from(SESSION);
-        SchemaTableName newTableName = new SchemaTableName(schemaName, tableNewNumber);
-        clickHouseClient.renameTable(identity, null, schemaName, tableNumber, newTableName);
-
-        expectedTables.remove(3); //remove table "number_*"
-        expectedTables.add(actualTable.replace("number", "new_number")); //add table "new_number_*"
-        allTables = getAllTables(schemaName);
-        actualTables = getActualTables(allTables, expectedTables);
-        assertEquals(actualTables, expectedTables);
-
-        /* rename table new_number_* to number_* */
-        newTableName = new SchemaTableName(schemaName, tableNumber);
-        clickHouseClient.renameTable(identity, null, schemaName, tableNewNumber, newTableName);
-
-        expectedTables.remove(3); //remove table "new_number"
-        expectedTables.add(actualTable); //add table "number"
-        allTables = getAllTables(schemaName);
-        actualTables = getActualTables(allTables, expectedTables);
-        assertEquals(actualTables, expectedTables);
-    }
-
-    /**
-     * testRenameColumn
-     */
-    @Test
-    public void testRenameColumn()
-            throws NoSuchFieldException, IllegalAccessException
-    {
-        String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
-        // get actual table table_with_float_col_*
-        String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("table_with_float_col"));
-
-        List<String> expectedColumns = Arrays.asList("col1", "col2", "col3", "col4");
-        List<String> actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-
-        JdbcIdentity identity = JdbcIdentity.from(SESSION);
-        SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
-        JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
-
-        List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
-        JdbcColumnHandle columnHandle = null;
-        for (JdbcColumnHandle column : columns) {
-            if ("col4".equalsIgnoreCase(column.getColumnName())) {
-                columnHandle = column;
-                break;
-            }
-        }
-
-        Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
-        field.setAccessible(true);
-        field.set(tableHandle, null);
-
-        clickHouseClient.renameColumn(identity, tableHandle, columnHandle, "newcol4");
-
-        expectedColumns = Arrays.asList("col1", "col2", "col3", "newcol4");
-        actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-    }
-
-    /**
-     * testAddColumn
-     */
-    @Test
-    public void testAddColumn()
-            throws NoSuchFieldException, IllegalAccessException
-    {
-        String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
-        // get actual table student_*
-        String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("student"));
-
-        List<String> expectedColumns = Arrays.asList("id");
-        List<String> actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-
-        JdbcIdentity identity = JdbcIdentity.from(SESSION);
-        SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
-        JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
-        ColumnMetadata columnMetadata = new ColumnMetadata("name", VARCHAR);
-
-        Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
-        field.setAccessible(true);
-        field.set(tableHandle, null);
-        clickHouseClient.addColumn(SESSION, tableHandle, columnMetadata);
-
-        expectedColumns = Arrays.asList("id", "name");
-        actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-    }
-
-    /**
-     * testDropColumn
-     */
-    @Test
-    public void testDropColumn()
-            throws NoSuchFieldException, IllegalAccessException
-    {
-        String schemaName = getNameByUpperCaseIdentifiers(database.getSchema());
-        String tableName = getNameByUpperCaseIdentifiers(database.getActualTable("example")); // get actual table example_*
-
-        List<String> expectedColumns = Arrays.asList("text", "text_short", "value");
-        List<String> actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-
-        JdbcIdentity identity = JdbcIdentity.from(SESSION);
-        SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
-        JdbcTableHandle tableHandle = clickHouseClient.getTableHandle(identity, schemaTableName).get();
-        List<JdbcColumnHandle> columns = clickHouseClient.getColumns(SESSION, tableHandle);
-
-        JdbcColumnHandle columnHandle = null;
-        for (JdbcColumnHandle column : columns) {
-            if ("value".equalsIgnoreCase(column.getColumnName())) {
-                columnHandle = column;
-                break;
-            }
-        }
-
-        Field field = JdbcTableHandle.class.getDeclaredField("catalogName");
-        field.setAccessible(true);
-        field.set(tableHandle, null);
-        clickHouseClient.dropColumn(identity, tableHandle, columnHandle);
-
-        expectedColumns = Arrays.asList("text", "text_short");
-        actualColumns = getActualColumns(schemaName, tableName);
-        assertEquals(actualColumns, expectedColumns);
-    }
-
-    @Test(enabled = false)
-    public void testGetColumns()
-    {
-    }
-
-    private String getNameByUpperCaseIdentifiers(String name)
-    {
-        try {
-            if (connection.getMetaData().storesUpperCaseIdentifiers()) {
-                return name.toUpperCase(Locale.ENGLISH);
-            }
-            else {
-                return name;
-            }
-        }
-        catch (SQLException e) {
-            throw new RuntimeException("Failed to get metadata or storesUpperCaseIdentifiers", e);
-        }
-    }
-
-    private List<String> getAllTables(String schemaName)
-    {
-        List<String> allTables = new ArrayList<>();
-
-        try (ResultSet resultSet = clickHouseClient.getTables(connection, Optional.of(schemaName), Optional.empty())) {
-            while (resultSet.next()) {
-                String tableName = resultSet.getString("TABLE_NAME");
-                allTables.add(tableName.toLowerCase(Locale.ENGLISH));
-            }
-
-            return allTables;
-        }
-        catch (SQLException e) {
-            throw new RuntimeException("Failed to get tables", e);
-        }
-    }
-
-    private List<String> getActualTables(List<String> allTables, List<String> expectedTables)
-    {
-        List<String> actualTables = new ArrayList<>();
-        for (String table : expectedTables) {
-            if (allTables.contains(table)) {
-                actualTables.add(table);
-            }
-        }
-        return actualTables;
-    }
-
-    private List<String> getActualColumns(String schemaName, String tableName)
-    {
-        try (ResultSet resultSet = connection.getMetaData().getColumns(null, schemaName, tableName, null)) {
-            List<String> actualColumns = new ArrayList<>();
-            while (resultSet.next()) {
-                String columnName = resultSet.getString("COLUMN_NAME");
-                actualColumns.add(columnName.toLowerCase(Locale.ENGLISH));
-            }
-
-            return actualColumns;
-        }
-        catch (SQLException e) {
-            throw new RuntimeException("Failed to get metadata or columns", e);
-        }
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseConfigTest.java

@@ -1,75 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.common.collect.ImmutableMap;
-import org.testng.annotations.Test;
-
-import java.util.Map;
-
-import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping;
-import static org.testng.Assert.assertEquals;
-
-/**
- * This is testing ClickHouseConfig functions
- */
-public class ClickHouseConfigTest
-{
-    private static final int COMMUNICATION_TIMEOUT = 10;
-
-    private static final int PACKET_SIZE = 150000;
-
-    /**
-     * This is testing ClickHousePropertyMappings
-     */
-    @Test
-    public void testClickHousePropertyMappings()
-    {
-        final String tableTypes = "TABLE,VIEW,USER DEFINED,SYNONYM,OLAP VIEW,JOIN VIEW,HIERARCHY VIEW" + ",CALC VIEW,SYSTEM TABLE,NO LOGGING TEMPORARY,GLOBAL TEMPORARY";
-        final String schemaPattern = "default";
-
-        Map<String, String> properties = new ImmutableMap.Builder<String, String>()
-                .put("clickhouse.table-types", tableTypes)
-                .put("clickhouse.schema-pattern", schemaPattern)
-                .put("clickhouse.query.pushdown.enabled", "false")
-                .put("clickhouse.socket_timeout", "100")
-                .build();
-
-        ClickHouseConfig expected = new ClickHouseConfig()
-                .setTableTypes(tableTypes)
-                .setSchemaPattern(schemaPattern)
-                .setQueryPushDownEnabled(false)
-                .setSocketTimeout(100);
-        assertFullMapping(properties, expected);
-    }
-
-    /**
-     * This is testing ClickHouseConfig class inner functions
-     */
-    @Test
-    public void testGetFuncions()
-    {
-        ClickHouseConfig config = new ClickHouseConfig();
-
-        String tableTypes = config.getTableTypes();
-        assertEquals(tableTypes, ClickHouseConstants.DEFAULT_TABLE_TYPES);
-
-        String schemaPattern = config.getSchemaPattern();
-        assertEquals(schemaPattern, null);
-
-        boolean isQueryPushDown = config.isQueryPushDownEnabled();
-        assertEquals(isQueryPushDown, true);
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseConstantsTest.java

@@ -1,27 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-public class ClickHouseConstantsTest
-{
-    private ClickHouseConstantsTest()
-    {
-    }
-
-    /**
-     * "src/test/java/io/hetu/core/plugin/clickhouse/test-clickhouse-dev.properties"
-     */
-    public static final String TEST_PROPERTY_FILE_PATH = "src/test/java/io/hetu/plugin/clickhouse/test-clickhouse-dev.properties";
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHousePluginTest.java

@@ -1,34 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import com.google.common.collect.ImmutableMap;
-import io.prestosql.spi.Plugin;
-import io.prestosql.spi.connector.ConnectorFactory;
-import io.prestosql.testing.TestingConnectorContext;
-import org.testng.annotations.Test;
-
-import static com.google.common.collect.Iterables.getOnlyElement;
-
-public class ClickHousePluginTest
-{
-    @Test
-    public void testCreateConnector()
-    {
-        Plugin plugin = new ClickHousePlugin();
-        ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories());
-        factory.create("test", ImmutableMap.of("connection-url", "jdbc:clickhouse://test"), new TestingConnectorContext());
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -1,221 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import io.airlift.log.Logger;
-import io.airlift.tpch.TpchTable;
-
-import javax.annotation.concurrent.GuardedBy;
-
-import java.io.File;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static io.airlift.configuration.ConfigurationLoader.loadPropertiesFrom;
-
-public final class ClickHouseServerTest
-{
-    private static final Logger LOG = Logger.get(ClickHouseConstantsTest.class);
-
-    @GuardedBy("this")
-    private static int referenceCount;
-
-    @GuardedBy("this")
-    private static ClickHouseServerTest instance;
-
-    private final AtomicBoolean tpchLoaded = new AtomicBoolean(false);
-    private final CountDownLatch tpchLoadComplete = new CountDownLatch(1);
-    private static List<String> actualTables = new ArrayList<>();
-    private final AtomicBoolean isServerUsable = new AtomicBoolean(false);
-    private final AtomicInteger atomicIntegerTimes = new AtomicInteger(0);
-    private String jdbcUrl;
-    private String user;
-    private String password;
-    private String schema;
-
-    public static synchronized ClickHouseServerTest getInstance()
-    {
-        if (referenceCount == 0) {
-            instance = new ClickHouseServerTest();
-            instance.startup();
-        }
-        referenceCount++;
-        return instance;
-    }
-
-    public static synchronized void shutDown()
-            throws SQLException
-    {
-        referenceCount--;
-        if (referenceCount == 0) {
-            instance.shutdown();
-            instance = null;
-        }
-    }
-
-    private void startup()
-    {
-        for (TpchTable<?> table : TpchTable.getTables()) {
-            String newTable = generateNewTableName(table.getTableName());
-            actualTables.add(newTable);
-        }
-    }
-
-    private void shutdown()
-            throws SQLException
-    {
-        if (isClickHouseServerAvailable() && tpchLoaded.get()) {
-            for (String table : actualTables) {
-                String sql = "DROP TABLE " + schema + "." + table;
-                executeInClickHouse(sql);
-            }
-        }
-    }
-
-    private void executeInClickHouse(String sql)
-            throws SQLException
-    {
-        try (Connection connection = DriverManager.getConnection(jdbcUrl, user, password);
-                Statement statement = connection.createStatement()) {
-            statement.execute(sql);
-        }
-    }
-
-    private ClickHouseServerTest()
-    {
-        File file = new File(ClickHouseConstantsTest.TEST_PROPERTY_FILE_PATH);
-        try {
-            Map<String, String> properties = new HashMap<>(loadPropertiesFrom(file.getPath()));
-            LOG.info("test-clickhouse properties: %s", properties);
-
-            String connectionUrl = properties.get("connection.url");
-            String connectionUser = properties.get("connection.user");
-            String connectionPass = properties.get("connection.password");
-            String connectionSchema = properties.get("connection.schema");
-
-            if (connectionUrl != null) {
-                this.jdbcUrl = connectionUrl;
-            }
-
-            if (connectionUser != null) {
-                this.user = connectionUser;
-            }
-
-            if (connectionPass != null) {
-                this.password = connectionPass;
-            }
-
-            if (connectionSchema != null) {
-                this.schema = connectionSchema;
-            }
-        }
-        catch (IOException e) {
-            LOG.warn("Failed to load properties for file %s", file);
-        }
-    }
-
-    public boolean isClickHouseServerAvailable()
-    {
-        if (atomicIntegerTimes.getAndIncrement() > 0) {
-            return isServerUsable.get();
-        }
-
-        try {
-            Connection connection = DriverManager.getConnection(this.jdbcUrl, this.user, this.password);
-            isServerUsable.set(true);
-            return isServerUsable.get();
-        }
-        catch (SQLException e) {
-            isServerUsable.set(false);
-            return isServerUsable.get();
-        }
-    }
-
-    public String getJdbcUrl()
-    {
-        return jdbcUrl;
-    }
-
-    public String getUser()
-    {
-        return user;
-    }
-
-    public String getPassword()
-    {
-        return password;
-    }
-
-    public String getSchema()
-    {
-        return schema;
-    }
-
-    public boolean isTpchLoaded()
-    {
-        return tpchLoaded.getAndSet(true);
-    }
-
-    public void setTpchLoaded()
-    {
-        tpchLoadComplete.countDown();
-    }
-
-    public void waitTpchLoaded()
-            throws InterruptedException
-    {
-        tpchLoadComplete.await(2, TimeUnit.MINUTES);
-    }
-
-    public static String generateNewTableName(String tableName)
-    {
-        return tableName + "_" + UUID.randomUUID().toString().replace("-", "");
-    }
-
-    public static String getActualTable(String tablePattern)
-    {
-        return getActualTable(actualTables, tablePattern);
-    }
-
-    public static String getActualTable(List<String> tables, String tablePattern)
-    {
-        String actualTable = tablePattern;
-
-        for (String table : tables) {
-            int lastIndex = table.lastIndexOf("_");
-            if (lastIndex == -1) {
-                continue;
-            }
-
-            if (table.substring(0, lastIndex).equalsIgnoreCase(tablePattern)) {
-                actualTable = tablePattern + table.substring(lastIndex);
-                break;
-            }
-        }
-        return actualTable;
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/DataBaseTest.java

@@ -1,145 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package io.hetu.core.plugin.clickhouse;
-
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
-import io.prestosql.plugin.jdbc.ConnectionFactory;
-import io.prestosql.plugin.jdbc.DriverConnectionFactory;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.sql.builder.functioncall.JdbcExternalFunctionHub;
-
-import java.lang.reflect.InvocationTargetException;
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
-
-import static io.prestosql.plugin.jdbc.DriverConnectionFactory.basicConnectionProperties;
-import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR;
-
-public class DataBaseTest
-        implements AutoCloseable
-{
-    private final Connection connection;
-
-    private final ClickHouseClient clickHouseClient;
-
-    private ClickHouseServerTest clickHouseServerTest;
-
-    private List<String> tables = new ArrayList<>();
-
-    /**
-     * constructor
-     */
-    public DataBaseTest(ClickHouseServerTest clickHouseServer)
-            throws SQLException
-    {
-        this.clickHouseServerTest = clickHouseServer;
-
-        BaseJdbcConfig jdbcConfig = new BaseJdbcConfig();
-        ClickHouseConfig clickHouseConfig = new ClickHouseConfig();
-
-        jdbcConfig.setConnectionUrl(clickHouseServer.getJdbcUrl());
-        jdbcConfig.setConnectionUser(clickHouseServer.getUser());
-        jdbcConfig.setConnectionPassword(clickHouseServer.getPassword());
-
-        clickHouseConfig.setTableTypes("TABLE,VIEW");
-        clickHouseConfig.setSchemaPattern(clickHouseServer.getSchema());
-        clickHouseConfig.setQueryPushDownEnabled(false);
-
-        Driver driver = null;
-        try {
-            driver = (Driver) Class.forName(ClickHouseConstants.CLICKHOUSE_JDBC_DRIVER_CLASS_NAME).getConstructor(((Class<?>[]) null)).newInstance();
-        }
-        catch (InstantiationException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-        catch (IllegalAccessException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-        catch (ClassNotFoundException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-        catch (InvocationTargetException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-        catch (NoSuchMethodException e) {
-            throw new PrestoException(JDBC_ERROR, e);
-        }
-        ConnectionFactory connectionFactory = new DriverConnectionFactory(driver, jdbcConfig.getConnectionUrl(), Optional.ofNullable(jdbcConfig.getUserCredentialName()), Optional.ofNullable(jdbcConfig.getPasswordCredentialName()), basicConnectionProperties(jdbcConfig));
-
-        clickHouseClient = new ClickHouseClient(jdbcConfig, clickHouseConfig, connectionFactory, new JdbcExternalFunctionHub());
-        connection =
-                DriverManager.getConnection(clickHouseServer.getJdbcUrl(), clickHouseServer.getUser(), clickHouseServer.getPassword());
-
-        connection.createStatement()
-                .execute(buildCreateTableSql("example", "(text varchar,text_short varchar(32), value bigint)"));
-        connection.createStatement().execute(buildCreateTableSql("student", "(id varchar)"));
-        connection.createStatement()
-                .execute(buildCreateTableSql("table_with_float_col", "(col1 bigint, col2 double, col3 float, col4 real)"));
-        connection.createStatement()
-                .execute(buildCreateTableSql("number", "(text varchar, text_short varchar(32), value bigint)"));
-    }
-
-    @Override
-    public void close()
-            throws SQLException
-    {
-        for (String table : tables) {
-            String sql = "DROP TABLE " + clickHouseServerTest.getSchema() + "." + table;
-            connection.createStatement().execute(sql);
-        }
-
-        ClickHouseServerTest.shutDown();
-    }
-
-    public Connection getConnection()
-    {
-        return connection;
-    }
-
-    public ClickHouseClient getClickHouseClient()
-    {
-        return clickHouseClient;
-    }
-
-    public List<String> getTables()
-    {
-        return tables;
-    }
-
-    public String getSchema()
-    {
-        return clickHouseServerTest.getSchema();
-    }
-
-    public String getActualTable(String tablePattern)
-    {
-        return ClickHouseServerTest.getActualTable(tables, tablePattern);
-    }
-
-    private String buildCreateTableSql(String tableName, String columnInfo)
-    {
-        String newTableName = ClickHouseServerTest.generateNewTableName(tableName);
-        tables.add(newTableName);
-        String sql = "CREATE TABLE " + clickHouseServerTest.getSchema() + "." + newTableName + columnInfo + "engine=MergeTree() order by tuple()";
-        System.out.println(sql);
-
-        return sql;
-    }
-}

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/test-clickhouse-dev.properties

@@ -1,10 +0,0 @@
-#
-# WARNING
-# ^^^^^^^
-# This configuration file is for development only and should NOT be used
-# in production. For example configuration, see the Hetu documentation.
-# please set your data source info here
-connection.url=jdbc:clickhouse://localhost:8123
-connection.user=default
-connection.password=yourpassword
-connection.schema=test

hetu-clickhouse/src/test/testng.xml

@@ -1,10 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd" >
-<suite name="TestGroupSuite">
-    <test name="GroupTest">
-        <classes>
-            <class name="io.hetu.core.plugin.clickhouse.ClickHouseConfigTest"/>
-            <class name="io.hetu.core.plugin.clickhouse.ClickHouseClientTest"/>
-            <class name="io.hetu.core.plugin.clickhouse.ClickHousePluginTest"/>
-        </classes>
-    </test>
-</suite>

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/algorithm/SequenceUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -18,14 +18,15 @@ package io.hetu.core.common.algorithm;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.PeekingIterator;
 
-import java.util.Arrays;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.PriorityQueue;
 
 /**
- * Algorithms for ordered iterator operations (merge, union, intersect)
+ * Algorithms for sequence operations (merge, union, intersect)
+ *
+ * @author Han
  */
 public class SequenceUtils
 {
@@ -45,12 +46,6 @@ public class SequenceUtils
         return merge(iterators, false);
     }
 
-    @SafeVarargs
-    public static <T extends Comparable<T>> Iterator<T> union(Iterator<T>... iterators)
-    {
-        return union(Arrays.asList(iterators));
-    }
-
     /**
      * Merge k sorted iterators into one by keeping the order and only one copy of each element.
      *
@@ -73,57 +68,34 @@ public class SequenceUtils
 
         return new Iterator<T>()
         {
-            T next;
+            T prev;
 
             public boolean hasNext()
             {
-                if (pq.isEmpty()) {
-                    return false;
-                }
-                else if (keepDuplication) {
-                    PeekingIterator<T> curr = pq.poll();
-                    next = curr.next();
-                    if (curr.hasNext()) {
-                        pq.add(curr);
-                    }
-                    return true;
-                }
-                else {
-                    PeekingIterator<T> curr = pq.poll();
-                    next = curr.next();
-                    if (curr.hasNext()) {
-                        pq.add(curr);
-                    }
-                    while (!pq.isEmpty()) {
-                        PeekingIterator<T> it = pq.poll();
-                        if (it.peek().equals(next)) {
-                            it.next();
-                            if (it.hasNext()) {
-                                pq.add(it);
-                            }
-                        }
-                        else {
-                            pq.add(it);
-                            return true;
-                        }
-                    }
-                    return true;
-                }
+                return !pq.isEmpty();
             }
 
             public T next()
             {
-                return next;
+                PeekingIterator<T> curr = pq.poll();
+                T val = curr.hasNext() ? curr.next() : null;
+                if (curr.hasNext()) {
+                    pq.add(curr);
+                }
+                if (keepDuplication) {
+                    return val;
+                }
+                else {
+                    if (prev == null || !prev.equals(val)) {
+                        prev = val;
+                        return val;
+                    }
+                    return next();
+                }
             }
         };
     }
 
-    @SafeVarargs
-    public static <T extends Comparable<T>> Iterator<T> merge(boolean keepDuplication, Iterator<T>... iterators)
-    {
-        return merge(Arrays.asList(iterators), keepDuplication);
-    }
-
     /**
      * Intersect k sorted iterators to find the common values in them.
      *
@@ -199,12 +171,6 @@ public class SequenceUtils
         };
     }
 
-    @SafeVarargs
-    public static <T extends Comparable<T>> Iterator<T> intersect(Iterator<T>... iterators)
-    {
-        return intersect(Arrays.asList(iterators));
-    }
-
     private static <T> T allEquals(T[] arr)
     {
         if (arr.length < 1) {

hetu-common/src/main/java/io/hetu/core/common/filesystem/TempFolder.java

@@ -13,13 +13,9 @@
  */
 package io.hetu.core.common.filesystem;
 
-import io.airlift.log.Logger;
-
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Comparator;
 import java.util.UUID;
 
 /**
@@ -28,24 +24,14 @@ import java.util.UUID;
 public class TempFolder
         implements AutoCloseable
 {
-    private static final Logger LOG = Logger.get(TempFolder.class);
-    private final String prefix;
     private File root;
 
-    public TempFolder()
-    {
-        this("");
-    }
-
-    public TempFolder(String user)
-    {
-        this.prefix = "hetu-tmp-folder-" + user;
-    }
+    public TempFolder() {}
 
     public TempFolder create()
             throws IOException
     {
-        root = Files.createTempDirectory(prefix).toFile();
+        root = Files.createTempDirectory("hetu-tmp-folder-").toFile();
         return this;
     }
 
@@ -86,36 +72,6 @@ public class TempFolder
         throw new IOException("Not able to create folder " + relativePath);
     }
 
-    @Override
-    public void close()
-    {
-        if (root != null && root.exists()) {
-            // retry deletion for 3 times
-            for (int i = 0; i < 3; i++) {
-                if (deleteRecursively(root)) {
-                    return;
-                }
-            }
-
-            if (!root.exists()) {
-                return;
-            }
-
-            LOG.warn("Temporary folder can not be deleted. Shutdown hook added: " + root.toPath().toAbsolutePath());
-            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
-                try {
-                    Files.walk(root.toPath())
-                            .sorted(Comparator.reverseOrder())
-                            .map(Path::toFile)
-                            .forEach(File::delete);
-                }
-                catch (IOException e) {
-                    LOG.warn("Temporary folder not deleted. Manual deletion required: " + root.toPath().toAbsolutePath());
-                }
-            }));
-        }
-    }
-
     private boolean deleteRecursively(File fileToDelete)
     {
         if (fileToDelete.delete()) {
@@ -131,4 +87,14 @@ public class TempFolder
         }
         return fileToDelete.delete();
     }
+
+    @Override
+    public void close()
+    {
+        if (root != null && root.exists()) {
+            if (!deleteRecursively(root)) {
+                throw new RuntimeException("Temporary folder not deleted. Manual deletion required.");
+            }
+        }
+    }
 }

hetu-common/src/main/java/io/hetu/core/common/heuristicindex/IndexCacheKey.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -12,48 +12,28 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.prestosql.spi.heuristicindex;
-
-import io.prestosql.spi.connector.CreateIndexMetadata;
+package io.hetu.core.common.heuristicindex;
 
 import java.util.Objects;
 
 public class IndexCacheKey
 {
-    public static final long LAST_MODIFIED_TIME_PLACE_HOLDER = 0;
-
-    private final String path;
-    private final long lastModifiedTime;
-    private final IndexRecord record;
-    private final CreateIndexMetadata.Level indexLevel;
-    private boolean noCloseFlag; // Indicate that this index should not be closed at removal
+    private String path;
+    private long lastModifiedTime;
+    private String indexLevel = "STRIPE";
 
     /**
-     * @param path path to the file the index files should be read for
+     * @param path             path to the file the index files should be read for
      * @param lastModifiedTime lastModifiedTime of the file, used to validate the indexes
-     * @param record the index record associated with the cache key
-     * @param indexLevel see Index.Level in presto-spi
+     * @param indexLevel       see Index.Level in presto-spi
      */
-    public IndexCacheKey(String path, long lastModifiedTime, IndexRecord record, CreateIndexMetadata.Level indexLevel)
+    public IndexCacheKey(String path, long lastModifiedTime, String indexLevel)
     {
         this.path = path;
         this.lastModifiedTime = lastModifiedTime;
-        this.record = record;
         this.indexLevel = indexLevel;
     }
 
-    /**
-     * Create a cache with a index level it could be Stripe or partition
-     *
-     * @param path
-     * @param lastModifiedTime
-     * @param record the index record associated with the cache key
-     */
-    public IndexCacheKey(String path, long lastModifiedTime, IndexRecord record)
-    {
-        this(path, lastModifiedTime, record, CreateIndexMetadata.Level.STRIPE);
-    }
-
     /**
      * Create a cache with a index level it could be Stripe or partition
      *
@@ -62,7 +42,7 @@ public class IndexCacheKey
      */
     public IndexCacheKey(String path, long lastModifiedTime)
     {
-        this(path, lastModifiedTime, null, CreateIndexMetadata.Level.STRIPE);
+        this(path, lastModifiedTime, "STRIPE");
     }
 
     public String getPath()
@@ -75,27 +55,12 @@ public class IndexCacheKey
         return lastModifiedTime;
     }
 
-    public IndexRecord getRecord()
-    {
-        return record;
-    }
-
-    public CreateIndexMetadata.Level getIndexLevel()
+    public String getIndexLevel()
     {
         return this.indexLevel;
     }
 
-    public void setNoCloseFlag(boolean flag)
-    {
-        this.noCloseFlag = flag;
-    }
-
-    public boolean skipCloseIndex()
-    {
-        return noCloseFlag;
-    }
-
-    // only the path is used as the key
+    // only the path should be used as the key
     // the lastModifiedTime time is only used to check if index is valid
     @Override
     public boolean equals(Object o)

hetu-common/src/main/java/io/hetu/core/common/util/SecurePathWhiteList.java

@@ -16,7 +16,6 @@ package io.hetu.core.common.util;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
@@ -37,7 +36,6 @@ public class SecurePathWhiteList
 
     /**
      * Due to security concerns, all data must be read from one of the whitelisted paths
-     *
      * @return
      * @throws IOException
      */
@@ -55,11 +53,6 @@ public class SecurePathWhiteList
         return securePathwhiteList;
     }
 
-    public static boolean isSecurePath(Path absolutePath) throws IOException
-    {
-        return isSecurePath(absolutePath.toString());
-    }
-
     public static boolean isSecurePath(String absolutePath) throws IOException
     {
         // absolutePath

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,6 +47,16 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
+
+        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
+        // as per link above, the default SSLContext will be constructed using the default KeyManager and
+        // default TrustManager. Those can be configured using the following system properties:
+        // javax.net.ssl.keyStore
+        // javax.net.ssl.keyStorePassword
+        // javax.net.ssl.keyStoreType
+        // javax.net.ssl.trustStore
+        // javax.net.ssl.trustStorePassword
+        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,7 +13,6 @@
  */
 package io.hetu.core.common.util;
 
-import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -30,8 +29,6 @@ import java.util.Optional;
 
 public class TrustStore
 {
-    private static final Logger LOGGER = Logger.get(TrustStore.class);
-
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -51,7 +48,6 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
-            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/algorithm/TestSequenceUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -18,7 +18,6 @@ package io.hetu.core.common.algorithm;
 import com.google.common.collect.ImmutableList;
 import org.testng.annotations.Test;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
@@ -34,16 +33,15 @@ public class TestSequenceUtils
     List<Integer> l3 = ImmutableList.of(5);
     Iterator<Integer> emptyIterator = Collections.emptyIterator();
 
-    private static <T1, T2> void assertIteratorSame(Iterator<T1> iterator1, Iterator<T2> iterator2)
+    private static void assertIteratorSame(Iterator iterator1, Iterator iterator2)
     {
         while (iterator1.hasNext()) {
             assertTrue(iterator2.hasNext());
-            T1 v1 = iterator1.next();
-            T2 v2 = iterator2.next();
-            System.out.println(v1 + "---" + v2);
-            assertEquals(v1, v2);
+            assertEquals(iterator2.next(), iterator1.next());
+        }
+        if (iterator2.hasNext()) {
+            System.out.println(iterator2.next());
         }
-        System.out.println();
         assertFalse(iterator2.hasNext());
     }
 
@@ -75,32 +73,4 @@ public class TestSequenceUtils
         assertIteratorSame(SequenceUtils.intersect(ImmutableList.of(l1.iterator(), l2.iterator(), l3.iterator(), emptyIterator)),
                 ImmutableList.of().iterator());
     }
-
-    @Test
-    public void testRepeatingValues()
-    {
-        List<String> l = new ArrayList<>(10000);
-        for (int i = 0; i < 10000; i++) {
-            l.add(String.valueOf(10000));
-        }
-        Iterator<String> it = SequenceUtils.union(ImmutableList.of(l.iterator()));
-        // detect stack overflow
-        while (it.hasNext()) {
-            it.next();
-        }
-    }
-
-    @Test
-    public void testDupAsEnd()
-    {
-        List<Integer> i1 = ImmutableList.of(1, 4, 5, 5, 7, 7, 7);
-        List<Integer> i2 = ImmutableList.of(2, 2, 2, 3, 5, 5, 5);
-        List<Integer> i3 = ImmutableList.of(6, 6, 6, 6);
-        List<Integer> i4 = ImmutableList.of(3);
-
-        assertIteratorSame(SequenceUtils.union(ImmutableList.of(i1.iterator(), i2.iterator(), i3.iterator(), i4.iterator())),
-                ImmutableList.of(1, 2, 3, 4, 5, 6, 7).iterator());
-        assertIteratorSame(SequenceUtils.merge(ImmutableList.of(i1.iterator(), i2.iterator(), i3.iterator(), i4.iterator()), true),
-                ImmutableList.of(1, 2, 2, 2, 3, 3, 4, 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7).iterator());
-    }
 }

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
+            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
+            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>io.hetu.core</groupId>
-        <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
-    </parent>
-
-    <artifactId>hetu-cube</artifactId>
-    <name>hetu-cube</name>
-    <packaging>jar</packaging>
-
-    <properties>
-        <air.main.basedir>${project.parent.basedir}</air.main.basedir>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <scope>provided</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-</project>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeAggregateFunction.java

@@ -1,47 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public enum CubeAggregateFunction
-{
-    SUM("sum"),
-    COUNT("count"),
-    AVG("avg"),
-    MIN("min"),
-    MAX("max");
-
-    private final String name;
-
-    public static final Set<String> SUPPORTED_FUNCTIONS = Stream.of(CubeAggregateFunction.values()).map(CubeAggregateFunction::getName).collect(Collectors.toSet());
-
-    CubeAggregateFunction(String name)
-    {
-        this.name = name;
-    }
-
-    public String getName()
-    {
-        return this.name;
-    }
-
-    public String toString()
-    {
-        return this.name;
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -1,80 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.Objects;
-
-public class CubeFilter
-{
-    private final String sourceTablePredicate;
-    private final String cubePredicate;
-
-    @JsonCreator
-    public CubeFilter(
-            @JsonProperty("sourceTablePredicate") String sourceTablePredicate,
-            @JsonProperty("cubePredicate") String cubePredicate)
-    {
-        this.sourceTablePredicate = sourceTablePredicate;
-        this.cubePredicate = cubePredicate;
-    }
-
-    public CubeFilter(String sourceTablePredicate)
-    {
-        this(sourceTablePredicate, null);
-    }
-
-    public String getSourceTablePredicate()
-    {
-        return sourceTablePredicate;
-    }
-
-    public String getCubePredicate()
-    {
-        return cubePredicate;
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        CubeFilter that = (CubeFilter) o;
-        return Objects.equals(sourceTablePredicate, that.sourceTablePredicate)
-                && Objects.equals(cubePredicate, that.cubePredicate);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(sourceTablePredicate, cubePredicate);
-    }
-
-    @Override
-    public String toString()
-    {
-        return "CubeFilter{" +
-                "sourceTablePredicate='" + sourceTablePredicate + '\'' +
-                ", cubePredicate='" + cubePredicate + '\'' +
-                '}';
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeMetadata.java

@@ -1,117 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import io.hetu.core.spi.cube.aggregator.AggregationSignature;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-public interface CubeMetadata
-        extends Serializable
-{
-    /**
-     * Returns name of the cube
-     */
-    String getCubeName();
-
-    /**
-     * Returns the name of the source table
-     */
-    String getSourceTableName();
-
-    /**
-     * Returns the last updated time of the cube
-     */
-    long getLastUpdatedTime();
-
-    /**
-     * Returns the last updated time of the source table
-     */
-    long getSourceTableLastUpdatedTime();
-
-    /**
-     * Return the names of the dimension columns
-     */
-    List<String> getDimensions();
-
-    /**
-     * Return the names of the aggregation columns
-     */
-    List<String> getAggregations();
-
-    /**
-     * Cube selection filter
-     */
-    CubeFilter getCubeFilter();
-
-    /**
-     * Return the group by columns
-     */
-    Set<String> getGroup();
-
-    /**
-     * Checks if metadata matches the CubeStatement
-     * @param statement cube statement
-     * @return true - if metadata matches CubeStatement
-     *         false - otherwise
-     */
-    boolean matches(CubeStatement statement);
-
-    /**
-     * Filters all metadata that matches the cube statement
-     * @param metadataList metadata list
-     * @param statement cube statement
-     * @return all metadata that is matching the cube statement
-     */
-    static List<CubeMetadata> filter(List<CubeMetadata> metadataList, CubeStatement statement)
-    {
-        return metadataList.stream()
-                .filter(metadata -> metadata.matches(statement))
-                .collect(Collectors.toList());
-    }
-
-    /**
-     * Retrieves the cube column matching the given aggregation signature
-     * @return name of the aggregation column if found
-     */
-    Optional<String> getColumn(AggregationSignature aggSignature);
-
-    /**
-     * Return the aggregation function associated with given cube column
-     * @return name of the aggregation function
-     */
-    Optional<String> getAggregationFunction(String column);
-
-    /**
-     * Get the aggregation information of the given cube column
-     * @param column name of the cube column
-     */
-    Optional<AggregationSignature> getAggregationSignature(String column);
-
-    /**
-     * Return all aggregation column information
-     */
-    List<AggregationSignature> getAggregationSignatures();
-
-    /**
-     * Return the status of the cube
-     */
-    CubeStatus getCubeStatus();
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeMetadataBuilder.java

@@ -1,37 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import java.util.Set;
-
-public interface CubeMetadataBuilder
-{
-    void addDimensionColumn(String name, String originalColumn);
-
-    void addAggregationColumn(String name, String aggregationFunction, String originalColumn, boolean distinct);
-
-    void addGroup(Set<String> group);
-
-    void withCubeFilter(CubeFilter cubeFilter);
-
-    void setCubeStatus(CubeStatus cubeStatus);
-
-    void setTableLastUpdatedTime(long tableLastUpdatedTime);
-
-    void setCubeLastUpdatedTime(long cubeLastUpdatedTime);
-
-    CubeMetadata build();
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -1,163 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import io.hetu.core.spi.cube.aggregator.AggregationSignature;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.StringJoiner;
-
-import static java.util.Objects.requireNonNull;
-
-public class CubeStatement
-{
-    private final String from;
-    private final Set<String> groupBy;
-    private final Set<String> selection;
-    private final List<AggregationSignature> aggregations;
-
-    private CubeStatement(
-            Set<String> selection,
-            String from,
-            Set<String> groupBy,
-            List<AggregationSignature> aggregations)
-    {
-        this.selection = requireNonNull(selection, "selection is null");
-        this.from = requireNonNull(from, "from is null");
-        this.groupBy = requireNonNull(groupBy, "groupBy is null");
-        this.aggregations = requireNonNull(aggregations, "aggregations is null");
-    }
-
-    public static Builder newBuilder()
-    {
-        return new Builder();
-    }
-
-    public Set<String> getSelection()
-    {
-        return selection;
-    }
-
-    public String getFrom()
-    {
-        return from;
-    }
-
-    public List<AggregationSignature> getAggregations()
-    {
-        return aggregations;
-    }
-
-    public Set<String> getGroupBy()
-    {
-        return groupBy;
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        CubeStatement that = (CubeStatement) o;
-        return Objects.equals(selection, that.selection) &&
-                Objects.equals(from, that.from) &&
-                Objects.equals(groupBy, that.groupBy) &&
-                Objects.equals(aggregations, that.aggregations);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(selection, from, groupBy, aggregations);
-    }
-
-    @Override
-    public String toString()
-    {
-        StringJoiner columns = new StringJoiner(", ");
-        selection.forEach(columns::add);
-        aggregations.forEach(agg -> columns.add(agg.toString()));
-
-        StringJoiner groupingColumns = new StringJoiner(", ");
-        groupBy.forEach(groupingColumns::add);
-
-        return "SELECT " + columns +
-                " FROM " + from +
-                (groupBy.isEmpty() ? "" : " GROUP BY " + groupingColumns);
-    }
-
-    public static class Builder
-    {
-        private String from;
-        private final Set<String> groupBy = new HashSet<>();
-        private final Set<String> selection = new HashSet<>();
-        private final List<AggregationSignature> aggregations = new ArrayList<>();
-
-        private Builder()
-        {
-            // Do nothing
-        }
-
-        public Builder select(String column, String... columns)
-        {
-            this.selection.add(column);
-            this.selection.addAll(Arrays.asList(columns));
-            return this;
-        }
-
-        public Builder aggregate(AggregationSignature signature)
-        {
-            this.aggregations.add(signature);
-            return this;
-        }
-
-        public Builder from(String from)
-        {
-            this.from = from;
-            return this;
-        }
-
-        public Builder groupByAddString(String column)
-        {
-            this.groupBy.add(column);
-            return this;
-        }
-
-        public Builder groupByAddStringList(String... columns)
-        {
-            this.groupBy.addAll(Arrays.asList(columns));
-            return this;
-        }
-
-        public CubeStatement build()
-        {
-            if (this.aggregations.isEmpty() && this.selection.isEmpty()) {
-                throw new UnsupportedOperationException("Cannot construct a cube statement without selection and aggregation");
-            }
-            return new CubeStatement(Collections.unmodifiableSet(selection), from, Collections.unmodifiableSet(groupBy), Collections.unmodifiableList(aggregations));
-        }
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatus.java

@@ -1,45 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-public enum CubeStatus
-{
-    INACTIVE(0),
-    READY(1);
-
-    private final int value;
-
-    CubeStatus(int value)
-    {
-        this.value = value;
-    }
-
-    public int getValue()
-    {
-        return this.value;
-    }
-
-    public static CubeStatus forValue(int value)
-    {
-        if (value == 0) {
-            return INACTIVE;
-        }
-        else if (value == 1) {
-            return READY;
-        }
-        return null;
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/aggregator/AggregationSignature.java

@@ -1,142 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube.aggregator;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import static io.hetu.core.spi.cube.CubeAggregateFunction.AVG;
-import static io.hetu.core.spi.cube.CubeAggregateFunction.COUNT;
-import static io.hetu.core.spi.cube.CubeAggregateFunction.MAX;
-import static io.hetu.core.spi.cube.CubeAggregateFunction.MIN;
-import static io.hetu.core.spi.cube.CubeAggregateFunction.SUM;
-
-public class AggregationSignature
-        implements Serializable, Comparable<AggregationSignature>
-{
-    private static final AggregationSignature COUNT_SIGNATURE = new AggregationSignature(COUNT.getName(), "*", false);
-
-    private String function;
-    private String dimension;
-    private boolean distinct;
-
-    @JsonCreator
-    public AggregationSignature(
-            @JsonProperty("function") String function,
-            @JsonProperty("dimension") String dimension,
-            @JsonProperty("distinct") boolean distinct)
-    {
-        this.function = function;
-        this.dimension = dimension;
-        this.distinct = distinct;
-    }
-
-    public static AggregationSignature count()
-    {
-        return COUNT_SIGNATURE;
-    }
-
-    public static AggregationSignature count(String dimension, boolean distinct)
-    {
-        return new AggregationSignature(COUNT.getName(), dimension, distinct);
-    }
-
-    public static AggregationSignature sum(String dimension, boolean distinct)
-    {
-        return new AggregationSignature(SUM.toString(), dimension, distinct);
-    }
-
-    public static AggregationSignature avg(String dimension, boolean distinct)
-    {
-        return new AggregationSignature(AVG.toString(), dimension, distinct);
-    }
-
-    public static AggregationSignature min(String dimension, boolean distinct)
-    {
-        return new AggregationSignature(MIN.getName(), dimension, distinct);
-    }
-
-    public static AggregationSignature max(String dimension, boolean distinct)
-    {
-        return new AggregationSignature(MAX.getName(), dimension, distinct);
-    }
-
-    @JsonProperty
-    public String getFunction()
-    {
-        return function;
-    }
-
-    @JsonProperty
-    public String getDimension()
-    {
-        return dimension;
-    }
-
-    @JsonProperty
-    public boolean isDistinct()
-    {
-        return distinct;
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(function, dimension);
-    }
-
-    @Override
-    public boolean equals(Object obj)
-    {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null || getClass() != obj.getClass()) {
-            return false;
-        }
-        AggregationSignature other = (AggregationSignature) obj;
-        return Objects.equals(this.function, other.function) &&
-                Objects.equals(this.dimension, other.dimension) &&
-                Objects.equals(this.distinct, other.distinct);
-    }
-
-    @Override
-    public String toString()
-    {
-        return this.function + "(" + (this.distinct ? "distinct " : "") + this.dimension + ")";
-    }
-
-    @Override
-    public int compareTo(AggregationSignature aggregationSignature)
-    {
-        int nameComparison = function.compareTo(aggregationSignature.function);
-        if (0 != nameComparison) {
-            return nameComparison;
-        }
-        else {
-            int dimensionComparison = dimension.compareTo(aggregationSignature.dimension);
-            if (0 != dimensionComparison) {
-                return dimensionComparison;
-            }
-            else {
-                return Boolean.compare(distinct, aggregationSignature.distinct);
-            }
-        }
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/io/CubeMetaStore.java

@@ -1,75 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube.io;
-
-import io.hetu.core.spi.cube.CubeMetadata;
-import io.hetu.core.spi.cube.CubeMetadataBuilder;
-
-import java.util.List;
-import java.util.Optional;
-
-/**
- * CubeMetaStore provides APIs to retrieve, update the Cube metadata
- * from the underlying metastore.
- */
-public interface CubeMetaStore
-{
-    /**
-     * Persist cube metadata in the underlying metastore
-     * @param cubeMetadata cube metadata
-     */
-    void persist(CubeMetadata cubeMetadata);
-
-    /**
-     * Create a new Metadata builder
-     * @param cubeName Name of the cube
-     * @param sourceTableName Name of the table from which cube was created
-     * @return a metadata builder
-     */
-    CubeMetadataBuilder getBuilder(String cubeName, String sourceTableName);
-
-    /**
-     * Create new metadata builder from the existing metadata
-     * @param existingMetadata existing metadata
-     * @return a metadata builder
-     */
-    CubeMetadataBuilder getBuilder(CubeMetadata existingMetadata);
-
-    /**
-     * Returns the list of cube metadata associated with the given table.
-     * @param tableName fully qualified name of the table
-     * @return list of cube metadata
-     */
-    List<CubeMetadata> getMetadataList(String tableName);
-
-    /**
-     * Find a cube metadata associated with the given name
-     * @param cubeName name of the cube
-     * @return optional cube metadata
-     */
-    Optional<CubeMetadata> getMetadataFromCubeName(String cubeName);
-
-    /**
-     * Return all cube information
-     * @return list of cube metadata
-     */
-    List<CubeMetadata> getAllCubes();
-
-    /**
-     * Remove cube metadata
-     */
-    void removeCube(CubeMetadata cubeMetadata);
-}

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -1,71 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import io.hetu.core.spi.cube.aggregator.AggregationSignature;
-import org.testng.annotations.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-
-public class TestCubeStatement
-{
-    @Test
-    public void testCubeStatement()
-    {
-        CubeStatement statement = CubeStatement.newBuilder()
-                .select("name", "address", "nationkey")
-                .aggregate(AggregationSignature.count())
-                .from("tpch.tiny.customer")
-                .groupByAddString("address")
-                .groupByAddStringList("name", "nationkey")
-                .build();
-
-        assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");
-        assertEquals(statement.getSelection(), new HashSet<>(Arrays.asList("name", "address", "nationkey")), "incorrect selection");
-        assertEquals(statement.getGroupBy(), new HashSet<>(Arrays.asList("name", "address", "nationkey")), "incorrect address");
-        assertEquals(statement.getAggregations(), Collections.singletonList(AggregationSignature.count()), "incorrect aggregations");
-    }
-
-    @Test
-    public void testCubeEquality()
-    {
-        CubeStatement statement1 = CubeStatement.newBuilder()
-                .select("name", "address", "nationkey")
-                .aggregate(AggregationSignature.count())
-                .from("tpch.tiny.customer")
-                .build();
-
-        CubeStatement statement2 = CubeStatement.newBuilder()
-                .select("name", "address", "nationkey")
-                .aggregate(AggregationSignature.count())
-                .from("tpch.tiny.customer")
-                .build();
-
-        CubeStatement statement3 = CubeStatement.newBuilder()
-                .select("name", "address")
-                .aggregate(AggregationSignature.count())
-                .from("tpch.tiny.customer")
-                .build();
-
-        assertEquals(statement1, statement2, "statements are not equal");
-        assertNotEquals(statement1, statement3, "statements are not equal");
-    }
-}

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>
@@ -73,11 +73,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>io.hetu.core</groupId>
-            <artifactId>presto-expressions</artifactId>
-        </dependency>
-
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>slice</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,7 +55,6 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
-    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -20,7 +20,6 @@ import io.airlift.configuration.ConfigDescription;
 import io.airlift.configuration.ConfigSecuritySensitive;
 import io.airlift.units.DataSize;
 import io.airlift.units.Duration;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
 import io.prestosql.spi.function.Mandatory;
 
 import javax.annotation.Nullable;
@@ -104,8 +103,6 @@ public class DataCenterConfig
 
     private boolean isQueryPushDownEnabled = true;
 
-    private JdbcPushDownModule queryPushDownModule = JdbcPushDownModule.DEFAULT;
-
     private Duration metadataCacheTtl = new Duration(1, TimeUnit.SECONDS); // DataCenter metadata cache eviction time
 
     private long metadataCacheMaximumSize = DEFAULT_METADATA_CACHE_MAX_SIZE; // DataCenter metadata cache max size
@@ -247,6 +244,10 @@ public class DataCenterConfig
      * @param connectionUser the connection user name.
      * @return DataCenterConfig object.
      */
+    @Mandatory(name = "connection-user",
+            description = "User to connect to remote data center",
+            defaultValue = "lk",
+            required = true)
     @Config("connection-user")
     public DataCenterConfig setConnectionUser(String connectionUser)
     {
@@ -690,25 +691,6 @@ public class DataCenterConfig
         return this;
     }
 
-    public JdbcPushDownModule getQueryPushDownModule()
-    {
-        return queryPushDownModule;
-    }
-
-    /**
-     * set queryPushDownEnabled
-     *
-     * @param queryPushDownModule Push Down Module
-     * @return DataCenterConfig object
-     */
-    @Config("dc.query.pushdown.module")
-    @ConfigDescription("query push down module [FULL_PUSHDOWN/BASE_PUSHDOWN]")
-    public DataCenterConfig setQueryPushDownModule(JdbcPushDownModule queryPushDownModule)
-    {
-        this.queryPushDownModule = queryPushDownModule;
-        return this;
-    }
-
     public DataSize getRemoteHttpServerMaxRequestHeaderSize()
     {
         return remoteHeaderSize;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConnector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -15,19 +15,15 @@
 
 package io.hetu.core.plugin.datacenter;
 
-import com.google.common.collect.ImmutableSet;
 import io.airlift.bootstrap.LifeCycleManager;
 import io.airlift.log.Logger;
 import io.hetu.core.plugin.datacenter.client.DataCenterClient;
 import io.hetu.core.plugin.datacenter.client.DataCenterStatementClientFactory;
-import io.hetu.core.plugin.datacenter.optimization.DataCenterPlanOptimizer;
 import io.hetu.core.plugin.datacenter.pagesource.DataCenterPageSourceProvider;
-import io.prestosql.spi.ConnectorPlanOptimizer;
 import io.prestosql.spi.connector.CachedConnectorMetadata;
 import io.prestosql.spi.connector.Connector;
 import io.prestosql.spi.connector.ConnectorMetadata;
 import io.prestosql.spi.connector.ConnectorPageSourceProvider;
-import io.prestosql.spi.connector.ConnectorPlanOptimizerProvider;
 import io.prestosql.spi.connector.ConnectorSplitManager;
 import io.prestosql.spi.connector.ConnectorTransactionHandle;
 import io.prestosql.spi.transaction.IsolationLevel;
@@ -38,7 +34,6 @@ import javax.inject.Inject;
 
 import java.util.Collection;
 import java.util.Map;
-import java.util.Set;
 
 import static io.hetu.core.plugin.datacenter.DataCenterTransactionHandle.INSTANCE;
 import static java.util.Objects.requireNonNull;
@@ -65,8 +60,6 @@ public class DataCenterConnector
 
     private final OkHttpClient httpClient;
 
-    private final ConnectorPlanOptimizer planOptimizer;
-
     /**
      * Constructor of data center connector.
      *
@@ -75,18 +68,14 @@ public class DataCenterConnector
      * @param typeManager the type manager.
      */
     @Inject
-    public DataCenterConnector(
-            LifeCycleManager lifeCycleManager,
-            DataCenterConfig dataCenterConfig,
-            TypeManager typeManager,
-            DataCenterPlanOptimizer planOptimizer)
+    public DataCenterConnector(LifeCycleManager lifeCycleManager, DataCenterConfig dataCenterConfig,
+            TypeManager typeManager)
     {
         this.lifeCycleManager = requireNonNull(lifeCycleManager, "lifeCycleManager is null");
         this.httpClient = DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
         this.dataCenterClient = new DataCenterClient(dataCenterConfig, this.httpClient, typeManager);
         this.splitManager = new DataCenterSplitManager(dataCenterConfig, this.dataCenterClient);
         this.pageSourceProvider = new DataCenterPageSourceProvider(dataCenterConfig, this.httpClient, typeManager);
-        this.planOptimizer = planOptimizer;
         if (dataCenterConfig.isMetadataCacheEnabled()) {
             this.metadata = new CachedConnectorMetadata(new DataCenterMetadata(dataCenterClient, dataCenterConfig),
                     dataCenterConfig.getMetadataCacheTtl(), dataCenterConfig.getMetadataCacheMaximumSize());
@@ -96,25 +85,6 @@ public class DataCenterConnector
         }
     }
 
-    @Override
-    public ConnectorPlanOptimizerProvider getConnectorPlanOptimizerProvider()
-    {
-        return new ConnectorPlanOptimizerProvider()
-        {
-            @Override
-            public Set<ConnectorPlanOptimizer> getLogicalPlanOptimizers()
-            {
-                return ImmutableSet.of(planOptimizer);
-            }
-
-            @Override
-            public Set<ConnectorPlanOptimizer> getPhysicalPlanOptimizers()
-            {
-                return ImmutableSet.of();
-            }
-        };
-    }
-
     @Override
     public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean isReadOnly)
     {

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConnectorFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -22,10 +22,6 @@ import io.prestosql.spi.connector.Connector;
 import io.prestosql.spi.connector.ConnectorContext;
 import io.prestosql.spi.connector.ConnectorFactory;
 import io.prestosql.spi.connector.ConnectorHandleResolver;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpressionService;
 
 import java.util.Map;
 
@@ -58,15 +54,7 @@ public class DataCenterConnectorFactory
         requireNonNull(requiredConfig, "requiredConfig is null");
         try {
             // A plugin is not required to use Guice; it is just very convenient
-            Bootstrap app = new Bootstrap(
-                    binder -> {
-                        binder.bind(FunctionMetadataManager.class).toInstance(context.getFunctionMetadataManager());
-                        binder.bind(StandardFunctionResolution.class).toInstance(context.getStandardFunctionResolution());
-                        binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService());
-                        binder.bind(DeterminismEvaluator.class).toInstance(context.getRowExpressionService().getDeterminismEvaluator());
-                    },
-                    new JsonModule(),
-                    new DataCenterModule(context.getTypeManager()));
+            Bootstrap app = new Bootstrap(new JsonModule(), new DataCenterModule(context.getTypeManager()));
 
             Injector injector = app.strictConfig()
                     .doNotInitializeLogging()

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterMetadata.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -32,9 +32,13 @@ import io.prestosql.spi.connector.LimitApplicationResult;
 import io.prestosql.spi.connector.SchemaNotFoundException;
 import io.prestosql.spi.connector.SchemaTableName;
 import io.prestosql.spi.connector.SchemaTablePrefix;
+import io.prestosql.spi.connector.SubQueryApplicationResult;
 import io.prestosql.spi.connector.TableNotFoundException;
+import io.prestosql.spi.sql.SqlQueryWriter;
 import io.prestosql.spi.statistics.TableStatistics;
+import io.prestosql.spi.type.Type;
 
+import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -242,7 +246,50 @@ public class DataCenterMetadata
     }
 
     @Override
-    public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, boolean includeColumnStatistics)
+    public Optional<SubQueryApplicationResult<ConnectorTableHandle>> applySubQuery(ConnectorSession session,
+            ConnectorTableHandle handle, String subQuery, Map<String, Type> types)
+    {
+        if (!isQueryPushDownEnabled || subQuery.getBytes(StandardCharsets.ISO_8859_1).length >= maxRemoteHeaderSize) {
+            return Optional.empty();
+        }
+
+        // If the subQuery pushed down to the connector, table name, limit or predicate push downs are not necessary
+        // Therefore, either of the table name can be used for the new TableHandle as long as the subQuery is valid
+        requireNonNull(subQuery, "cannot apply null sub-query");
+        DataCenterTableHandle tableHandle = (DataCenterTableHandle) handle;
+
+        // If we can get the columns from the sub-query, it should be able to push sub-query down
+        List<DataCenterColumn> columns = dataCenterClient.getColumns(subQuery);
+        if (columns.isEmpty()) {
+            return Optional.empty();
+        }
+        DataCenterTableHandle newTableHandle = new DataCenterTableHandle(tableHandle.getCatalogName(),
+                tableHandle.getSchemaName(), tableHandle.getTableName(), OptionalLong.empty(), subQuery);
+
+        ImmutableMap.Builder<String, ColumnHandle> columnHandleBuilder = new ImmutableMap.Builder<>();
+        ImmutableMap.Builder<String, Type> typesBuilder = new ImmutableMap.Builder<>();
+
+        columns.forEach(column -> {
+            columnHandleBuilder.put(column.getName(),
+                    new DataCenterColumnHandle(column.getName(), column.getType(), 0));
+            typesBuilder.put(column.getName(), column.getType());
+        });
+
+        return Optional.of(
+                new SubQueryApplicationResult<>(newTableHandle, columnHandleBuilder.build(), typesBuilder.build()));
+    }
+
+    @Override
+    public Optional<SqlQueryWriter> getSqlQueryWriter()
+    {
+        if (!isQueryPushDownEnabled) {
+            return Optional.empty();
+        }
+        return Optional.of(new DataCenterSqlQueryWriter());
+    }
+
+    @Override
+    public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint)
     {
         Map<String, ColumnHandle> columnHandles = getColumnHandles(session, tableHandle);
         String tableFullName = tableHandle.getSchemaPrefixedTableName();

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterModule.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -18,8 +18,6 @@ package io.hetu.core.plugin.datacenter;
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import com.google.inject.Scopes;
-import io.hetu.core.plugin.datacenter.optimization.DataCenterPlanOptimizer;
-import io.hetu.core.plugin.datacenter.optimization.DataCenterQueryGenerator;
 import io.prestosql.spi.type.TypeManager;
 
 import static io.airlift.configuration.ConfigBinder.configBinder;
@@ -50,8 +48,6 @@ public class DataCenterModule
     {
         binder.bind(TypeManager.class).toInstance(typeManager);
         binder.bind(DataCenterConnector.class).in(Scopes.SINGLETON);
-        binder.bind(DataCenterPlanOptimizer.class).in(Scopes.SINGLETON);
-        binder.bind(DataCenterQueryGenerator.class).in(Scopes.SINGLETON);
         configBinder(binder).bindConfig(DataCenterConfig.class);
     }
 }

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterSqlQueryWriter.java

@@ -13,23 +13,27 @@
  * limitations under the License.
  */
 
-package io.prestosql.catalog.showcatalog;
+package io.hetu.core.plugin.datacenter;
 
-import com.google.inject.Binder;
-import com.google.inject.Scopes;
-import io.airlift.configuration.AbstractConfigurationAwareModule;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import io.prestosql.spi.sql.expression.Selection;
+import io.prestosql.sql.builder.BaseSqlQueryWriter;
 
-import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder;
+import java.util.Map;
+import java.util.Optional;
 
-public class ShowCatalogModule
-        extends AbstractConfigurationAwareModule
+/**
+ * Implementation of BaseSqlQueryWriter. It knows how to write
+ * Hetu SQL for the logical plan.
+ */
+public class DataCenterSqlQueryWriter
+        extends BaseSqlQueryWriter
 {
     @Override
-    protected void setup(Binder binder)
+    public String formatIdentifier(Optional<Map<String, Selection>> qualifiedNames, String identifier)
     {
-        jaxrsBinder(binder).bind(ShowCatalogResource.class);
-        jaxrsBinder(binder).bind(MultiPartFeature.class);
-        binder.bind(ShowCatalogService.class).in(Scopes.SINGLETON);
+        if (qualifiedNames.isPresent()) {
+            return qualifiedNames.get().get(identifier).getExpression();
+        }
+        return '"' + identifier.replace("\"", "\"\"") + '"';
     }
 }

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -17,7 +17,6 @@ package io.hetu.core.plugin.datacenter;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Joiner;
 import io.prestosql.spi.connector.ConnectorTableHandle;
 import io.prestosql.spi.connector.SchemaTableName;
 
@@ -44,7 +43,7 @@ public final class DataCenterTableHandle
 
     private final OptionalLong limit;
 
-    private final String pushDownSql;
+    private final String subQuery;
 
     /**
      * Constructor of data center table handle.
@@ -56,11 +55,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this(catalogName,
-                requireNonNull(schemaName, "schemaName is null"),
-                requireNonNull(tableName, "tableName is null"),
-                requireNonNull(limit, "limit is null"),
-                "");
+        this.catalogName = catalogName;
+        this.schemaName = requireNonNull(schemaName, "schemaName is null");
+        this.tableName = requireNonNull(tableName, "tableName is null");
+        this.limit = requireNonNull(limit, "limit is null");
+        this.subQuery = "";
     }
 
     /**
@@ -70,25 +69,25 @@ public final class DataCenterTableHandle
      * @param schemaName schema name.
      * @param tableName table name.
      * @param limit the limit number of this query need.
-     * @param pushDownSql the sub query statement that want to be pushed down to remote data center.
+     * @param subQuery the sub query statement that want to be pushed down to remote data center.
      */
     @JsonCreator
     public DataCenterTableHandle(@JsonProperty("catalogName") String catalogName,
             @JsonProperty("schemaName") String schemaName, @JsonProperty("tableName") String tableName,
-            @JsonProperty("limit") OptionalLong limit, @JsonProperty("subQuery") String pushDownSql)
+            @JsonProperty("limit") OptionalLong limit, @JsonProperty("subQuery") String subQuery)
     {
         this.catalogName = catalogName;
         this.schemaName = requireNonNull(schemaName, "schemaName is null");
         this.tableName = requireNonNull(tableName, "tableName is null");
         this.limit = requireNonNull(limit, "limit is null");
-        this.pushDownSql = pushDownSql;
+        this.subQuery = subQuery;
     }
 
     @Override
     public ConnectorTableHandle createFrom(ConnectorTableHandle connectorTableHandle)
     {
         DataCenterTableHandle dataCenterTableHandle = (DataCenterTableHandle) connectorTableHandle;
-        return new DataCenterTableHandle(catalogName, schemaName, dataCenterTableHandle.tableName, dataCenterTableHandle.getLimit(), dataCenterTableHandle.getPushDownSql());
+        return new DataCenterTableHandle(catalogName, schemaName, dataCenterTableHandle.tableName, dataCenterTableHandle.getLimit(), dataCenterTableHandle.getSubQuery());
     }
 
     @JsonProperty
@@ -125,16 +124,15 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
-    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;
     }
 
     @JsonProperty
-    public String getPushDownSql()
+    public String getSubQuery()
     {
-        return pushDownSql;
+        return subQuery;
     }
 
     @Override
@@ -161,14 +159,6 @@ public final class DataCenterTableHandle
     @Override
     public String toString()
     {
-        StringBuilder builder = new StringBuilder();
-        if (!pushDownSql.isEmpty()) {
-            Joiner.on(SPLIT_DOT).skipNulls().appendTo(builder, catalogName, "{" + pushDownSql + "}");
-        }
-        else {
-            Joiner.on(SPLIT_DOT).skipNulls().appendTo(builder, catalogName, schemaName, tableName);
-        }
-        limit.ifPresent(value -> builder.append(" limit=").append(value));
-        return builder.toString();
+        return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;
     }
 }

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -1,324 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.plugin.datacenter.optimization;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import io.airlift.log.Logger;
-import io.hetu.core.plugin.datacenter.DataCenterColumn;
-import io.hetu.core.plugin.datacenter.DataCenterColumnHandle;
-import io.hetu.core.plugin.datacenter.DataCenterConfig;
-import io.hetu.core.plugin.datacenter.DataCenterTableHandle;
-import io.hetu.core.plugin.datacenter.client.DataCenterClient;
-import io.hetu.core.plugin.datacenter.client.DataCenterStatementClientFactory;
-import io.prestosql.expressions.LogicalRowExpressions;
-import io.prestosql.plugin.jdbc.optimization.JdbcConverterContext;
-import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorContext;
-import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorResult;
-import io.prestosql.spi.ConnectorPlanOptimizer;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.SymbolAllocator;
-import io.prestosql.spi.connector.CatalogName;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.ConnectorSession;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.metadata.TableHandle;
-import io.prestosql.spi.operator.ReuseExchangeOperator;
-import io.prestosql.spi.plan.Assignments;
-import io.prestosql.spi.plan.FilterNode;
-import io.prestosql.spi.plan.GroupIdNode;
-import io.prestosql.spi.plan.MarkDistinctNode;
-import io.prestosql.spi.plan.PlanNode;
-import io.prestosql.spi.plan.PlanNodeIdAllocator;
-import io.prestosql.spi.plan.PlanVisitor;
-import io.prestosql.spi.plan.ProjectNode;
-import io.prestosql.spi.plan.Symbol;
-import io.prestosql.spi.plan.TableScanNode;
-import io.prestosql.spi.predicate.TupleDomain;
-import io.prestosql.spi.relation.CallExpression;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpression;
-import io.prestosql.spi.relation.VariableReferenceExpression;
-import io.prestosql.spi.type.Type;
-import io.prestosql.spi.type.TypeManager;
-import io.prestosql.spi.type.UnknownType;
-import okhttp3.OkHttpClient;
-
-import javax.inject.Inject;
-
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalLong;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.IntStream;
-
-import static com.google.common.base.Preconditions.checkState;
-import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.getGroupingSetColumn;
-import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.replaceGroupingSetColumns;
-import static io.prestosql.spi.function.OperatorType.CAST;
-
-public class DataCenterPlanOptimizer
-        implements ConnectorPlanOptimizer
-{
-    private static final Logger log = Logger.get(DataCenterPlanOptimizer.class);
-
-    private static final String DATACENTER_CATALOG_PREFIX = "dc.";
-    private static final Set<Class<? extends PlanNode>> UNSUPPORTED_ROOT_NODE = ImmutableSet.of(GroupIdNode.class, MarkDistinctNode.class);
-
-    private final DataCenterClient client;
-    private final DataCenterConfig config;
-    private final TypeManager typeManager;
-    private final DataCenterQueryGenerator queryGenerator;
-    private final StandardFunctionResolution functionResolution;
-    private final LogicalRowExpressions logicalRowExpressions;
-
-    @Inject
-    public DataCenterPlanOptimizer(
-            TypeManager typeManager,
-            DataCenterConfig config,
-            DataCenterQueryGenerator query,
-            DeterminismEvaluator determinismEvaluator,
-            FunctionMetadataManager functionManager,
-            StandardFunctionResolution functionResolution)
-    {
-        OkHttpClient httpClient = DataCenterStatementClientFactory.newHttpClient(config);
-        this.client = new DataCenterClient(config, httpClient, typeManager);
-        this.config = config;
-        this.typeManager = typeManager;
-        this.queryGenerator = query;
-        this.functionResolution = functionResolution;
-        this.logicalRowExpressions = new LogicalRowExpressions(
-                determinismEvaluator,
-                functionResolution,
-                functionManager);
-    }
-
-    @Override
-    public PlanNode optimize(PlanNode maxSubPlan, ConnectorSession session, Map<String, Type> types, SymbolAllocator symbolAllocator, PlanNodeIdAllocator idAllocator)
-    {
-        if (!config.isQueryPushDownEnabled()) {
-            return maxSubPlan;
-        }
-        // Some node cannot be push down root node.
-        if (UNSUPPORTED_ROOT_NODE.contains(maxSubPlan.getClass())) {
-            return maxSubPlan;
-        }
-        return maxSubPlan.accept(new Visitor(idAllocator, types, session, symbolAllocator), null);
-    }
-
-    private static PlanNode replaceChildren(PlanNode node, List<PlanNode> children)
-    {
-        for (int i = 0; i < node.getSources().size(); i++) {
-            if (children.get(i) != node.getSources().get(i)) {
-                return node.replaceChildren(children);
-            }
-        }
-        return node;
-    }
-
-    private class Visitor
-            extends PlanVisitor<PlanNode, Void>
-    {
-        private final PlanNodeIdAllocator idAllocator;
-        private final ConnectorSession session;
-        private final Map<String, Type> types;
-        private final SymbolAllocator symbolAllocator;
-        private final IdentityHashMap<FilterNode, Void> filtersSplitUp = new IdentityHashMap<>();
-
-        public Visitor(
-                PlanNodeIdAllocator idAllocator,
-                Map<String, Type> types,
-                ConnectorSession session,
-                SymbolAllocator symbolAllocator)
-        {
-            this.idAllocator = idAllocator;
-            this.types = types;
-            this.session = session;
-            this.symbolAllocator = symbolAllocator;
-        }
-
-        @Override
-        public PlanNode visitPlan(PlanNode node, Void context)
-        {
-            Optional<PlanNode> pushDownPlan = tryCreatingNewScanNode(node);
-            return pushDownPlan.orElseGet(() -> replaceChildren(
-                    node, node.getSources().stream().map(source -> source.accept(this, null)).collect(toImmutableList())));
-        }
-
-        @Override
-        public PlanNode visitFilter(FilterNode node, Void context)
-        {
-            if (filtersSplitUp.containsKey(node)) {
-                return this.visitPlan(node, context);
-            }
-            filtersSplitUp.put(node, null);
-            FilterNode nodeToRecurseInto = node;
-            List<RowExpression> pushable = new ArrayList<>();
-            List<RowExpression> nonPushable = new ArrayList<>();
-
-            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
-                try {
-                    conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
-                    pushable.add(conjunct);
-                }
-                catch (PrestoException pe) {
-                    nonPushable.add(conjunct);
-                }
-            }
-            if (!pushable.isEmpty()) {
-                FilterNode pushableFilter = new FilterNode(idAllocator.getNextId(), node.getSource(), logicalRowExpressions.combineConjuncts(pushable));
-                Optional<FilterNode> nonPushableFilter = nonPushable.isEmpty() ? Optional.empty() : Optional.of(new FilterNode(idAllocator.getNextId(), pushableFilter, logicalRowExpressions.combineConjuncts(nonPushable)));
-
-                filtersSplitUp.put(pushableFilter, null);
-                if (nonPushableFilter.isPresent()) {
-                    FilterNode nonPushableFilterNode = nonPushableFilter.get();
-                    filtersSplitUp.put(nonPushableFilterNode, null);
-                    nodeToRecurseInto = nonPushableFilterNode;
-                }
-                else {
-                    nodeToRecurseInto = pushableFilter;
-                }
-            }
-            return this.visitFilter(nodeToRecurseInto, context);
-        }
-
-        private Optional<PlanNode> tryCreatingNewScanNode(PlanNode node)
-        {
-            Optional<JdbcQueryGeneratorResult> result = queryGenerator.generate(node, typeManager);
-            if (!result.isPresent()) {
-                return Optional.empty();
-            }
-
-            JdbcQueryGeneratorContext context = result.get().getContext();
-            JdbcQueryGeneratorResult.GeneratedSql generatedSql = result.get().getGeneratedSql();
-            if (!generatedSql.isPushDown()) {
-                return Optional.empty();
-            }
-
-            JdbcQueryGeneratorContext.GroupIdNodeInfo groupIdNodeInfo = context.getGroupIdNodeInfo();
-            String sql = generatedSql.getSql();
-            // replace grouping sets column
-            if (groupIdNodeInfo.isGroupByComplexOperation()) {
-                sql = replaceGroupingSetColumns(sql);
-            }
-
-            if (sql.getBytes(StandardCharsets.ISO_8859_1).length >= config.getRemoteHttpServerMaxRequestHeaderSize().toBytes()) {
-                log.debug("Generated sql is too long, push down failed.");
-                return Optional.empty();
-            }
-            List<DataCenterColumn> columnsList;
-            try {
-                columnsList = client.getColumns(sql);
-            }
-            catch (PrestoException e) {
-                log.warn("query push down failed for [%s]", e.getMessage());
-                return Optional.empty();
-            }
-            if (columnsList.isEmpty()) {
-                log.debug("Get columns from generated sql failed.");
-                return Optional.empty();
-            }
-
-            Map<String, ColumnHandle> columns = new HashMap<>();
-            IntStream.range(0, columnsList.size()).forEach(i -> {
-                DataCenterColumn column = columnsList.get(i);
-                columns.put(column.getName(), new DataCenterColumnHandle(column.getName(), column.getType(), i));
-            });
-            ImmutableList.Builder<Symbol> scanOutputs = new ImmutableList.Builder<>();
-            ImmutableMap.Builder<Symbol, ColumnHandle> columnHandles = new ImmutableMap.Builder<>();
-            ImmutableMap.Builder<Symbol, RowExpression> assignments = new ImmutableMap.Builder<>();
-
-            for (Symbol symbol : node.getOutputSymbols()) {
-                String name = symbol.getName().toLowerCase(Locale.ENGLISH);
-                String aliasName = groupIdNodeInfo.isGroupByComplexOperation()
-                        ? getGroupingSetColumn(name)
-                        : name;
-                if (!types.containsKey(name) || !columns.containsKey(aliasName)) {
-                    log.debug("Get type of column [%s] failed", name);
-                    return Optional.empty();
-                }
-                Type prestoType = types.get(name);
-                Type dcType = ((DataCenterColumnHandle) columns.get(aliasName)).getColumnType();
-
-                if (prestoType.equals(dcType)) {
-                    scanOutputs.add(symbol);
-                    columnHandles.put(symbol, columns.get(aliasName));
-                    assignments.put(symbol, new VariableReferenceExpression(symbol.getName(), prestoType));
-                }
-                else {
-                    if (prestoType instanceof UnknownType) {
-                        log.debug("Can't cast from type[%s] to type[%s]", dcType.getDisplayName(), prestoType.getDisplayName());
-                        return Optional.empty();
-                    }
-                    // If Jdbc return a different type from Presto's expected type, add a CAST expression
-                    Symbol scanSymbol = symbolAllocator.newSymbol(symbol.getName(), dcType);
-                    scanOutputs.add(scanSymbol);
-                    columnHandles.put(scanSymbol, columns.get(aliasName));
-                    assignments.put(symbol, new CallExpression(
-                            CAST.name(),
-                            functionResolution.castFunction(prestoType.getTypeSignature(), dcType.getTypeSignature()),
-                            prestoType,
-                            ImmutableList.of(new VariableReferenceExpression(scanSymbol.getName(), dcType)),
-                            Optional.empty()));
-                }
-            }
-
-            checkState(context.getCatalogName().isPresent(), "CatalogName is null");
-            checkState(context.getSchemaTableName().isPresent(), "schemaTableName is null");
-            checkState(context.getTransaction().isPresent(), "transaction is null");
-            CatalogName catalogName = context.getCatalogName().get();
-            String tableCatalogName = catalogName.getCatalogName().startsWith(DATACENTER_CATALOG_PREFIX)
-                    ? catalogName.getCatalogName().substring(DATACENTER_CATALOG_PREFIX.length())
-                    : catalogName.getCatalogName();
-
-            TableHandle newTableHandle = new TableHandle(
-                    catalogName,
-                    new DataCenterTableHandle(
-                            tableCatalogName,
-                            context.getSchemaTableName().get().getSchemaName(),
-                            context.getSchemaTableName().get().getTableName(),
-                            OptionalLong.empty(),
-                            sql),
-                    context.getTransaction().get(),
-                    Optional.empty());
-            return Optional.of(
-                    new ProjectNode(
-                            this.idAllocator.getNextId(),
-                            new TableScanNode(
-                                    idAllocator.getNextId(),
-                                    newTableHandle,
-                                    scanOutputs.build(),
-                                    columnHandles.build(),
-                                    TupleDomain.all(),
-                                    Optional.empty(),
-                                    ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT,
-                                    new UUID(0, 0),
-                                    0,
-                                    false),
-                            new Assignments(assignments.build())));
-        }
-    }
-}

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterQueryGenerator.java

@@ -1,142 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.plugin.datacenter.optimization;
-
-import io.hetu.core.plugin.datacenter.DataCenterColumnHandle;
-import io.hetu.core.plugin.datacenter.DataCenterConfig;
-import io.hetu.core.plugin.datacenter.DataCenterTableHandle;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcQueryGenerator;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcRowExpressionConverter;
-import io.prestosql.plugin.jdbc.optimization.BaseJdbcSqlStatementWriter;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownParameter;
-import io.prestosql.plugin.jdbc.optimization.JdbcQueryGeneratorContext;
-import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.connector.ColumnHandle;
-import io.prestosql.spi.connector.SchemaTableName;
-import io.prestosql.spi.function.FunctionMetadataManager;
-import io.prestosql.spi.function.StandardFunctionResolution;
-import io.prestosql.spi.metadata.TableHandle;
-import io.prestosql.spi.plan.PlanNode;
-import io.prestosql.spi.plan.PlanVisitor;
-import io.prestosql.spi.plan.TableScanNode;
-import io.prestosql.spi.predicate.TupleDomain;
-import io.prestosql.spi.relation.DeterminismEvaluator;
-import io.prestosql.spi.relation.RowExpressionService;
-import io.prestosql.spi.sql.expression.Selection;
-import io.prestosql.spi.type.TypeManager;
-
-import javax.inject.Inject;
-
-import java.util.LinkedHashMap;
-import java.util.Optional;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_QUERY_GENERATOR_FAILURE;
-import static io.prestosql.plugin.jdbc.optimization.JdbcPlanOptimizerUtils.quote;
-
-public class DataCenterQueryGenerator
-        extends BaseJdbcQueryGenerator
-{
-    @Inject
-    public DataCenterQueryGenerator(DataCenterConfig config, RowExpressionService rowExpressionService, FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution, DeterminismEvaluator determinismEvaluator)
-    {
-        super(new JdbcPushDownParameter("\"", false, config.getQueryPushDownModule(), functionResolution),
-                new BaseJdbcRowExpressionConverter(functionManager, functionResolution, rowExpressionService, determinismEvaluator),
-                new BaseJdbcSqlStatementWriter(new JdbcPushDownParameter("\"", false, config.getQueryPushDownModule(), functionResolution)));
-    }
-
-    @Override
-    protected PlanVisitor<Optional<JdbcQueryGeneratorContext>, Void> getVisitor(TypeManager typeManager)
-    {
-        return new DataCenterPlanVisitor(typeManager);
-    }
-
-    protected class DataCenterPlanVisitor
-            extends BaseJdbcPlanVisitor
-    {
-        public DataCenterPlanVisitor(TypeManager typeManager)
-        {
-            super(typeManager);
-        }
-
-        @Override
-        public Optional<JdbcQueryGeneratorContext> visitPlan(PlanNode node, Void contextIn)
-        {
-            log.debug(GENERATE_FAILED_LOG, "Don't know how to handle plan node of type " + node);
-            return Optional.empty();
-        }
-
-        @Override
-        public Optional<JdbcQueryGeneratorContext> visitTableScan(TableScanNode node, Void contextIn)
-        {
-            checkAvailable(node);
-            checkArgument(node.getTable().getConnectorHandle() instanceof DataCenterTableHandle,
-                    "Expected to find Data Center table handle for the scan node");
-            TupleDomain<ColumnHandle> constraint = node.getEnforcedConstraint();
-            if (constraint != null && constraint.getDomains().isPresent()) {
-                if (!constraint.getDomains().get().isEmpty()) {
-                    // Predicate is pushed down
-                    throw new PrestoException(JDBC_QUERY_GENERATOR_FAILURE, "Cannot push down table scan with predicates pushed down");
-                }
-            }
-            TableHandle tableHandle = node.getTable();
-            DataCenterTableHandle dcTableHandle = (DataCenterTableHandle) node.getTable().getConnectorHandle();
-            checkArgument(dcTableHandle.getPushDownSql().isEmpty(), "Data center should not have sql before pushdown");
-            LinkedHashMap<String, Selection> selections = new LinkedHashMap<>();
-            node.getOutputSymbols().forEach(outputColumn -> {
-                DataCenterColumnHandle dcColumn = (DataCenterColumnHandle) node.getAssignments().get(outputColumn);
-                selections.put(outputColumn.getName(), new Selection(dcColumn.getColumnName(), outputColumn.getName()));
-            });
-            StringBuilder table = new StringBuilder();
-            if (!isNullOrEmpty(dcTableHandle.getCatalogName())) {
-                table.append(quote(quote, dcTableHandle.getCatalogName())).append('.');
-            }
-            if (!isNullOrEmpty(dcTableHandle.getSchemaName())) {
-                table.append(quote(quote, dcTableHandle.getSchemaName())).append('.');
-            }
-            table.append(quote(quote, dcTableHandle.getTableName()));
-
-            JdbcQueryGeneratorContext.Builder contextBuilder = new JdbcQueryGeneratorContext.Builder()
-                    .setCatalogName(Optional.of(tableHandle.getCatalogName()))
-                    .setTransaction(Optional.of(tableHandle.getTransaction()))
-                    .setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
-                    .setSelections(selections)
-                    .setFrom(Optional.of(table.toString()));
-
-            String catalogName = dcTableHandle.getCatalogName();
-            if (catalogName != null) {
-                contextBuilder.setRemoteCatalogName(catalogName);
-            }
-            String schemaName = dcTableHandle.getSchemaName();
-            if (schemaName != null) {
-                contextBuilder.setRemoteSchemaName(schemaName);
-            }
-            String tableName = dcTableHandle.getTableName();
-            if (tableName != null) {
-                contextBuilder.setRemoteTablename(tableName);
-            }
-
-            // If LIMIT has been push down, add it to context
-            if (dcTableHandle.getLimit().isPresent()) {
-                contextBuilder.setLimit(dcTableHandle.getLimit());
-                contextBuilder.setHasPushDown(true);
-            }
-
-            return Optional.of(contextBuilder.build());
-        }
-    }
-}

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/pagesource/DataCenterPageSource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -23,7 +23,6 @@ import io.prestosql.spi.Page;
 import io.prestosql.spi.connector.ColumnHandle;
 import io.prestosql.spi.connector.ConnectorPageSource;
 import io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter;
-import io.prestosql.spi.dynamicfilter.CombinedDynamicFilter;
 import io.prestosql.spi.dynamicfilter.DynamicFilter;
 import io.prestosql.spi.dynamicfilter.DynamicFilterSupplier;
 import io.prestosql.spi.dynamicfilter.HashSetDynamicFilter;
@@ -102,10 +101,8 @@ public class DataCenterPageSource
     @Override
     public Page getNextPage()
     {
-        if (dynamicFilterSupplier.isPresent() && !dynamicFilterSupplier.get().getDynamicFilters().isEmpty()) {
-            /*  applying only for the first map in the dynamic filter since we do not have
-                more than one element as we do not expect disjuncts in this connector   */
-            applyDynamicFilters(dynamicFilterSupplier.get().getDynamicFilters().get(0));
+        if (dynamicFilterSupplier.isPresent()) {
+            applyDynamicFilters(dynamicFilterSupplier.get().getDynamicFilters());
         }
 
         if (!this.pages.isEmpty()) {
@@ -134,17 +131,8 @@ public class DataCenterPageSource
                     BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromHashSetDynamicFilter((HashSetDynamicFilter) df);
                     builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
                 }
-                else if (df instanceof CombinedDynamicFilter) {
-                    BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromCombinedDynamicFilter((CombinedDynamicFilter) df);
-                    if (bloomFilterDynamicFilter != null) {
-                        builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
-                    }
-                }
-                else if (df instanceof BloomFilterDynamicFilter) {
-                    builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
-                }
                 else {
-                    LOGGER.info("Dynamic Filter (type: " + df.getClass().getSimpleName() + ") skipped for DC connector");
+                    builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
                 }
             }
         }

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/pagesource/DataCenterPageSourceProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -50,13 +50,14 @@ public class DataCenterPageSourceProvider
 {
     private static final int STRING_CAPACITY = 16;
     private static final String ENABLE_CROSS_REGION_DYNAMIC_FILTER = "cross_region_dynamic_filter_enabled";
+    private static final String OPTIMIZE_DYNAMIC_FILTER_GENERATION = "optimize_dynamic_filter_generation";
     private static final String EXCHANGE_COMPRESSION = "exchange_compression";
 
     private final DataCenterConfig config;
 
     private final OkHttpClient httpClient;
 
-    private final TypeManager typeManager;
+    private TypeManager typeManager;
 
     /**
      * Constructor of data center page source provider.
@@ -90,7 +91,7 @@ public class DataCenterPageSourceProvider
 
         sql.append(" FROM ");
 
-        if (tableHandler.getPushDownSql() == null || "".equals(tableHandler.getPushDownSql())) {
+        if (tableHandler.getSubQuery() == null || "".equals(tableHandler.getSubQuery())) {
             if (!isNullOrEmpty(catalog)) {
                 sql.append(catalog).append('.');
             }
@@ -101,7 +102,7 @@ public class DataCenterPageSourceProvider
             sql.append(table);
         }
         else {
-            sql.append("(").append(tableHandler.getPushDownSql()).append(") pushdown");
+            sql.append(tableHandler.getSubQuery());
         }
 
         if (limit.isPresent()) {
@@ -128,8 +129,9 @@ public class DataCenterPageSourceProvider
         Map<String, String> properties = new HashMap<>();
 
         // Only set the session if there is any dynamic filter for this page source
-        if (dynamicFilterSupplier != null && dynamicFilterSupplier.isPresent()) {
+        if (dynamicFilterSupplier != null) {
             properties.put(ENABLE_CROSS_REGION_DYNAMIC_FILTER, "true");
+            properties.put(OPTIMIZE_DYNAMIC_FILTER_GENERATION, "false"); // close removeUnsupportedDynamicFilter optimizer
         }
         if (config.isCompressionEnabled()) {
             properties.put(EXCHANGE_COMPRESSION, "true");

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -365,6 +365,7 @@ public class TestCrossRegionDynamicFilter
         assertQuery("SELECT COUNT(*) FROM dc.tpch.tiny.lineitem JOIN orders ON dc.tpch.tiny.lineitem.orderkey = orders.orderkey AND NOT (orders.comment LIKE '%forges%')");
         assertQuery("SELECT COUNT(*) FROM dc.tpch.tiny.lineitem JOIN orders ON dc.tpch.tiny.lineitem.orderkey = orders.orderkey AND NOT (orders.comment LIKE dc.tpch.tiny.lineitem.comment)");
         assertQuery("SELECT COUNT(*) FROM dc.tpch.tiny.lineitem JOIN orders ON dc.tpch.tiny.lineitem.orderkey = orders.orderkey AND dc.tpch.tiny.lineitem.quantity + length(orders.comment) > 7");
+        assertQuery("SELECT COUNT(*) FROM dc.tpch.tiny.lineitem JOIN orders ON dc.tpch.tiny.lineitem.orderkey = orders.orderkey AND NULL");
     }
 
     @Test
@@ -1392,24 +1393,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner distributedQueryRunner = null;
+        DistributedQueryRunner queryRunner = null;
         try {
-            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            distributedQueryRunner.installPlugin(new DataCenterPlugin());
-            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            distributedQueryRunner.installPlugin(new TpchPlugin());
-            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
+            queryRunner.installPlugin(new DataCenterPlugin());
+            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            queryRunner.installPlugin(new TpchPlugin());
+            queryRunner.createCatalog("tpch", "tpch", properties);
 
-            return distributedQueryRunner;
+            return queryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, distributedQueryRunner);
+            closeAllSuppress(e, queryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -19,7 +19,6 @@ import com.google.common.collect.ImmutableMap;
 import io.airlift.configuration.testing.ConfigAssertions;
 import io.airlift.units.DataSize;
 import io.airlift.units.Duration;
-import io.prestosql.plugin.jdbc.optimization.JdbcPushDownModule;
 import org.testng.annotations.Test;
 
 import java.net.URI;
@@ -58,7 +57,6 @@ public class TestDataCenterConfig
                 .setKerberosUseCanonicalHostname(false)
                 .setExtraCredentials(null)
                 .setQueryPushDownEnabled(true)
-                .setQueryPushDownModule(JdbcPushDownModule.DEFAULT)
                 .setHttpRequestReadTimeout(READ_TIMEOUT)
                 .setHttpRequestConnectTimeout(CONNECT_TIMEOUT)
                 .setClientTimeout(new Duration(10, TimeUnit.MINUTES))
@@ -98,7 +96,6 @@ public class TestDataCenterConfig
                 .put("dc.ssl.truststore.password", "ssl.truststore.password")
                 .put("dc.ssl.truststore.path", "ssl.truststore.path")
                 .put("dc.query.pushdown.enabled", "false")
-                .put("dc.query.pushdown.module", "FULL_PUSHDOWN")
                 .put("dc.http-request-readTimeout", "5m")
                 .put("dc.http-request-connectTimeout", "5m")
                 .put("dc.http-client-timeout", "5m")
@@ -134,7 +131,6 @@ public class TestDataCenterConfig
                 .setKerberosUseCanonicalHostname(true)
                 .setExtraCredentials("extra.credentials")
                 .setQueryPushDownEnabled(false)
-                .setQueryPushDownModule(JdbcPushDownModule.FULL_PUSHDOWN)
                 .setHttpRequestReadTimeout(new Duration(5, TimeUnit.MINUTES))
                 .setHttpRequestConnectTimeout(new Duration(5, TimeUnit.MINUTES))
                 .setClientTimeout(new Duration(5, TimeUnit.MINUTES))

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterMetadata.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -104,6 +104,12 @@ public class TestDataCenterMetadata
             return 0;
         }
 
+        @Override
+        public boolean isLegacyTimestamp()
+        {
+            return false;
+        }
+
         @Override
         public <T> T getProperty(String name, Class<T> type)
         {

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
+openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,17 +24,15 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-The following is a detailed description of audit logging properties:
+Other audit logging properties include: 
 
-`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
+`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
 
-`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
+`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
 
 Example configuration file:
 
@@ -43,6 +41,7 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
+hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
+hetu.event.listener.audit.filecount=1
+hetu.event.listener.audit.limit=100000
 ```

hetu-docs/en/admin/cluster-node-management.md

@@ -0,0 +1,71 @@
+
+
+# Cluster Node Management
+
+There can potentially be multiple nodes (coordinators and workers) in a openLooKeng cluster. Sometimes admin operations need to be performed on these nodes, for scaling or maintenance purposes.
+
+## Shutdown a node
+
+If a cluster does not need a node anymore (e.g. as part of a scaling down operation), the node can be shutdown.
+
+Such process is _graceful_, in that
+- the cluster does not assign new workload to this node, and
+- the node attempts to finish all existing workloads before shutting down its main process.
+
+The shutdown process is _irreversible_. It can be initiated through the REST API, by putting the `SHUTTING_DOWN` state to the `info/state` endpoint of the node. For example, with `curl` and assuming the node to shutdown has address `1.2.3.4:8080`:
+
+```sh
+curl -X PUT -H "Content-Type: application/json" http://1.2.3.4:8080/v1/info/state \
+    -d '"SHUTTING_DOWN"'
+```
+
+## Isolate a node
+
+_Isolation_ takes a node off of the cluster _temporarily_, for example to perform maintenance tasks on it. The node can be added back to the cluster afterwards.
+
+- Isolation can be graceful in a way similar to shutdown: wait for workloads to finish first. Nodes in this waiting period are in the `isolating` state.
+- It can also be non-graceful, to enter the `isolated` directly.
+- `Isolating` and `isolated` nodes are not assigned new workloads.
+
+Nodes' isolation status are changed via the REST API, with different target states. For example:
+
+```sh
+# To gracefully isolate a node
+curl -X PUT -H "Content-Type: application/json" http://1.2.3.4:8080/v1/info/state \
+    -d '"ISOLATING"'
+
+# To isolate a node immediately
+curl -X PUT -H "Content-Type: application/json" http://1.2.3.4:8080/v1/info/state \
+    -d '"ISOLATED"'
+
+# To make the node available again
+curl -X PUT -H "Content-Type: application/json" http://1.2.3.4:8080/v1/info/state \
+    -d '"ACTIVE"'
+```
+
+## Node state transitions
+
+A openLooKeng node (coordinator or worker) can be in one of 5 states:
+- Inactive
+- Active
+- Isolating
+- Isolated
+- Shutting down
+
+Shutdown and isolation operations move nodes among these states. The following table shows which transitions are allowed, where "-" indicates that the transition is allowed but has no effect.
+
+|from\to|inactive|active|isolating|isolated|shutting_down|exit|
+|-|-|-|-|-|-|-|
+|inactive| | | | | 5 | |
+|active| | - | 2 | 3 | 5 | |
+|isolating| | 1 | - | 3, 4 | 5 | |
+|isolated| | 1 | 2 | - | 5 | |
+|shutting_down| | | | | - | 6 |
+
+Transition details:
+1. Restore the node back to normal operation
+1. Request to isolate the node, by not assigning new workload on it, and waiting for existing workloads to finish
+1. Immediately isolate the node, by not assigning new workload. If there are existing workloads, they are deemed unimportant and may not finish
+1. Automatic transition when a node is `ISOLATING` and active workloads finish
+1. Request to shut down the node, by not assigning new workload on it, and waiting for existing workloads to finish
+1. Automatic transition when a node is `SHUTTING_DOWN` and active workloads finish