!880 Add ClickHouse connector on version 1.2

Merge pull request !880 from Heatao/branch-1.2

.gitignore

@@ -1,7 +1,6 @@
 *.iml
 *.ipr
 *.iws
-*.icloud
 target/
 /var
 /*/var/
@@ -28,4 +27,4 @@ node_modules
 */*.iml
 */.idea
 */.gitignore
-*/dependency-reduced-pom.xml
+*/dependency-reduced-pom.xml

OWNERS

@@ -7,4 +7,3 @@ approvers:
 - farhan3
 - fbird2020
 - lizheng920625
-- giteezhangjingfang

docker/container-test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (C) 2020-2021. Huawei Technologies Co., Ltd. All rights reserved.
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

docker/default/etc/catalog/memory.properties

@@ -0,0 +1,14 @@
+# Copyright (C) 2020. Huawei Technologies Co., Ltd. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+connector.name=memory

hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.7.0-SNAPSHOT</version>
+    <version>1.2.1-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>
@@ -361,17 +361,13 @@
           <artifactId>bootstrap</artifactId>
           <groupId>io.airlift</groupId>
         </exclusion>
-        <exclusion>
-          <artifactId>jetty-util</artifactId>
-          <groupId>org.eclipse.jetty</groupId>
-        </exclusion>
       </exclusions>
     </dependency>
 
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-compress</artifactId>
-      <version>1.21</version>
+      <version>1.19</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,7 +108,8 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this(metastore, null);
+            this.metastore = metastore;
+            this.schemaName = null;
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -232,6 +233,7 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
+        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,17 +73,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putShorts(inputRowId, count, value);
+            columnVector.putShorts(rowId, count, value);
         }
     }
 
@@ -98,17 +97,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putInts(inputRowId, count, value);
+            columnVector.putInts(rowId, count, value);
         }
     }
 
@@ -123,17 +121,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putLongs(inputRowId, count, value);
+            columnVector.putLongs(rowId, count, value);
         }
     }
 
@@ -148,12 +145,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -168,17 +164,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putDoubles(inputRowId, count, value);
+            columnVector.putDoubles(rowId, count, value);
         }
     }
 
@@ -201,12 +196,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -229,17 +223,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putNull(counter++);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putNulls(inputRowId, count);
+            columnVector.putNulls(rowId, count);
         }
     }
 
@@ -326,72 +319,66 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -18,7 +18,7 @@ import com.google.gson.Gson;
 import io.prestosql.plugin.hive.HiveACIDWriteType;
 import io.prestosql.plugin.hive.HiveFileWriter;
 import io.prestosql.plugin.hive.HiveType;
-import io.prestosql.plugin.hive.util.FieldSetterFactory;
+import io.prestosql.plugin.hive.HiveWriteUtils;
 import io.prestosql.spi.Page;
 import io.prestosql.spi.PrestoException;
 import io.prestosql.spi.block.Block;
@@ -64,7 +64,6 @@ import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.log4j.Logger;
-import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -105,7 +104,7 @@ public class CarbondataFileWriter
     private final Object row;
     private final SettableStructObjectInspector tableInspector;
     private final List<StructField> structFields;
-    private final FieldSetterFactory.FieldSetter[] setters;
+    private final HiveWriteUtils.FieldSetter[] setters;
     private final Properties properties;
     private final Optional<AcidOutputFormat.Options> acidOptions;
     private final HiveACIDWriteType acidWriteType;
@@ -127,16 +126,15 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        Path localOutPutPath = paramOutPutPath;
-        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
+        this.outPutPath = requireNonNull(outPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            localOutPutPath = new Path(properties.getProperty("location"));
+            outPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -185,12 +183,9 @@ public class CarbondataFileWriter
 
         row = tableInspector.create();
 
-        setters = new FieldSetterFactory.FieldSetter[structFields.size()];
-
-        FieldSetterFactory fieldSetterFactory = new FieldSetterFactory(DateTimeZone.UTC);
-
+        setters = new HiveWriteUtils.FieldSetter[structFields.size()];
         for (int i = 0; i < setters.length; i++) {
-            setters[i] = fieldSetterFactory.create(tableInspector, row, structFields.get(i),
+            setters[i] = HiveWriteUtils.createFieldSetter(tableInspector, row, structFields.get(i),
                     fileColumnTypes.get(structFields.get(i).getFieldID()));
         }
 
@@ -212,7 +207,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -227,25 +222,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path finalOutPutPath = this.outPutPath;
-        Properties finalProperties = this.properties;
-        JobConf finalConfiguration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
+        Path outPutPath = this.outPutPath;
+        Properties properties = this.properties;
+        JobConf configuration = this.configuration;
+        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
-                        finalProperties, Reporter.NULL);
+                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
+                        properties, Reporter.NULL);
     }
 
     @Override
@@ -286,7 +281,7 @@ public class CarbondataFileWriter
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter finalRecordWriter = null;
+        FileSinkOperator.RecordWriter recordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -335,7 +330,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -352,7 +347,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            finalRecordWriter = this.recordWriter;
+            recordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -366,8 +361,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (finalRecordWriter != null) {
-                finalRecordWriter.write(serDe.serialize(row, tableInspector));
+            if (recordWriter != null) {
+                recordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,7 +48,6 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
-    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuFilterUtil.java

@@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
             if (rawData instanceof Slice) {
                 String value = ((Slice) rawData).toStringUtf8();
                 if (type.getTypeInfo() instanceof CharTypeInfo) {
-                    StringBuilder padding = new StringBuilder();
+                    String padding = "";
                     int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
                     int truncatedLength = value.length();
                     for (int i = 0; i < paddedLength - truncatedLength; i++) {
-                        padding.append(" ");
+                        padding += " ";
                     }
                     return value + padding;
                 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuOutputFormat.java

@@ -103,6 +103,7 @@ public class CarbondataHetuOutputFormat<T>
         OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
         JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
         carbonOutputCommitter.setupJob(jobContext);
+        CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
         org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
         return new FileSinkOperator.RecordWriter()
         {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataLocationService.java

@@ -80,6 +80,8 @@ public class CarbondataLocationService
     {
         // TODO: check and make it compatible for cloud scenario
 
+        HdfsEnvironment.HdfsContext context =
+                new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
         Path targetPath = new Path(table.getStorage().getLocation());
 
         return new LocationHandle(targetPath, targetPath, true,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java

@@ -92,28 +92,15 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.features.TableOperation;
-import org.apache.carbondata.core.fileoperations.FileWriteOperation;
 import org.apache.carbondata.core.index.Segment;
 import org.apache.carbondata.core.locks.CarbonLockFactory;
 import org.apache.carbondata.core.locks.CarbonLockUtil;
 import org.apache.carbondata.core.locks.ICarbonLock;
 import org.apache.carbondata.core.locks.LockUsage;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
 import org.apache.carbondata.core.metadata.SegmentFileStore;
-import org.apache.carbondata.core.metadata.converter.SchemaConverter;
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.StructField;
-import org.apache.carbondata.core.metadata.schema.PartitionInfo;
-import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.TableSchema;
-import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
 import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
 import org.apache.carbondata.core.mutate.data.BlockMappingVO;
@@ -126,7 +113,6 @@ import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.ObjectSerializationUtil;
 import org.apache.carbondata.core.util.ThreadLocalSessionInfo;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.core.writer.ThriftWriter;
 import org.apache.carbondata.hadoop.api.CarbonOutputCommitter;
 import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
 import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
@@ -156,21 +142,19 @@ import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
 import org.apache.log4j.Logger;
+import org.joda.time.DateTimeZone;
 
 import java.io.File;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.NoSuchElementException;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
@@ -237,10 +221,6 @@ public class CarbondataMetadata
     private List<SegmentUpdateDetails> blockUpdateDetailsList;
     private State currentState = State.OTHER;
     private CarbonLoadModel carbonLoadModel;
-    private AbsoluteTableIdentifier absoluteTableIdentifier;
-    private TableInfo tableInfo;
-    private SchemaTableName schemaTableName;
-
     private String user;
     private Optional<String> tableStorageLocation;
     private String carbondataTableStore;
@@ -265,14 +245,11 @@ public class CarbondataMetadata
         CREATE_TABLE_AS,
         DROP_TABLE,
         OTHER,
-        ADD_COLUMN,
-        DROP_COLUMN,
-        RENAME_COLUMN
     }
 
     public CarbondataMetadata(SemiTransactionalHiveMetastore metastore,
-            HdfsEnvironment hdfsEnvironment, HivePartitionManager partitionManager,
-            boolean writesToNonManagedTablesEnabled,
+            HdfsEnvironment hdfsEnvironment, HivePartitionManager partitionManager, DateTimeZone timeZone,
+            boolean allowCorruptWritesForTesting, boolean writesToNonManagedTablesEnabled,
             boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
             TypeManager typeManager, LocationService locationService,
             JsonCodec<PartitionUpdate> partitionUpdateCodec,
@@ -282,7 +259,7 @@ public class CarbondataMetadata
             CarbondataTableReader carbondataTableReader, String carbondataTableStore, long carbondataMajorVacuumSegSize, long carbondataMinorVacuumSegCount,
             ScheduledExecutorService executorService, ScheduledExecutorService hiveMetastoreClientService)
     {
-        super(metastore, hdfsEnvironment, partitionManager,
+        super(metastore, hdfsEnvironment, partitionManager, timeZone, allowCorruptWritesForTesting,
                 writesToNonManagedTablesEnabled, createsOfNonManagedTablesEnabled, tableCreatesWithLocationAllowed,
                 typeManager, locationService, partitionUpdateCodec, typeTranslator, hetuVersion,
                 hiveStatisticsProvider, accessControlMetadata, false, 2, 0.0, executorService,
@@ -306,13 +283,13 @@ public class CarbondataMetadata
 
     private void setupCommitWriter(Properties hiveSchema, Path outputPath, Configuration initialConfiguration, boolean isOverwrite) throws PrestoException
     {
-        CarbonLoadModel finalCarbonLoadModel;
+        CarbonLoadModel carbonLoadModel;
         TaskAttemptID taskAttemptID = TaskAttemptID.forName(initialConfiguration.get("mapred.task.id"));
         try {
             ThreadLocalSessionInfo.setConfigurationToCurrentThread(initialConfiguration);
-            finalCarbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
-            finalCarbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
-            CarbonTableOutputFormat.setLoadModel(initialConfiguration, finalCarbonLoadModel);
+            carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
+            carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
+            CarbonTableOutputFormat.setLoadModel(initialConfiguration, carbonLoadModel);
         }
         catch (IOException ex) {
             LOG.error("Error while creating carbon load model", ex);
@@ -360,13 +337,13 @@ public class CarbondataMetadata
         this.user = session.getUser();
         return hdfsEnvironment.doAs(user, () -> {
             SchemaTableName tableName = parent.getSchemaTableName();
-            Optional<Table> finalTable =
+            Optional<Table> table =
                     metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-            if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+            if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
                 throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
             }
 
-            this.table = finalTable;
+            this.table = table;
             Path outputPath =
                     new Path(parent.getLocationHandle().getJsonSerializableTargetPath());
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -382,7 +359,7 @@ public class CarbondataMetadata
             }
 
             /* Create committer object */
-            setupCommitWriter(finalTable, outputPath, initialConfiguration, isOverwrite);
+            setupCommitWriter(table, outputPath, initialConfiguration, isOverwrite);
 
             return new CarbondataInsertTableHandle(parent.getSchemaName(),
                     parent.getTableName(),
@@ -411,18 +388,18 @@ public class CarbondataMetadata
     }
 
     @Override
-    public CarbondataUpdateTableHandle beginUpdateAsInsert(ConnectorSession session, ConnectorTableHandle tableHandle)
+    public CarbondataUpdateTableHandle beginUpdate(ConnectorSession session, ConnectorTableHandle tableHandle)
     {
         currentState = State.UPDATE;
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -430,8 +407,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -470,13 +447,13 @@ public class CarbondataMetadata
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         List<HiveColumnHandle> inputColumns = parent.getInputColumns().stream().filter(HiveColumnHandle::isRequired).collect(toList());
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -484,8 +461,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -539,10 +516,10 @@ public class CarbondataMetadata
     }
 
     @Override
-    public Optional<ConnectorOutputMetadata> finishUpdateAsInsert(ConnectorSession session,
-                                                                  ConnectorUpdateTableHandle updateHandle,
-                                                                  Collection<Slice> fragments,
-                                                                  Collection<ComputedStatistics> computedStatistics)
+    public Optional<ConnectorOutputMetadata> finishUpdate(ConnectorSession session,
+                                                          ConnectorUpdateTableHandle updateHandle,
+                                                          Collection<Slice> fragments,
+                                                          Collection<ComputedStatistics> computedStatistics)
     {
         HiveUpdateTableHandle updateTableHandle = (HiveUpdateTableHandle) updateHandle;
         HiveInsertTableHandle insertTableHandle = new HiveInsertTableHandle(
@@ -643,7 +620,7 @@ public class CarbondataMetadata
 
         return hdfsEnvironment.doAs(session.getUser(), () -> {
             Properties hiveSchema = MetastoreUtil.getHiveSchema(this.table.get());
-            CarbonTable finalCarbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
+            CarbonTable carbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
                     carbondataVacuumTableHandle.getTableName(),
                     hiveSchema,
                     initialConfiguration);
@@ -705,7 +682,7 @@ public class CarbondataMetadata
                     SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath()));
                     String source;
                     for (String currPartitionName : partitionNames) {
-                        source = finalCarbonTable.getTablePath() + "/" + currPartitionName;
+                        source = carbonTable.getTablePath() + "/" + currPartitionName;
                         moveFromTempFolder(source + "/" + carbonLoadModel.getSegmentId() + "_" + timeStamp + ".tmp", source);
                     }
                     segmentFilesToBeUpdatedLatest.add(new Segment(carbonLoadModel.getSegmentId(), segmentFileName));
@@ -719,7 +696,7 @@ public class CarbondataMetadata
                 for (CarbondataSegmentInfoUtil segmentInfo : newMergedSegmentInfoUtilList) {
                     String mergedLoadNumber = segmentInfo.getDestinationSegment();
                     try {
-                        String segmentFileName = SegmentFileStore.writeSegmentFile(finalCarbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
+                        String segmentFileName = SegmentFileStore.writeSegmentFile(carbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
                     }
                     catch (IOException e) {
                         throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while merging segment files", e);
@@ -759,7 +736,7 @@ public class CarbondataMetadata
     }
 
     @Override
-    public ColumnHandle getDeleteRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle)
+    public ColumnHandle getUpdateRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle)
     {
         // Hive connector only supports metadata delete. It does not support generic row-by-row deletion.
         // Metadata delete is implemented in Hetu by generating a plan for row-by-row delete first,
@@ -771,14 +748,6 @@ public class CarbondataMetadata
                 Optional.empty());
     }
 
-    @Override
-    public ColumnHandle getUpdateRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle, List<ColumnHandle> updatedColumns)
-    {
-        return new HiveColumnHandle(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID,
-                HIVE_STRING, HIVE_STRING.getTypeSignature(), -13, SYNTHESIZED,
-                Optional.empty());
-    }
-
     @Override
     protected Map<String, String> getEmptyTableProperties(ConnectorTableMetadata tableMetadata,
                                                           Optional<HiveBucketProperty> bucketProperty,
@@ -875,14 +844,6 @@ public class CarbondataMetadata
                 case DROP_TABLE: {
                     break;
                 }
-                case ADD_COLUMN:
-                case DROP_COLUMN:
-                case RENAME_COLUMN: {
-                    hdfsEnvironment.doAs(user, () -> {
-                        revertAlterTableChanges();
-                    });
-                    break;
-                }
                 default: {
                     return;
                 }
@@ -900,9 +861,9 @@ public class CarbondataMetadata
     private LocationHandle getCarbonDataTableCreationPath(ConnectorSession session, ConnectorTableMetadata tableMetadata, HiveWriteUtils.OpertionType opertionType) throws PrestoException
     {
         Path targetPath = null;
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String tableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         Optional<String> location = getCarbondataLocation(tableMetadata.getProperties());
         LocationHandle locationHandle;
         FileSystem fileSystem;
@@ -914,32 +875,32 @@ public class CarbondataMetadata
                     throw new PrestoException(NOT_SUPPORTED, format("Setting %s property is not allowed", LOCATION_PROPERTY));
                 }
                 /* if path not having prefix with filesystem type, than we will take fileSystem type from core-site.xml using below methods */
-                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, finalSchemaName), new Path(location.get()));
+                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, schemaName), new Path(location.get()));
                 targetLocation = fileSystem.getFileStatus(new Path(location.get())).getPath().toString();
-                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, finalSchemaName, tableName), targetLocation, false);
+                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, schemaName, tableName), targetLocation, false);
             }
             else {
-                updateEmptyCarbondataTableStorePath(session, finalSchemaName);
+                updateEmptyCarbondataTableStorePath(session, schemaName);
                 targetLocation = carbondataTableStore;
-                targetLocation = targetLocation + File.separator + finalSchemaName + File.separator + tableName;
+                targetLocation = targetLocation + File.separator + schemaName + File.separator + tableName;
                 targetPath = new Path(targetLocation);
             }
         }
         catch (IllegalArgumentException | IOException e) {
             throw new PrestoException(NOT_SUPPORTED, format("Error %s store path %s ", e.getMessage(), targetLocation));
         }
-        locationHandle = locationService.forNewTable(metastore, session, finalSchemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
+        locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
         return locationHandle;
     }
 
     @Override
     public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
     {
-        SchemaTableName localSchemaTableName = tableMetadata.getTable();
-        String localSchemaName = localSchemaTableName.getSchemaName();
-        String tableName = localSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = localSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE;
         List<String> partitionedBy = new ArrayList<String>();
         List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@@ -947,29 +908,29 @@ public class CarbondataMetadata
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
 
-        metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
         Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
 
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                        new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
+                        new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                         new Path(locationHandle.getJsonSerializableTargetPath())));
 
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             try {
                 Map<String, String> serdeParameters = initSerDeProperties(tableName);
-                Table localTable = buildTableObject(
+                Table table = buildTableObject(
                         session.getQueryId(),
-                        localSchemaName,
+                        schemaName,
                         tableName,
                         session.getUser(),
                         columnHandles,
@@ -981,11 +942,11 @@ public class CarbondataMetadata
                         true, // carbon table is set as external table
                         prestoVersion,
                         serdeParameters);
-                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
-                HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
+                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
+                HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
                 metastore.createTable(
                         session,
-                        localTable,
+                        table,
                         principalPrivileges,
                         Optional.empty(),
                         ignoreExisting,
@@ -1051,15 +1012,6 @@ public class CarbondataMetadata
                     super.commit();
                     break;
                 }
-                case ADD_COLUMN:
-                case RENAME_COLUMN:
-                case DROP_COLUMN: {
-                    hdfsEnvironment.doAs(user, () -> {
-                        writeSchemaFile();
-                    });
-                    super.commit();
-                    break;
-                }
                 default: {
                     super.commit();
                 }
@@ -1092,8 +1044,8 @@ public class CarbondataMetadata
     public CarbondataTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
     {
         requireNonNull(tableName, "tableName is null");
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent()) {
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!table.isPresent()) {
             return null;
         }
 
@@ -1102,14 +1054,14 @@ public class CarbondataMetadata
             throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, "Unexpected table present in Hive metastore: " + tableName);
         }
 
-        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(finalTable.get()), finalTable.get().getParameters());
+        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(table.get()), table.get().getParameters());
 
         return new CarbondataTableHandle(
                 tableName.getSchemaName(),
                 tableName.getTableName(),
-                finalTable.get().getParameters(),
-                getPartitionKeyColumnHandles(finalTable.get()),
-                HiveBucketing.getHiveBucketHandle(finalTable.get()));
+                table.get().getParameters(),
+                getPartitionKeyColumnHandles(table.get()),
+                HiveBucketing.getHiveBucketHandle(table.get()));
     }
 
     private Optional<ConnectorOutputMetadata> finishUpdateAndDelete(ConnectorSession session,
@@ -1133,12 +1085,12 @@ public class CarbondataMetadata
 
         hdfsEnvironment.doAs(user, () -> {
             if (blockUpdateDetailsList.size() > 0) {
-                CarbonTable finalCarbonTable = getCarbonTable(tableHandle.getSchemaName(),
+                CarbonTable carbonTable = getCarbonTable(tableHandle.getSchemaName(),
                         tableHandle.getTableName(),
                         MetastoreUtil.getHiveSchema(table.get()),
                         initialConfiguration);
 
-                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(finalCarbonTable);
+                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(carbonTable);
                 SegmentUpdateDetails[] segementDetailsList = statusManager.getUpdateStatusDetails();
                 for (SegmentUpdateDetails segementDetails : segementDetailsList) {
                     segementDetails.getDeletedRowsInBlock();
@@ -1179,26 +1131,28 @@ public class CarbondataMetadata
                                             List<HiveColumnHandle> columnHandles,
                                             Map<String, String> tableProperties)
     {
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
 
         partitionedBy.addAll(CarbondataTableProperties.getPartitionedBy(tableMetadata.getProperties()));
         sortBy.addAll(CarbondataTableProperties.getSortedBy(tableMetadata.getProperties()));
         Optional<HiveBucketProperty> bucketProperty = Optional.empty();
         columnHandles.addAll(getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator));
-        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName)));
+        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, schemaName, tableName)));
     }
 
     @Override
     public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
     {
+        verifyJvmTimeZone();
+
         // get the root directory for the database
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = finalSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE_AS;
 
         List<String> partitionedBy = new ArrayList<String>();
@@ -1206,7 +1160,7 @@ public class CarbondataMetadata
         List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
-        metastore.getDatabase(finalSchemaName).orElseThrow(() -> new SchemaNotFoundException(finalSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         // to avoid type mismatch between HiveStorageFormat & Carbondata StorageFormat this hack no option
         HiveStorageFormat tableStorageFormat = HiveStorageFormat.valueOf("CARBON");
@@ -1222,29 +1176,29 @@ public class CarbondataMetadata
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE_AS);
         Path targetPath = locationService.getTableWriteInfo(locationHandle, false).getTargetPath();
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(finalSchemaName, finalTableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
 
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                                new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName),
+                                new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                                 new Path(locationHandle.getJsonSerializableTargetPath())));
             // Create Carbondata metadata folder and Schema file
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             Path outputPath = new Path(locationHandle.getJsonSerializableTargetPath());
-            Properties schema = readSchemaForCarbon(finalSchemaName, finalTableName, targetPath, columnHandles, partitionColumns);
+            Properties schema = readSchemaForCarbon(schemaName, tableName, targetPath, columnHandles, partitionColumns);
             // Create committer object
             setupCommitWriter(schema, outputPath, initialConfiguration, false);
         });
         try {
             CarbondataOutputTableHandle result = new CarbondataOutputTableHandle(
-                    finalSchemaName,
-                    finalTableName,
+                    schemaName,
+                    tableName,
                     columnHandles,
-                    metastore.generatePageSinkMetadata(new HiveIdentity(session), finalSchemaTableName),
+                    metastore.generatePageSinkMetadata(new HiveIdentity(session), schemaTableName),
                     locationHandle,
                     tableStorageFormat,
                     partitionStorageFormat,
@@ -1255,7 +1209,7 @@ public class CarbondataMetadata
                     EncodedLoadModel, jobContext.getConfiguration().get(LOAD_MODEL)));
 
             LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
-            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), finalSchemaTableName);
+            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
             return result;
         }
         catch (RuntimeException ex) {
@@ -1386,7 +1340,7 @@ public class CarbondataMetadata
         List<Segment> segmentFilesToBeUpdated = blockUpdateDetailsList.stream()
                 .map(SegmentUpdateDetails::getSegmentName)
                 .map(Segment::new).collect(Collectors.toList());
-        List<Segment> finalSegmentFilesToBeUpdatedLatest = new ArrayList<>();
+        List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
         List<Segment> segmentFilesToBeDeleted = blockUpdateDetailsList.stream()
                 .filter(segmentUpdateDetails -> segmentUpdateDetails.getSegmentStatus() != null &&
                         segmentUpdateDetails.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE))
@@ -1396,12 +1350,12 @@ public class CarbondataMetadata
         for (Segment segment : segmentFilesToBeUpdated) {
             String file =
                     SegmentFileStore.writeSegmentFile(carbonTable, segment.getSegmentNo(), timeStamp.toString());
-            finalSegmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
+            segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
         }
         if (!(updateSegmentStatusSuccess &&
                 CarbonUpdateUtil.updateTableMetadataStatus(new HashSet<>(segmentFilesToBeUpdated),
                         carbonTable, timeStamp.toString(), true, segmentFilesToBeDeleted,
-                        finalSegmentFilesToBeUpdatedLatest, ""))) {
+                        segmentFilesToBeUpdatedLatest, ""))) {
             CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timeStamp.toString());
         }
     }
@@ -1463,10 +1417,11 @@ public class CarbondataMetadata
                 Properties hiveschema = MetastoreUtil.getHiveSchema(table);
                 Configuration configuration = jobContext.getConfiguration();
                 configuration.set(SET_OVERWRITE, "false");
-                CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
-                LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
-                loadModel.setSegmentId(loadMetadataDetails.getLoadName());
-                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
+                CarbonLoadModel carbonLoadModel =
+                        HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
+                LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
+                carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
             }
             catch (IOException e) {
                 LOG.error("Error occurred while committing the insert job.", e);
@@ -1553,14 +1508,14 @@ public class CarbondataMetadata
         try {
             hdfsEnvironment.doAs(session.getUser(), () -> {
                 metastore.dropTable(session, handle.getSchemaName(), handle.getTableName());
-                Configuration finalInitialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
+                Configuration initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
                         .getConfiguration(new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
                                 handle.getTableName()), new Path(this.tableStorageLocation.get())));
 
                 Properties schema = MetastoreUtil.getHiveSchema(target.get());
                 schema.setProperty("tablePath", this.tableStorageLocation.get());
                 this.carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(),
-                        schema, finalInitialConfiguration);
+                        schema, initialConfiguration);
                 takeLocks(State.DROP_TABLE);
                 AbsoluteTableIdentifier identifier = this.carbonTable.getAbsoluteTableIdentifier();
                 if (SegmentStatusManager.isLoadInProgressInTable(carbonTable)) {
@@ -1569,7 +1524,7 @@ public class CarbondataMetadata
                 try {
                     //Simultaneous case after acquiring locks we should check table exist.
                     //if table is not there clean the lock folders
-                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, finalInitialConfiguration);
+                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, initialConfiguration);
                 }//CarbonFileException
                 catch (RuntimeException e) {
                     try {
@@ -1620,350 +1575,6 @@ public class CarbondataMetadata
         return serdeParameters;
     }
 
-    @Override
-    public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column)
-    {
-        currentState = State.ADD_COLUMN;
-        updateSchemaInfo(session, tableHandle, column, null, null);
-
-        super.addColumn(session, tableHandle, column);
-    }
-
-    @Override
-    public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target)
-    {
-        currentState = State.RENAME_COLUMN;
-        updateSchemaInfo(session, tableHandle, null, source, target);
-
-        super.renameColumn(session, tableHandle, source, target);
-    }
-
-    @Override
-    public void dropColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle column)
-    {
-        currentState = State.DROP_COLUMN;
-        updateSchemaInfo(session, tableHandle, null, column, null);
-
-        super.dropColumn(session, tableHandle, column);
-    }
-
-    private void updateSchemaInfo(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column, ColumnHandle source, String target)
-    {
-        HiveTableHandle handle = (HiveTableHandle) tableHandle;
-        table = metastore.getTable(new HiveIdentity(session), handle.getSchemaName(), handle.getTableName());
-        String tablePath = table.get().getStorage().getLocation();
-        hdfsEnvironment.doAs(user, () -> {
-            initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
-                    .getConfiguration(
-                            new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
-                                    handle.getTableName()), new Path(tablePath)));
-            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", tablePath);
-            carbonTable = getCarbonTable(handle.getSchemaName(),
-                    handle.getTableName(),
-                    schema,
-                    initialConfiguration);
-        });
-        acquireLocksForAlter();
-        schemaTableName = handle.getSchemaTableName();
-        absoluteTableIdentifier = AbsoluteTableIdentifier.from(tablePath, handle.getTableName(), handle.getSchemaName());
-        tableInfo = carbonTable.getTableInfo();
-        SchemaEvolutionEntry schemaEvolutionEntry;
-        switch (currentState) {
-            case ADD_COLUMN: {
-                schemaEvolutionEntry = updateSchemaInfoAddColumn(column);
-                break;
-            }
-            case RENAME_COLUMN: {
-                schemaEvolutionEntry = updateSchemaInfoRenameColumn(source, target);
-                break;
-            }
-            case DROP_TABLE: {
-                schemaEvolutionEntry = updateSchemaInfoDropColumn(source);
-                break;
-            }
-            default: {
-                return;
-            }
-        }
-        if (schemaEvolutionEntry != null) {
-            tableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList()
-                    .add(schemaEvolutionEntry);
-        }
-    }
-
-    private SchemaEvolutionEntry updateSchemaInfoAddColumn(ColumnMetadata column)
-    {
-        HiveColumnHandle columnHandle = new HiveColumnHandle(column.getName(), HiveType.toHiveType(typeTranslator, column.getType()),
-                column.getType().getTypeSignature(), tableInfo.getFactTable().getListOfColumns().size(), HiveColumnHandle.ColumnType.REGULAR, Optional.empty());
-        TableSchema tableSchema = tableInfo.getFactTable();
-        List<ColumnSchema> tableColumns = tableSchema.getListOfColumns();
-        int currentSchemaOrdinal = tableColumns.stream().max(Comparator.comparing(ColumnSchema::getSchemaOrdinal))
-                .orElseThrow(NoSuchElementException::new).getSchemaOrdinal() + 1;
-        List<ColumnSchema> longStringColumns = new ArrayList<>();
-        List<ColumnSchema> allColumns = tableColumns.stream().filter(cols -> cols.isDimensionColumn()
-                && !cols.getDataType().isComplexType() && cols.getSchemaOrdinal() != -1 && (cols.getDataType() != DataTypes.VARCHAR)).collect(toList());
-
-        TableSchemaBuilder schemaBuilder = new TableSchemaBuilder();
-        List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
-        ColumnSchema newColumn = schemaBuilder.addColumn(new StructField(columnHandle.getName(),
-                        CarbondataHetuFilterUtil.spi2CarbondataTypeMapper(columnHandle)), null,
-                false, false);
-        newColumn.setSchemaOrdinal(currentSchemaOrdinal);
-        columnSchemas.add(newColumn);
-
-        if (newColumn.getDataType() == DataTypes.VARCHAR) {
-            longStringColumns.add(newColumn);
-        }
-        else if (newColumn.isDimensionColumn()) {
-            // add the column which is not long string
-            allColumns.add(newColumn);
-        }
-        // put the old long string columns
-        allColumns.addAll(tableColumns.stream().filter(cols -> cols.isDimensionColumn() && (cols.getDataType() == DataTypes.VARCHAR)).collect(toList()));
-        // and the new long string column after old long string columns
-        allColumns.addAll(longStringColumns);
-        // put complex type columns at the end of dimension columns
-        allColumns.addAll(tableColumns.stream().filter(cols -> cols.isDimensionColumn() &&
-                (cols.isComplexColumn() || cols.getSchemaOrdinal() == -1)).collect(toList()));
-        // original measure columns
-        allColumns.addAll(tableColumns.stream().filter(cols -> !cols.isDimensionColumn()).collect(toList()));
-        // add new measure column
-        if (!newColumn.isDimensionColumn()) {
-            allColumns.add(newColumn);
-        }
-        allColumns.stream().filter(cols -> !cols.isInvisible()).collect(Collectors.groupingBy(ColumnSchema::getColumnName))
-                .forEach((columnName, schemaList) -> {
-                    if (schemaList.size() > 2) {
-                        throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Duplicate columns found"));
-                    }
-                });
-        if (newColumn.isComplexColumn()) {
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Complex column cannot be added"));
-        }
-
-        List<ColumnSchema> finalAllColumns = allColumns;
-        allColumns.stream().forEach(columnSchema -> {
-            List<ColumnSchema> colWithSameId = finalAllColumns.stream().filter(x ->
-                    x.getColumnUniqueId().equals(columnSchema.getColumnUniqueId())).collect(toList());
-            if (colWithSameId.size() > 1) {
-                throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Two columns can not have same columnId"));
-            }
-        });
-        if (tableInfo.getFactTable().getPartitionInfo() != null) {
-            List<ColumnSchema> par = tableInfo.getFactTable().getPartitionInfo().getColumnSchemaList();
-            allColumns = allColumns.stream().filter(cols -> !par.contains(cols)).collect(toList());
-            allColumns.addAll(par);
-        }
-        tableSchema.setListOfColumns(allColumns);
-        tableInfo.setLastUpdatedTime(timeStamp);
-        tableInfo.setFactTable(tableSchema);
-        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
-        schemaEvolutionEntry.setTimeStamp(timeStamp);
-        schemaEvolutionEntry.setAdded(columnSchemas);
-
-        return schemaEvolutionEntry;
-    }
-
-    private SchemaEvolutionEntry updateSchemaInfoRenameColumn(ColumnHandle source, String target)
-    {
-        HiveColumnHandle oldColumnHandle = (HiveColumnHandle) source;
-        String oldColumnName = oldColumnHandle.getColumnName();
-        String newColumnName = target;
-        if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_COLUMN_RENAME, oldColumnHandle.getName())) {
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Alter table rename column is  not supported for index indexschema"));
-        }
-
-        TableSchema tableSchema = tableInfo.getFactTable();
-        List<ColumnSchema> tableColumns = tableSchema.getListOfColumns();
-
-        if (!tableColumns.stream().map(cols -> cols.getColumnName()).collect(toList()).contains(oldColumnHandle.getName())) {
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Column " + oldColumnHandle.getName() + "does not exist in " +
-                    carbonTable.getDatabaseName() + "." + carbonTable.getTableName()));
-        }
-
-        List<ColumnSchema> carbonColumns = carbonTable.getCreateOrderColumn().stream().filter(cols -> !cols.isInvisible())
-                .map(cols -> cols.getColumnSchema()).collect(toList());
-
-        ColumnSchema oldCarbonColumn = carbonColumns.stream().filter(cols -> cols.getColumnName().equalsIgnoreCase(oldColumnName)).findFirst().get();
-        validateColumnsForRenaming(oldCarbonColumn);
-
-        TableSchemaBuilder schemaBuilder = new TableSchemaBuilder();
-        ColumnSchema deletedColumn = schemaBuilder.addColumn(new StructField(oldColumnHandle.getName(),
-                CarbondataHetuFilterUtil.spi2CarbondataTypeMapper(oldColumnHandle)), null, false, false);
-
-        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
-        tableColumns.forEach(cols -> {
-            if (cols.getColumnName().equalsIgnoreCase(oldColumnName)) {
-                cols.setColumnName(newColumnName);
-                schemaEvolutionEntry.setTimeStamp(timeStamp);
-                schemaEvolutionEntry.setAdded(Arrays.asList(cols));
-                schemaEvolutionEntry.setRemoved(Arrays.asList(deletedColumn));
-            }
-        });
-
-        Map<String, String> tableProperties = tableInfo.getFactTable().getTableProperties();
-        tableProperties.forEach((tablePropertyKey, tablePropertyValue) -> {
-            if (tablePropertyKey.equalsIgnoreCase(oldColumnName)) {
-                tableProperties.put(tablePropertyKey, newColumnName);
-            }
-        });
-
-        tableInfo.setLastUpdatedTime(System.currentTimeMillis());
-        tableInfo.setFactTable(tableSchema);
-        return schemaEvolutionEntry;
-    }
-
-    private SchemaEvolutionEntry updateSchemaInfoDropColumn(ColumnHandle column)
-    {
-        HiveColumnHandle columnHandle = (HiveColumnHandle) column;
-        TableSchema tableSchema = tableInfo.getFactTable();
-        List<ColumnSchema> tableColumns = tableSchema.getListOfColumns();
-        int currentSchemaOrdinal = tableColumns.stream().max(Comparator.comparing(ColumnSchema::getSchemaOrdinal))
-                .orElseThrow(NoSuchElementException::new).getSchemaOrdinal() + 1;
-
-        TableSchemaBuilder schemaBuilder = new TableSchemaBuilder();
-        List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
-        ColumnSchema newColumn = schemaBuilder.addColumn(new StructField(columnHandle.getColumnName(),
-                        CarbondataHetuFilterUtil.spi2CarbondataTypeMapper(columnHandle)), null,
-                false, false);
-        newColumn.setSchemaOrdinal(currentSchemaOrdinal);
-        columnSchemas.add(newColumn);
-
-        PartitionInfo partitionInfo = tableInfo.getFactTable().getPartitionInfo();
-        if (partitionInfo != null) {
-            List<String> partitionColumnSchemaList = tableInfo.getFactTable().getPartitionInfo()
-                    .getColumnSchemaList().stream().map(cols -> cols.getColumnName()).collect(toList());
-            if (partitionColumnSchemaList.stream().anyMatch(partitionColumn -> partitionColumn.equals(newColumn.getColumnName()))) {
-                throw new PrestoException(GENERIC_INTERNAL_ERROR, "Partition columns cannot be dropped");
-            }
-            // when table has two columns, dropping unpartitioned column will be wrong
-            if (tableColumns.stream().filter(cols -> !cols.getColumnName().equals(newColumn.getColumnName()))
-                    .map(cols -> cols.getColumnName()).equals(partitionColumnSchemaList)) {
-                throw new PrestoException(GENERIC_INTERNAL_ERROR, "Cannot have table with all columns as partition columns");
-            }
-        }
-
-        if (!tableColumns.stream().filter(cols -> cols.getColumnName().equals(newColumn.getColumnName())).collect(toList()).isEmpty()) {
-            if (newColumn.isComplexColumn()) {
-                throw new PrestoException(GENERIC_INTERNAL_ERROR, "Complex column cannot be dropped");
-            }
-        }
-        else {
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, "Cannot have table with all columns as partition columns");
-        }
-        tableInfo.setLastUpdatedTime(System.currentTimeMillis());
-        tableInfo.setFactTable(tableSchema);
-
-        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
-        schemaEvolutionEntry.setTimeStamp(timeStamp);
-        schemaEvolutionEntry.setRemoved(columnSchemas);
-
-        return schemaEvolutionEntry;
-    }
-
-    private void revertAlterTableChanges()
-    {
-        String tableName = absoluteTableIdentifier.getTableName();
-        String databaseName = absoluteTableIdentifier.getDatabaseName();
-        TableInfo finalTableInfo = carbonTable.getTableInfo();
-        List<SchemaEvolutionEntry> evolutionEntryList = finalTableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
-        Long updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).getTimeStamp();
-        LOG.info("Reverting changes for " + databaseName + "." + tableName);
-        List<ColumnSchema> addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getAdded();
-        List<ColumnSchema> removedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getRemoved();
-        if (updatedTime == timeStamp) {
-            switch (currentState) {
-                case ADD_COLUMN: {
-                    carbonTable.getTableInfo().getFactTable().getListOfColumns().removeAll(addedSchemas);
-                    break;
-                }
-                case DROP_COLUMN: {
-                    finalTableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
-                        if (cols.isInvisible() && removedCols.getColumnUniqueId().equals(cols.getColumnUniqueId())) {
-                            cols.setInvisible(false);
-                        }
-                    }));
-                    break;
-                }
-                default:
-                    break;
-            }
-            evolutionEntryList.remove(evolutionEntryList.size() - 1);
-            writeSchemaFile();
-        }
-    }
-
-    private void writeSchemaFile()
-    {
-        try {
-            String schemaFilePath = CarbonTablePath.getSchemaFilePath(table.get().getStorage().getLocation());
-            SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
-            ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
-            thriftWriter.open(FileWriteOperation.OVERWRITE);
-            thriftWriter.write(schemaConverter.fromWrapperToExternalTableInfo(tableInfo, absoluteTableIdentifier.getTableName(),
-                    absoluteTableIdentifier.getDatabaseName()));
-            thriftWriter.close();
-            FileFactory.getCarbonFile(schemaFilePath).setLastModifiedTime(timeStamp);
-            carbondataTableReader.deleteTableFromCarbonCache(new SchemaTableName(absoluteTableIdentifier.getDatabaseName(), absoluteTableIdentifier.getTableName()));
-            CarbonMetadata.getInstance().removeTable(absoluteTableIdentifier.getTablePath(), absoluteTableIdentifier.getDatabaseName());
-            CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
-            LOG.info("Schema file written");
-        }
-        catch (IOException e) {
-            //TODO handle cases while exception thrown
-            releaseLocks();
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Error while writing to schema file", e));
-        }
-    }
-
-    private void acquireLocksForAlter()
-    {
-        metadataLock = CarbonLockFactory.getCarbonLockObj(carbonTable
-                .getAbsoluteTableIdentifier(), LockUsage.METADATA_LOCK);
-        compactionLock = CarbonLockFactory.getCarbonLockObj(carbonTable
-                .getAbsoluteTableIdentifier(), LockUsage.COMPACTION_LOCK);
-        try {
-            boolean lockStatus = metadataLock.lockWithRetries();
-            if (lockStatus) {
-                LOG.info("Successfully able to get the table metadata file lock");
-            }
-            else {
-                throw new Exception("Table is already locked");
-            }
-
-            if (compactionLock.lockWithRetries()) {
-                LOG.info("Successfully able to get compaction lock");
-            }
-            else {
-                throw new RuntimeException("Unable to get compaction locks");
-            }
-        }
-        catch (Exception e) {
-            LOG.error("Exception in alter operation", e);
-            releaseLocks();
-            throw new PrestoException(GENERIC_INTERNAL_ERROR, "Error while taking locks", e);
-        }
-    }
-
-    private void validateColumnsForRenaming(ColumnSchema oldColumn)
-    {
-        if (carbonTable != null) {
-            // if the column rename is for complex column, block the operation
-            if (oldColumn.isComplexColumn()) {
-                throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Rename column is not supported for complex datatype"));
-            }
-            // if column rename operation is on partition column, then fail the rename operation
-            if (null != carbonTable.getPartitionInfo()) {
-                List<ColumnSchema> partitionColumns = carbonTable.getPartitionInfo().getColumnSchemaList();
-                if (!partitionColumns.stream().filter(cols -> cols.getColumnName()
-                        .equalsIgnoreCase(oldColumn.getColumnName())).collect(toList()).isEmpty()) {
-                    throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Cannot rename a partition column"));
-                }
-            }
-        }
-    }
-
     private void writeSegmentFileAndSetLoadModel()
     {
         hdfsEnvironment.doAs(user, () -> {
@@ -2026,38 +1637,38 @@ public class CarbondataMetadata
     @Override
     protected ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName)
     {
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent() || finalTable.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
             throw new TableNotFoundException(tableName);
         }
 
-        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(finalTable.get(), typeManager);
+        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
         ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             columns.add(metadataGetter.apply(columnHandle));
         }
 
         // External location property
         ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
-        properties.put(LOCATION_PROPERTY, finalTable.get().getStorage().getLocation());
+        properties.put(LOCATION_PROPERTY, table.get().getStorage().getLocation());
 
         // Storage format property
         properties.put(HiveTableProperties.STORAGE_FORMAT_PROPERTY, CarbondataStorageFormat.CARBON);
 
         // Partitioning property
-        List<String> partitionedBy = finalTable.get().getPartitionColumns().stream()
+        List<String> partitionedBy = table.get().getPartitionColumns().stream()
                 .map(Column::getName)
                 .collect(toList());
         if (!partitionedBy.isEmpty()) {
             properties.put(HiveTableProperties.PARTITIONED_BY_PROPERTY, partitionedBy);
         }
 
-        Optional<String> comment = Optional.ofNullable(finalTable.get().getParameters().get(TABLE_COMMENT));
+        Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));
 
         // add partitioned columns into immutableColumns
         ImmutableList.Builder<ColumnMetadata> immutableColumns = ImmutableList.builder();
 
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             if (columnHandle.getColumnType().equals(HiveColumnHandle.ColumnType.PARTITION_KEY)) {
                 immutableColumns.add(metadataGetter.apply(columnHandle));
             }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -37,6 +37,7 @@ import io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore;
 import io.prestosql.plugin.hive.security.AccessControlMetadataFactory;
 import io.prestosql.plugin.hive.statistics.MetastoreHiveStatisticsProvider;
 import io.prestosql.spi.type.TypeManager;
+import org.joda.time.DateTimeZone;
 
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
@@ -49,6 +50,7 @@ public class CarbondataMetadataFactory
         extends HiveMetadataFactory
 {
     private static final Logger log = Logger.get(HiveMetadataFactory.class);
+    private final boolean allowCorruptWritesForTesting;
     private final boolean skipDeletionForAlter;
     private final boolean skipTargetCleanupOnRollback;
     private final boolean writesToNonManagedTablesEnabled;
@@ -58,6 +60,7 @@ public class CarbondataMetadataFactory
     private final HiveMetastore metastore;
     private final HdfsEnvironment hdfsEnvironment;
     private final HivePartitionManager partitionManager;
+    private final DateTimeZone timeZone;
     private final TypeManager typeManager;
     private final LocationService locationService;
     private final BoundedExecutor renameExecution;
@@ -90,8 +93,9 @@ public class CarbondataMetadataFactory
                                      AccessControlMetadataFactory accessControlMetadataFactory,
                                      CarbondataTableReader carbondataTableReader)
     {
-        this(metastore, hdfsEnvironment, partitionManager,
+        this(metastore, hdfsEnvironment, partitionManager, carbondataConfig.getDateTimeZone(),
                 carbondataConfig.getMaxConcurrentFileRenames(),
+                carbondataConfig.getAllowCorruptWritesForTesting(),
                 carbondataConfig.isSkipDeletionForAlter(),
                 carbondataConfig.isSkipTargetCleanupOnRollback(),
                 true,
@@ -104,13 +108,13 @@ public class CarbondataMetadataFactory
                 vacuumExecutorService, heartbeatService, hiveMetastoreClientService, typeTranslator, nodeVersion.toString(),
                 accessControlMetadataFactory, carbondataTableReader, carbondataConfig.getStoreLocation(),
                 carbondataConfig.getMajorVacuumSegSize(), carbondataConfig.getMinorVacuumSegCount(),
-                carbondataConfig.getAutoVacuumEnable(), carbondataConfig.getMetastoreWriteBatchSize());
+                carbondataConfig.getAutoVacuumEnable());
     }
 
     public CarbondataMetadataFactory(HiveMetastore metastore, HdfsEnvironment hdfsEnvironment,
-                                     HivePartitionManager partitionManager,
+                                     HivePartitionManager partitionManager, DateTimeZone timeZone,
                                      int maxConcurrentFileRenames,
-                                     boolean skipDeletionForAlter,
+                                     boolean allowCorruptWritesForTesting, boolean skipDeletionForAlter,
                                      boolean skipTargetCleanupOnRollback, boolean writesToNonManagedTablesEnabled,
                                      boolean createsOfNonManagedTablesEnabled, boolean tableCreatesWithLocationAllowed,
                                      long perTransactionCacheMaximumSize,
@@ -124,12 +128,14 @@ public class CarbondataMetadataFactory
                                      TypeTranslator typeTranslator, String hetuVersion,
                                      AccessControlMetadataFactory accessControlMetadataFactory,
                                      CarbondataTableReader carbondataTableReader, String storeLocation, long majorVacuumSegSize, long minorVacuumSegCount,
-                                     boolean autoVacuumEnable, int hmsWriteBatchSize)
+                                     boolean autoVacuumEnable)
     {
         super(metastore,
                 hdfsEnvironment,
                 partitionManager,
+                timeZone,
                 maxConcurrentFileRenames,
+                allowCorruptWritesForTesting,
                 skipDeletionForAlter,
                 skipTargetCleanupOnRollback,
                 writesToNonManagedTablesEnabled,
@@ -148,9 +154,8 @@ public class CarbondataMetadataFactory
                 typeTranslator,
                 hetuVersion,
                 accessControlMetadataFactory,
-                2, 0.0, false,
-                Optional.of(new Duration(5, TimeUnit.MINUTES)),
-                hmsWriteBatchSize);
+                2, 0.0, false, Optional.of(new Duration(5, TimeUnit.MINUTES)));
+        this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
         this.skipDeletionForAlter = skipDeletionForAlter;
         this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
         this.writesToNonManagedTablesEnabled = writesToNonManagedTablesEnabled;
@@ -160,6 +165,7 @@ public class CarbondataMetadataFactory
         this.metastore = requireNonNull(metastore, "metastore is null");
         this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
         this.partitionManager = requireNonNull(partitionManager, "partitionManager is null");
+        this.timeZone = requireNonNull(timeZone, "timeZone is null");
         this.typeManager = requireNonNull(typeManager, "typeManager is null");
         this.locationService = requireNonNull(locationService, "locationService is null");
         this.partitionUpdateCodec = requireNonNull(partitionUpdateCodec, "partitionUpdateCodec is null");
@@ -168,6 +174,13 @@ public class CarbondataMetadataFactory
         this.hetuVersion = requireNonNull(hetuVersion, "hetuVersion is null");
         this.accessControlMetadataFactory = requireNonNull(accessControlMetadataFactory,
                 "accessControlMetadataFactory is null");
+        if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
+            log.warn(
+                    "Hive writes are disabled. To write data to Hive, your JVM timezone must match the " +
+                            "Hive storage timezone. Add -Duser.timezone=%s to your JVM arguments",
+                    timeZone.getID());
+        }
+
         this.renameExecution = new BoundedExecutor(executorService, maxConcurrentFileRenames);
         this.vacuumExecutorService = requireNonNull(vacuumExecutorService, "vacuumExecutorService is null");
         this.hiveMetastoreClientService = requireNonNull(hiveMetastoreClientService, "hiveMetastoreClientService is null");
@@ -191,18 +204,20 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
+        SemiTransactionalHiveMetastore metastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
                         vacuumExecutorService, this.vacuumCleanupInterval, this.skipDeletionForAlter,
                         this.skipTargetCleanupOnRollback,
                         this.hiveTransactionHeartbeatInterval,
-                        this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
+                        this.heartbeatService, hiveMetastoreClientService);
 
-        return new CarbondataMetadata(semiTransactionalHiveMetastore,
+        return new CarbondataMetadata(metastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
+                this.timeZone,
+                this.allowCorruptWritesForTesting,
                 this.writesToNonManagedTablesEnabled,
                 this.createsOfNonManagedTablesEnabled,
                 this.tableCreatesWithLocationAllowed,
@@ -212,8 +227,8 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
+                new MetastoreHiveStatisticsProvider(metastore),
+                this.accessControlMetadataFactory.create(metastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        finalHdfsEnvironment.doAs(session.getUser(), () -> {
+        hdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,7 +163,6 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
-    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int columnBatchSize = 0;
+        int batchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    columnBatchSize = columnarBatch.numRows();
-                    if (columnBatchSize == 0) {
+                    batchSize = columnarBatch.numRows();
+                    if (batchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(columnBatchSize, blocks);
+            Page page = new Page(batchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataSplitManager.java

@@ -14,7 +14,6 @@
 package io.hetu.core.plugin.carbondata;
 
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.impl.CarbondataLocalInputSplit;
 import io.hetu.core.plugin.carbondata.impl.CarbondataLocalMultiBlockSplit;
@@ -140,7 +139,7 @@ public class CarbondataSplitManager
     @Override
     public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle,
             ConnectorSession session, ConnectorTableHandle tableHandle,
-            SplitSchedulingStrategy splitSchedulingStrategy, Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier,
+            SplitSchedulingStrategy splitSchedulingStrategy, Supplier<Set<DynamicFilter>> dynamicFilterSupplier,
             Optional<QueryType> queryType, Map<String, Object> queryProperties,
             Set<TupleDomain<ColumnMetadata>> userDefinedCachePredicates,
             boolean partOfReuse)
@@ -201,7 +200,7 @@ public class CarbondataSplitManager
                             0, 0, 0, 0,
                             properties, new ArrayList(), getHostAddresses(split.getLocations()),
                             OptionalInt.empty(), false, new HashMap<>(),
-                            Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
+                            Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
                     /* Todo: Make this part aligned with rest of the HiveSlipt loading flow...
                      *   and figure out how to pass valid transaction Ids to CarbonData? */
                 }
@@ -310,7 +309,7 @@ public class CarbondataSplitManager
                         schemaTableName.getTableName(), tablePath, 0L, 0L, 0L, 0L,
                         properties, new ArrayList(), getHostAddresses(currSplit.getLocations()),
                         OptionalInt.empty(), false, new HashMap<>(),
-                        Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of())));
+                        Optional.empty(), false, Optional.empty(), Optional.empty(), false)));
             }
         }
         LOGGER.info("Splits for compaction built and ready");

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableHandle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -42,10 +42,4 @@ public class CarbondataTableHandle
     {
         return false;
     }
-
-    @Override
-    public boolean isUpdateAsInsertSupported()
-    {
-        return true;
-    }
 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,17 +90,16 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
-        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            finalName = name.substring(0, name.length() - 4).trim();
+            name = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            finalName = name.substring(0, name.length() - 5).trim();
+            name = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(finalName, order);
+        return new SortingColumn(name, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -47,7 +47,6 @@ import java.util.Set;
 
 import static java.util.Objects.requireNonNull;
 import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_LOCATION;
-import static org.joda.time.DateTimeZone.UTC;
 
 public class CarbondataWriterFactory
         extends HiveWriterFactory
@@ -88,7 +87,7 @@ public class CarbondataWriterFactory
                 additionalTableParameters, bucketCount, sortedBy, locationHandle,
                 locationService, queryId, pageSinkMetadataProvider,
                 typeManager, hdfsEnvironment, pageSorter, sortBufferSize,
-                maxOpenSortFiles, immutablePartitions, UTC, session, nodeManager,
+                maxOpenSortFiles, immutablePartitions, session, nodeManager,
                 eventClient, hiveSessionProperties, hiveWriterStats, orcFileWriterFactory);
 
         this.additionalJobConf = requireNonNull(additionalJobConf, "Additional JobConf is null");
@@ -133,7 +132,6 @@ public class CarbondataWriterFactory
     {
     }
 
-    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,15 +143,14 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDecimal(inputRowId, value, precision);
+                columnVector.putDecimal(rowId, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -186,9 +185,8 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(inputRowId++, value);
+            columnVector.putByteArray(rowId++, value);
         }
     }
 
@@ -305,90 +303,84 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putFloat(inputRowId, src[i]);
+                columnVector.putFloat(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putShort(inputRowId, src[i]);
+                columnVector.putShort(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putInt(inputRowId, src[i]);
+                columnVector.putInt(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putLong(inputRowId, src[i]);
+                columnVector.putLong(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDouble(inputRowId, src[i]);
+                columnVector.putDouble(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putByte(inputRowId, src[i]);
+                columnVector.putByte(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/impl/CarbondataTableReader.java

@@ -56,10 +56,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.log4j.Logger;
 import org.apache.thrift.TBase;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -142,19 +139,10 @@ public class CarbondataTableReader
                 carbonTableCacheModel.getCarbonTable().isTransactionalTable()) {
             CarbonTable carbonTable = carbonTableCacheModel.getCarbonTable();
             try {
-                String schemaFilePath = CarbonTablePath
-                        .getSchemaFilePath(
-                                carbonTable.getTablePath());
-                FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
-
-                CarbonFile carbonFile = FileFactory.getCarbonFile(schemaFilePath,
-                        config);
-
-                long latestTime = carbonFile.getLastModifiedTime();
-                // Workaround for cache updation error due to round off of latestTime inside LocalCarbonFile
-                if (fileType == FileFactory.FileType.LOCAL) {
-                    latestTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
-                }
+                long latestTime = FileFactory.getCarbonFile(CarbonTablePath
+                                .getSchemaFilePath(
+                                        carbonTable.getTablePath()),
+                        config).getLastModifiedTime();
                 carbonTableCacheModel.setCurrentSchemaTime(latestTime);
                 if (!carbonTableCacheModel.isValid()) {
                     // Invalidate datamaps
@@ -162,7 +150,7 @@ public class CarbondataTableReader
                             .clearIndex(carbonTableCacheModel.getCarbonTable().getAbsoluteTableIdentifier());
                 }
             }
-            catch (CarbonFileException | IOException e) {
+            catch (CarbonFileException e) {
                 carbonCache.get().remove(schemaTableName);
             }
         }
@@ -223,11 +211,6 @@ public class CarbondataTableReader
                     tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
                     thriftReader.close();
                     modifiedTime = schemaFile.getLastModifiedTime();
-                    FileFactory.FileType fileType = FileFactory.getFileType(schemaFilePath);
-                    // Workaround for cache updation error due to round off of latesttime inside LocalCarbonFile
-                    if (fileType == FileFactory.FileType.LOCAL) {
-                        modifiedTime = Files.readAttributes(new File(schemaFilePath).toPath(), BasicFileAttributes.class).lastModifiedTime().toMillis();
-                    }
                 }
                 else {
                     tableInfo = CarbonUtil.inferSchema(tablePath, table.getTableName(), false, config);
@@ -340,7 +323,7 @@ public class CarbondataTableReader
                             inputSplits.get(j).stream().flatMap(f -> Arrays.stream(getLocations(f))).distinct()
                                     .toArray(String[]::new)));
                 }
-                LOGGER.error("Size of MultiblockList   " + multiBlockSplitList.size());
+                LOGGER.error("Size fo MultiblockList   " + multiBlockSplitList.size());
             }
         }
         catch (IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,9 +58,8 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIdx++] == 1);
+            type.writeBoolean(builder, src[srcIndex++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,9 +76,8 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(id++, value, precision);
+            putDecimal(rowId++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,9 +58,8 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(id++, value);
+            putInt(rowId++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -40,7 +40,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
 
 public class TestCarbonAutoVacuum
 {
@@ -100,6 +99,7 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
+        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -145,12 +145,11 @@ public class TestCarbonAutoVacuum
 
         try {
             CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
-            assertNotNull(catalog);
             connector = catalog.getConnector(catalog.getConnectorCatalogName());
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -216,7 +215,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -62,6 +62,7 @@ public class TestCarbondataAutoCleanup
     public void setup() throws Exception
     {
         logger.info("Setup begin: " + this.getClass().getSimpleName());
+        String dataPath = rootPath + "/src/test/resources/alldatatype.csv";
 
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
@@ -89,7 +90,6 @@ public class TestCarbondataAutoCleanup
         hetuServer.execute("drop table if exists testdb.testtableautocleanup6");
         hetuServer.execute("drop table if exists testdb.testtableautocleanup7");
         hetuServer.execute("drop table if exists testdb.testtableautocleanup8");
-        hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
         hetuServer.execute("drop schema if exists testdb");
         hetuServer.execute("drop schema if exists default");
         hetuServer.execute("create schema testdb");
@@ -130,7 +130,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -159,7 +159,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -189,7 +189,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -201,32 +201,32 @@ public class TestCarbondataAutoCleanup
     {
         try {
             hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = true");
-            hetuServer.execute("drop table if exists testdb.testtableautocleanupwithpushdown");
-            hetuServer.execute("CREATE TABLE testdb.testtableautocleanupwithpushdown (a int, b int)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (10, 11)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (110, 211)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (120, 311)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 411)");
-            hetuServer.execute("INSERT INTO testdb.testtableautocleanupwithpushdown VALUES (130, 511)");
-            hetuServer.execute("vacuum table testdb.testtableautocleanupwithpushdown AND WAIT");
+            hetuServer.execute("drop table if exists testdb.testtableautocleanup3");
+            hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)");
+            hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)");
+            hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT");
 
-            reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Metadata");
+            reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata");
             CarbondataMetadata.enableTracingCleanupTask(true);
 
-            hetuServer.execute("DELETE FROM testdb.testtableautocleanupwithpushdown WHERE a=130");
+            hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130");
             try {
                 CarbondataMetadata.waitForSubmittedTasksFinish();
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_0", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_1", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
-                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
+                assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-                logger.debug(exception.getMessage());
+
             }
         }
         finally {
             CarbondataMetadata.enableTracingCleanupTask(false);
-            hetuServer.execute("drop table testdb.testtableautocleanupwithpushdown");
+            hetuServer.execute("drop table testdb.testtableautocleanup3");
             hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = false");
         }
     }
@@ -254,7 +254,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -285,7 +285,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -314,7 +314,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -344,7 +344,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -374,7 +374,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -403,7 +403,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -15,7 +15,6 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -37,7 +36,6 @@ import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
-    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -116,7 +114,7 @@ public class TestsWithHiveConnector
         assertEquals(FileFactory.isFileExist(storePath +
                 "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
         hetuServer.execute("DROP TABLE hive.default.parttable");
     }
@@ -125,37 +123,37 @@ public class TestsWithHiveConnector
     public void deleteTransactionTableDirUsing2tables()
             throws SQLException
     {
-        hetuServer.execute("drop table if exists hive.default.parttable2");
+        hetuServer.execute("drop table if exists hive.default.parttable");
         hetuServer.execute("drop table if exists hive.default.parttable1");
         hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = true");
-        hetuServer.execute("create table hive.default.parttable2 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable2 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable2 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable2 values (3,2013)");
+        hetuServer.execute("create table hive.default.parttable (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
+        hetuServer.execute("insert into hive.default.parttable values (1,2011)");
+        hetuServer.execute("insert into hive.default.parttable values (2,2012)");
+        hetuServer.execute("insert into hive.default.parttable values (3,2013)");
 
         hetuServer.execute("create table hive.default.parttable1 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
         hetuServer.execute("insert into hive.default.parttable1 values (1,2011)");
         hetuServer.execute("insert into hive.default.parttable1 values (2,2012)");
         hetuServer.execute("insert into hive.default.parttable1 values (3,2013)");
 
-        hetuServer.execute("delete from hive.default.parttable2 where year >= (select max(year) from hive.default.parttable1)  ");
+        hetuServer.execute("delete from hive.default.parttable where year >= (select max(year) from hive.default.parttable1)  ");
         try {
             assertEquals(FileFactory.isFileExist(storePath +
-                    "hive.store/default/parttable2/year=2013", false), false);
+                    "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
-        hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
-        hetuServer.execute("delete from hive.default.parttable2 where year >= (select year from hive.default.parttable1 where orderkey=4 )  ");
+        hetuServer.execute("insert into hive.default.parttable values (4,2014)");
+        hetuServer.execute("delete from hive.default.parttable where year >= (select year from hive.default.parttable1 where orderkey=4 )  ");
         try {
             assertEquals(FileFactory.isFileExist(storePath +
-                    "hive.store/default/parttable2/year=2014", false), false);
+                    "hive.store/default/parttable/year=2014", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
-        hetuServer.execute("DROP TABLE hive.default.parttable2");
+        hetuServer.execute("DROP TABLE hive.default.parttable");
         hetuServer.execute("DROP TABLE hive.default.parttable1");
     }
 
@@ -163,72 +161,46 @@ public class TestsWithHiveConnector
     public void deleteTransactionTableDirDisable()
             throws SQLException
     {
-        hetuServer.execute("drop table if exists hive.default.parttable3");
+        hetuServer.execute("drop table if exists hive.default.parttable");
         hetuServer.execute("set session DELETE_TRANSACTIONAL_TABLE_DIRECT = false");
-        hetuServer.execute("create table hive.default.parttable3 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable3 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable3 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable3 values (3,2013)");
+        hetuServer.execute("create table hive.default.parttable (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
+        hetuServer.execute("insert into hive.default.parttable values (1,2011)");
+        hetuServer.execute("insert into hive.default.parttable values (2,2012)");
+        hetuServer.execute("insert into hive.default.parttable values (3,2013)");
 
-        hetuServer.execute("create table hive.default.parttable4 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
-        hetuServer.execute("insert into hive.default.parttable4 values (1,2011)");
-        hetuServer.execute("insert into hive.default.parttable4 values (2,2012)");
-        hetuServer.execute("insert into hive.default.parttable4 values (3,2013)");
+        hetuServer.execute("create table hive.default.parttable1 (orderkey int, year int) WITH (transactional = true , format = 'ORC', partitioned_by = ARRAY[ 'year' ] )");
+        hetuServer.execute("insert into hive.default.parttable1 values (1,2011)");
+        hetuServer.execute("insert into hive.default.parttable1 values (2,2012)");
+        hetuServer.execute("insert into hive.default.parttable1 values (3,2013)");
 
-        hetuServer.execute("delete from hive.default.parttable3 where year >= (select max(year) from hive.default.parttable4)  ");
+        hetuServer.execute("delete from hive.default.parttable where year >= (select max(year) from hive.default.parttable1)  ");
         try {
             assertEquals(FileFactory.isFileExist(storePath +
-                    "/hive.store/default/parttable3/year=2013", false), true);
+                    "/hive.store/default/parttable/year=2013", false), true);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
-        hetuServer.execute("DROP TABLE hive.default.parttable3");
+        hetuServer.execute("DROP TABLE hive.default.parttable");
     }
 
     @Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
     public void block_Carbondata_Table_from_Hive()
             throws SQLException
     {
-        hetuServer.execute("CREATE TABLE carbondata.default.demotable1 (c1 int)");
+        hetuServer.execute("CREATE TABLE carbondata.default.demotable (c1 int)");
 
         hetuServer.execute("use hive.default");
 
-        runQueryAndAssertErrorMessage("SELECT * FROM demotable1",
+        runQueryAndAssertErrorMessage("SELECT * FROM demotable",
                 "Hive connector can't read carbondata tables");
 
-        runQueryAndAssertErrorMessage("INSERT INTO demotable1 VALUES(1)",
+        runQueryAndAssertErrorMessage("INSERT INTO demotable VALUES(1)",
                 "Tables with MapredCarbonInputFormat are not supported by Hive connector");
 
-        runQueryAndAssertErrorMessage("UPDATE demotable1 SET c1=1",
+        runQueryAndAssertErrorMessage("UPDATE demotable SET c1=1",
                 "Tables with MapredCarbonInputFormat are not supported by Hive connector");
 
-        hetuServer.execute("use carbondata.default");
-        hetuServer.execute("DROP TABLE carbondata.default.demotable1");
-    }
-
-    @Test(dependsOnMethods = {"block_Hive_Table_from_Carbondata"})
-    public void block_Carbondata_Table_change_from_Hive()
-            throws SQLException
-    {
-        hetuServer.execute("CREATE TABLE carbondata.default.changetable (a int, b int, c int)");
-        hetuServer.execute("use hive.default");
-        //rename column
-        runQueryAndAssertErrorMessage("ALTER TABLE default.changetable RENAME COLUMN a TO aa",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //add column
-        runQueryAndAssertErrorMessage("ALTER TABLE default.changetable ADD COLUMN e int COMMENT 'Hello World'",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //drop column
-        runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable DROP COLUMN c",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //change table name
-        runQueryAndAssertErrorMessage("ALTER TABLE hive.default.changetable RENAME TO hive.default.changetable_N",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        //drop table
-        runQueryAndAssertErrorMessage("DROP TABLE default.changetable",
-                "Tables with MapredCarbonInputFormat are not supported by Hive connector");
-        hetuServer.execute("use carbondata.default");
-        hetuServer.execute("DROP TABLE carbondata.default.changetable");
+        hetuServer.execute("DROP TABLE hive.default.demotable");
     }
 
     private Map<String, String> createHiveProperties()
@@ -237,11 +209,6 @@ public class TestsWithHiveConnector
         hiveProperties.put("hive.metastore", "file");
         hiveProperties.put("hive.allow-drop-table", "true");
         hiveProperties.put("hive.non-managed-table-writes-enabled", "true");
-        hiveProperties.put("hive.allow-add-column", "true");
-        hiveProperties.put("hive.allow-drop-column", "true");
-        hiveProperties.put("hive.allow-rename-table", "true");
-        hiveProperties.put("hive.allow-comment-table", "true");
-        hiveProperties.put("hive.allow-rename-column", "true");
         hiveProperties.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store");
         return hiveProperties;
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -123,20 +123,14 @@ public class HetuTestServer
     public List<Map<String, Object>> executeQuery(String query) throws SQLException
     {
         logger.info(">>>>> Executing Query: " + query);
-        ResultSet rs = null;
         try {
-            rs = statement.executeQuery(query);
+            ResultSet rs = statement.executeQuery(query);
             return convertResultSetToList(rs);
         }
         catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
-        finally {
-            if (rs != null) {
-                rs.close();
-            }
-        }
     }
 
     private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
@@ -192,7 +186,7 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
@@ -203,7 +197,7 @@ public class HetuTestServer
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
         }
         catch (RuntimeException e) {

hetu-clickhouse/pom.xml

@@ -1,11 +1,12 @@
 <?xml version="1.0"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-clickhouse</artifactId>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -61,7 +61,6 @@ import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.sql.Types;
-import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -289,40 +288,6 @@ public class ClickHouseClient
         }
     }
 
-    @Override
-    public long getTableModificationTime(ConnectorSession session, JdbcTableHandle handle)
-    {
-        String sql = format(
-                "SELECT max(modification_time) as modified_time FROM system.parts WHERE database='%s' and table='%s'",
-                handle.getSchemaTableName().getSchemaName(),
-                handle.getSchemaTableName().getTableName());
-
-        try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session));
-                PreparedStatement preparedStatement = getPreparedStatement(connection, sql)) {
-            ResultSet resultSet = preparedStatement.executeQuery();
-            if (resultSet.next()) {
-                String updateTime = resultSet.getString("modified_time");
-                SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
-                return parser.parse(updateTime).getTime();
-            }
-            else {
-                // In the case where for some reason the table doesn't exist anymore in system.parts
-                return -1L;
-            }
-        }
-        catch (Exception e) {
-            // We want to make sure the query doesn't fail because of star-tree not being able to get last modified time
-            logger.error("Exception thrown while trying to get modified time", e);
-            return -1L;
-        }
-    }
-
-    @Override
-    public boolean isPreAggregationSupported(ConnectorSession session)
-    {
-        return true;
-    }
-
     @Override
     protected void renameTable(JdbcIdentity identity, String catalogName, String schemaName, String tableName, SchemaTableName newTable)
     {
@@ -345,9 +310,8 @@ public class ClickHouseClient
     }
 
     @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
+    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
     {
-        String newColumnName = inputNewColumnName;
         try (Connection connection = connectionFactory.openConnection(identity)) {
             if (connection.getMetaData().storesUpperCaseIdentifiers()) {
                 newColumnName = newColumnName.toUpperCase(ENGLISH);

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClientModule.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseConstants.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHousePlugin.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -14,23 +14,8 @@
  */
 package io.hetu.core.plugin.clickhouse;
 
-import io.prestosql.plugin.jdbc.BaseJdbcConfig;
 import io.prestosql.plugin.jdbc.JdbcPlugin;
-import io.prestosql.spi.function.ConnectorConfig;
-import io.prestosql.spi.queryeditorui.ConnectorUtil;
-import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
 
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Optional;
-
-@ConnectorConfig(connectorLabel = "Clickhouse: Query data in Clickhouse system",
-        propertiesEnabled = true,
-        catalogConfigFilesEnabled = true,
-        globalConfigFilesEnabled = true,
-        docLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html",
-        configLink = "https://openlookeng.io/docs/docs/connector/clickhouse.html#configuration")
 public class ClickHousePlugin
         extends JdbcPlugin
 {
@@ -38,14 +23,4 @@ public class ClickHousePlugin
     {
         super("clickhouse", new ClickHouseClientModule());
     }
-
-    @Override
-    public Optional<ConnectorWithProperties> getConnectorWithProperties()
-    {
-        ConnectorConfig connectorConfig = ClickHousePlugin.class.getAnnotation(ConnectorConfig.class);
-        ArrayList<Method> methods = new ArrayList<>();
-        methods.addAll(Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
-        methods.addAll(Arrays.asList(ClickHouseConfig.class.getDeclaredMethods()));
-        return ConnectorUtil.assembleConnectorProperties(connectorConfig, methods);
-    }
 }

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseQueryBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -41,7 +41,6 @@ public class ClickHouseApplyRemoteFunctionPushDown
     /**
      * rewrite the remote function to a executable function in the data source.
      */
-    @Override
     public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
     {
         if (!isConnectorSupportedRemoteFunction(callExpression)) {

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHousePushDownParameter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseQueryGenerator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseRowExpressionConverter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -37,9 +37,8 @@ public class ClickHouseSqlStatementWriter
     }
 
     @Override
-    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
+    public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
     {
-        String functionName = inputFunctionName;
         if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
             functionName = "varPop";
         }

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/externalfunc/ClickHouseExternalDateTimeFunctions.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/externalfunc/ClickHouseExternalFunctionHub.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/BuildInDirectMapFunctionCallRewriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/ClickHouseUnsupportedFunctionCallRewriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/RewriteUtil.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/UdfFunctionRewriteConstants.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/rewrite/functioncall/DateParseFunctionCallRewriter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseClientTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseConfigTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseConstantsTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHousePluginTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -205,7 +205,7 @@ public final class ClickHouseServerTest
     {
         String actualTable = tablePattern;
 
-        for (String table : tables) {
+        for (String table : tables) { //tableName + _ + UUID
             int lastIndex = table.lastIndexOf("_");
             if (lastIndex == -1) {
                 continue;

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/DataBaseTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/algorithm/SequenceUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-common/src/main/java/io/hetu/core/common/util/SecurePathWhiteList.java

@@ -16,7 +16,6 @@ package io.hetu.core.common.util;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
@@ -37,7 +36,6 @@ public class SecurePathWhiteList
 
     /**
      * Due to security concerns, all data must be read from one of the whitelisted paths
-     *
      * @return
      * @throws IOException
      */
@@ -55,11 +53,6 @@ public class SecurePathWhiteList
         return securePathwhiteList;
     }
 
-    public static boolean isSecurePath(Path absolutePath) throws IOException
-    {
-        return isSecurePath(absolutePath.toString());
-    }
-
     public static boolean isSecurePath(String absolutePath) throws IOException
     {
         // absolutePath

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,6 +47,16 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
+
+        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
+        // as per link above, the default SSLContext will be constructed using the default KeyManager and
+        // default TrustManager. Those can be configured using the following system properties:
+        // javax.net.ssl.keyStore
+        // javax.net.ssl.keyStorePassword
+        // javax.net.ssl.keyStoreType
+        // javax.net.ssl.trustStore
+        // javax.net.ssl.trustStorePassword
+        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,7 +13,6 @@
  */
 package io.hetu.core.common.util;
 
-import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -30,8 +29,6 @@ import java.util.Optional;
 
 public class TrustStore
 {
-    private static final Logger LOGGER = Logger.get(TrustStore.class);
-
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -51,7 +48,6 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
-            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/algorithm/TestSequenceUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
+            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
+            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-cube</artifactId>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeAggregateFunction.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -1,80 +0,0 @@
-/*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package io.hetu.core.spi.cube;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.Objects;
-
-public class CubeFilter
-{
-    private final String sourceTablePredicate;
-    private final String cubePredicate;
-
-    @JsonCreator
-    public CubeFilter(
-            @JsonProperty("sourceTablePredicate") String sourceTablePredicate,
-            @JsonProperty("cubePredicate") String cubePredicate)
-    {
-        this.sourceTablePredicate = sourceTablePredicate;
-        this.cubePredicate = cubePredicate;
-    }
-
-    public CubeFilter(String sourceTablePredicate)
-    {
-        this(sourceTablePredicate, null);
-    }
-
-    public String getSourceTablePredicate()
-    {
-        return sourceTablePredicate;
-    }
-
-    public String getCubePredicate()
-    {
-        return cubePredicate;
-    }
-
-    @Override
-    public boolean equals(Object o)
-    {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        CubeFilter that = (CubeFilter) o;
-        return Objects.equals(sourceTablePredicate, that.sourceTablePredicate)
-                && Objects.equals(cubePredicate, that.cubePredicate);
-    }
-
-    @Override
-    public int hashCode()
-    {
-        return Objects.hash(sourceTablePredicate, cubePredicate);
-    }
-
-    @Override
-    public String toString()
-    {
-        return "CubeFilter{" +
-                "sourceTablePredicate='" + sourceTablePredicate + '\'' +
-                ", cubePredicate='" + cubePredicate + '\'' +
-                '}';
-    }
-}

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeMetadata.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -56,11 +56,6 @@ public interface CubeMetadata
      */
     List<String> getAggregations();
 
-    /**
-     * Cube selection filter
-     */
-    CubeFilter getCubeFilter();
-
     /**
      * Return the group by columns
      */
@@ -110,6 +105,11 @@ public interface CubeMetadata
      */
     List<AggregationSignature> getAggregationSignatures();
 
+    /**
+     * Return cube predicate string
+     */
+    String getPredicateString();
+
     /**
      * Return the status of the cube
      */

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeMetadataBuilder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -25,7 +25,7 @@ public interface CubeMetadataBuilder
 
     void addGroup(Set<String> group);
 
-    void withCubeFilter(CubeFilter cubeFilter);
+    void withPredicate(String predicateString);
 
     void setCubeStatus(CubeStatus cubeStatus);
 

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -23,6 +23,7 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.StringJoiner;
 
@@ -30,12 +31,13 @@ import static java.util.Objects.requireNonNull;
 
 public class CubeStatement
 {
-    private final String from;
     private final Set<String> groupBy;
     private final Set<String> selection;
+    private final String from;
     private final List<AggregationSignature> aggregations;
+    private final String where;
 
-    private CubeStatement(
+    public CubeStatement(
             Set<String> selection,
             String from,
             Set<String> groupBy,
@@ -43,6 +45,21 @@ public class CubeStatement
     {
         this.selection = requireNonNull(selection, "selection is null");
         this.from = requireNonNull(from, "from is null");
+        this.where = null;
+        this.groupBy = requireNonNull(groupBy, "groupBy is null");
+        this.aggregations = requireNonNull(aggregations, "aggregations is null");
+    }
+
+    public CubeStatement(
+            Set<String> selection,
+            String from,
+            String where,
+            Set<String> groupBy,
+            List<AggregationSignature> aggregations)
+    {
+        this.selection = requireNonNull(selection, "selection is null");
+        this.from = requireNonNull(from, "from is null");
+        this.where = where;
         this.groupBy = requireNonNull(groupBy, "groupBy is null");
         this.aggregations = requireNonNull(aggregations, "aggregations is null");
     }
@@ -72,6 +89,11 @@ public class CubeStatement
         return groupBy;
     }
 
+    public Optional<String> getWhere()
+    {
+        return Optional.ofNullable(where);
+    }
+
     @Override
     public boolean equals(Object o)
     {
@@ -84,6 +106,7 @@ public class CubeStatement
         CubeStatement that = (CubeStatement) o;
         return Objects.equals(selection, that.selection) &&
                 Objects.equals(from, that.from) &&
+                Objects.equals(where, that.where) &&
                 Objects.equals(groupBy, that.groupBy) &&
                 Objects.equals(aggregations, that.aggregations);
     }
@@ -91,7 +114,7 @@ public class CubeStatement
     @Override
     public int hashCode()
     {
-        return Objects.hash(selection, from, groupBy, aggregations);
+        return Objects.hash(selection, from, where, groupBy, aggregations);
     }
 
     @Override
@@ -104,14 +127,21 @@ public class CubeStatement
         StringJoiner groupingColumns = new StringJoiner(", ");
         groupBy.forEach(groupingColumns::add);
 
+        StringBuilder whereBuilder = new StringBuilder();
+        if (where != null) {
+            whereBuilder.append(where.toString());
+        }
+
         return "SELECT " + columns +
                 " FROM " + from +
+                whereBuilder.toString() +
                 (groupBy.isEmpty() ? "" : " GROUP BY " + groupingColumns);
     }
 
     public static class Builder
     {
         private String from;
+        private String where;
         private final Set<String> groupBy = new HashSet<>();
         private final Set<String> selection = new HashSet<>();
         private final List<AggregationSignature> aggregations = new ArrayList<>();
@@ -140,15 +170,21 @@ public class CubeStatement
             return this;
         }
 
-        public Builder groupByAddString(String column)
+        public Builder where(String where)
         {
-            this.groupBy.add(column);
+            this.where = where;
             return this;
         }
 
-        public Builder groupByAddStringList(String... columns)
+        public Builder groupBy(String constraint)
         {
-            this.groupBy.addAll(Arrays.asList(columns));
+            this.groupBy.add(constraint);
+            return this;
+        }
+
+        public Builder groupBy(String... constraints)
+        {
+            this.groupBy.addAll(Arrays.asList(constraints));
             return this;
         }
 
@@ -157,7 +193,7 @@ public class CubeStatement
             if (this.aggregations.isEmpty() && this.selection.isEmpty()) {
                 throw new UnsupportedOperationException("Cannot construct a cube statement without selection and aggregation");
             }
-            return new CubeStatement(Collections.unmodifiableSet(selection), from, Collections.unmodifiableSet(groupBy), Collections.unmodifiableList(aggregations));
+            return new CubeStatement(Collections.unmodifiableSet(selection), from, where, Collections.unmodifiableSet(groupBy), Collections.unmodifiableList(aggregations));
         }
     }
 }

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatus.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-cube/src/main/java/io/hetu/core/spi/cube/aggregator/AggregationSignature.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-cube/src/main/java/io/hetu/core/spi/cube/io/CubeMetaStore.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -34,8 +34,8 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
-                .groupByAddString("address")
-                .groupByAddStringList("name", "nationkey")
+                .groupBy("address")
+                .groupBy("name", "nationkey")
                 .build();
 
         assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");
@@ -51,18 +51,21 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
+                .where("nationkey = 123")
                 .build();
 
         CubeStatement statement2 = CubeStatement.newBuilder()
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
+                .where("nationkey = 123")
                 .build();
 
         CubeStatement statement3 = CubeStatement.newBuilder()
                 .select("name", "address")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
+                .where("nationkey = 123")
                 .build();
 
         assertEquals(statement1, statement2, "statements are not equal");

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.2.1-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,7 +55,6 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
-    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConnector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConnectorFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterMetadata.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -242,7 +242,7 @@ public class DataCenterMetadata
     }
 
     @Override
-    public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, boolean includeColumnStatistics)
+    public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint)
     {
         Map<String, ColumnHandle> columnHandles = getColumnHandles(session, tableHandle);
         String tableFullName = tableHandle.getSchemaPrefixedTableName();

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterModule.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -56,11 +56,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this(catalogName,
-                requireNonNull(schemaName, "schemaName is null"),
-                requireNonNull(tableName, "tableName is null"),
-                requireNonNull(limit, "limit is null"),
-                "");
+        this.catalogName = catalogName;
+        this.schemaName = requireNonNull(schemaName, "schemaName is null");
+        this.tableName = requireNonNull(tableName, "tableName is null");
+        this.limit = requireNonNull(limit, "limit is null");
+        this.pushDownSql = "";
     }
 
     /**
@@ -125,7 +125,6 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
-    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -179,7 +179,7 @@ public class DataCenterPlanOptimizer
             List<RowExpression> pushable = new ArrayList<>();
             List<RowExpression> nonPushable = new ArrayList<>();
 
-            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
+            for (RowExpression conjunct : logicalRowExpressions.extractConjuncts(node.getPredicate())) {
                 try {
                     conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
                     pushable.add(conjunct);

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterQueryGenerator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -116,20 +116,6 @@ public class DataCenterQueryGenerator
                     .setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
                     .setSelections(selections)
                     .setFrom(Optional.of(table.toString()));
-
-            String catalogName = dcTableHandle.getCatalogName();
-            if (catalogName != null) {
-                contextBuilder.setRemoteCatalogName(catalogName);
-            }
-            String schemaName = dcTableHandle.getSchemaName();
-            if (schemaName != null) {
-                contextBuilder.setRemoteSchemaName(schemaName);
-            }
-            String tableName = dcTableHandle.getTableName();
-            if (tableName != null) {
-                contextBuilder.setRemoteTablename(tableName);
-            }
-
             // If LIMIT has been push down, add it to context
             if (dcTableHandle.getLimit().isPresent()) {
                 contextBuilder.setLimit(dcTableHandle.getLimit());

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/pagesource/DataCenterPageSource.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -102,10 +102,8 @@ public class DataCenterPageSource
     @Override
     public Page getNextPage()
     {
-        if (dynamicFilterSupplier.isPresent() && !dynamicFilterSupplier.get().getDynamicFilters().isEmpty()) {
-            /*  applying only for the first map in the dynamic filter since we do not have
-                more than one element as we do not expect disjuncts in this connector   */
-            applyDynamicFilters(dynamicFilterSupplier.get().getDynamicFilters().get(0));
+        if (dynamicFilterSupplier.isPresent()) {
+            applyDynamicFilters(dynamicFilterSupplier.get().getDynamicFilters());
         }
 
         if (!this.pages.isEmpty()) {

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/pagesource/DataCenterPageSourceProvider.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -1392,24 +1392,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner distributedQueryRunner = null;
+        DistributedQueryRunner queryRunner = null;
         try {
-            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            distributedQueryRunner.installPlugin(new DataCenterPlugin());
-            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            distributedQueryRunner.installPlugin(new TpchPlugin());
-            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
+            queryRunner.installPlugin(new DataCenterPlugin());
+            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            queryRunner.installPlugin(new TpchPlugin());
+            queryRunner.createCatalog("tpch", "tpch", properties);
 
-            return distributedQueryRunner;
+            return queryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, distributedQueryRunner);
+            closeAllSuppress(e, queryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterConfig.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterMetadata.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -104,6 +104,12 @@ public class TestDataCenterMetadata
             return 0;
         }
 
+        @Override
+        public boolean isLegacyTimestamp()
+        {
+            return false;
+        }
+
         @Override
         public <T> T getProperty(String name, Class<T> type)
         {

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
+openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,17 +24,15 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-The following is a detailed description of audit logging properties:
+Other audit logging properties include: 
 
-`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
+`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
 
-`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
+`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
 
 Example configuration file:
 
@@ -43,6 +41,7 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
+hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
+hetu.event.listener.audit.filecount=1
+hetu.event.listener.audit.limit=100000
 ```

hetu-docs/en/admin/extension-execution-planner.md

@@ -1,25 +0,0 @@
-#Extension Physical Execution Planner
-This section describes how to add an extension physical execution planner in openLooKeng. With the extension physical execution planner, openLooKeng can utilize other operator acceleration libraries to speed up the execution of SQL statements.
-
-##Configuration
-To enable extension physical execution feature, the following configs must be added in
-`config.properties`：
-
-``` properties
-extension_execution_planner_enabled=true
-extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
-extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
-```
-
-The above attributes are described below:
-
-- `extension_execution_planner_enabled`: Enable extension physical execution feature.
-- `extension_execution_planner_jar_path`: Set the file path of the extension physical execution jar package.
-- `extension_execution_planner_class_path`: Set the package path of extension physical execution generated class in jar。
-
-
-##Usage
-The below command can control the enablement of extension physical execution feature in WebUI or Cli while running openLooKeng:
-```
-set session extension_execution_planner_enabled=true/false
-```

hetu-docs/en/admin/meta-store.md

@@ -16,7 +16,6 @@ hetu.metastore.type=jdbc
 hetu.metastore.db.url=jdbc:mysql://....
 hetu.metastore.db.user=root
 hetu.metastore.db.password=123456
-hetu.metastore.cache.type=local
 ```
 
 The above properties are described below:
@@ -25,7 +24,6 @@ The above properties are described below:
 - `hetu.metastore.db.url`：URL of RDBMS to connect to.
 - `hetu.metastore.db.user` :User name of RDBMS to connect to. 
 - `hetu.metastore.db.password` :Password of RDBMS to connect to.
-- `hetu.metastore.cache.type` : Select the cache model, where local is the local cache and global is the distributed cache.
 
 ### HDFS Storage
 

hetu-docs/en/admin/multi-split-for-jdbc-data-source.md

@@ -1,44 +0,0 @@
-# JDBC Data Source Multi-Split Management
-
-## Overview
-
-This function applies to JDBC data sources. Data tables to be read are divided into multiple splits, and multiple worker nodes in the cluster simultaneously read the splits to accelerate data reading.
-
-## Properties
-
-Multi-split management is based on connectors. For a data table with this function enabled, add the following attributes to the configuration file of the connector to which the data table belong. For example, the configuration file corresponding to the **mysql** connector is **etc/mysql.properties**.
-
-Property list:
-
-Configure the properties as follows:
-
-```properties
-jdbc.table-split-enabled=true
-jdbc.table-split-stepCalc-refresh-interval=10s
-jdbc.table-split-stepCalc-threads=2
-jdbc.table-split-fields=[{"catalogName":"test_catalog", "schemaName":null, "tableName":"test_table", "splitField":"id","dataReadOnly":"true", "calcStepEnable":"false", "splitCount":"5","fieldMinValue":"1","fieldMaxValue":"10000"},{"catalogName":"test_catalog1", "schemaName":"test_schema1", "tableName":"test_tabl1", "splitField":"id", "dataReadOnly":"false", "calcStepEnable":"true", "splitCount":"5", "fieldMinValue":"","fieldMaxValue":""}]
-```
-
-Descriptions of the properties:
-
-- `jdbc.table-split-enabled`: whether to enable the multi-split data read function. The default value is **false**.
-- `jdbc.table-split-stepCalc-refresh-interval`: interval for dynamically updating splits. The default value is 5 minutes.
-- `jdbc.table-split-stepCalc-threads`: number of threads for dynamically updating splits. The default value is **4**.
-- `jdbc.table-split-fields`: split configuration of each data table. For details, see section "Split Configuration".
-
-### Split Configuration
-
-The configuration of each data table consists of multiple sub-properties, which are set in the JSON format. The description is as follows:
-
-> | Sub-property| Description| Suggestion|
-> |----------|----------|----------|
-> | `catalogName`| Name of the catalog to which the data table belongs in the data source, which corresponds to the value of the **TABLE\_CAT** field returned by the standard JDBC API **DatabaseMetaData.getTables**.| Set this sub-property to the actual value. If the value is empty, set it to **null**. |
-> | `schemaName`| Name of the schema to which the data table belongs in the data source, which corresponds to the value of the **TABLE\_SCHEM** field returned by the standard JDBC API **DatabaseMetaData.getTables**.| Set this sub-property to the actual value. If the value is empty, set it to **null**. |
-> | `tableName`| Name of the data table in the data source, which corresponds to the value of the **TABLE\_NAME** field returned by the standard JDBC API **DatabaseMetaData.getTables**.| Set this sub-property based on the actual value. |
-> | `splitField`| Column name of the split | Select a column whose value is an integer. You are advised to select a column with fewer duplicate values to divide the column into even splits.|
-> | `calcStepEnable`| Whether to dynamically adjust the split range| Set this sub-property to **true** for data tables with data changes. |
-> | `dataReadOnly`| Whether the data table is read-only| Set this sub-property to **true** for read-only data tables. |
-> | `splitCount`| Number of concurrent reads of data splits| Set this sub-property based on the optimal value. |
-> | `fieldMinValue`| Minimum value of the **splitField** field| Set this sub-property for read-only data tables based on the query result. Otherwise, leave this sub-property empty or set it to **null**. |
-> | `fieldMaxValue`| Maximum value of the **splitField** field| Set this sub-property for read-only data tables based on the query result. Otherwise, leave this sub-property empty or set it to **null**. |
-

hetu-docs/en/admin/properties.md

@@ -118,20 +118,6 @@ This section describes the most important config properties that may be used to
 >
 > This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
 
-### `query.suspend-query-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables running query temporary suspension when system is in low resource situation.
-
-### `query.max-suspended-queries`
-
-> -   **Type:** `integer`
-> -   **Default value:** `10`
->
-> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
-
 ## Spilling Properties
 
 ### `experimental.spill-enabled`
@@ -175,30 +161,6 @@ This section describes the most important config properties that may be used to
 >
 > This config property can be overridden by the `spill_window_operator` session property.
 
-
-### `experimental.spill-build-for-outer-join-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spill feature for right-outer and full-outer join operations.
->
->
->
-> This config property can be overridden by the `spill_build_for_outer_join_enabled` session property.
-
-### `experimental.inner-join-spill-filter-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables bloom filter based build-side spill matching for probe side spill decision.
->
->
->
-> This config property can be overridden by the `inner_join_spill_filter_enabled` session property.
-
-
 ### `experimental.spill-reuse-tablescan`
 
 > -   **Type:** `boolean`
@@ -217,7 +179,7 @@ This section describes the most important config properties that may be used to
 >
 > Directory where spilled content will be written. It can be a comma separated list to spill simultaneously to multiple directories, which helps to utilize multiple drives installed in the system.
 >
-> When `experimental.spiller-spill-to-hdfs` is to `true`, `experimental.spiller-spill-path` must contain only  a single directory.
+>
 >
 > It is not recommended to spill to system drives. Most importantly, do not spill to the drive on which the JVM logs are written, as disk overutilization might cause JVM to pause for lengthy periods, causing queries to fail.
 
@@ -278,71 +240,6 @@ This section describes the most important config properties that may be used to
 >
 > Enables using a randomly generated secret key (per spill file) to encrypt and decrypt data spilled to disk
 
-### `experimental.spill-direct-serde-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables to serialize/read the page directly to/from the stream.
-
-### `experimental.spill-prefetch-read-pages`
-
-> -   **Type:** `integer`
-> -   **Default value:** `1`
->
-> Sets number of pages prefetched while reading from spilled files.
-
-
-### `experimental.spill-use-kryo-serialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for spill to disk, instead of default java serializer.
-
-
-### `experimental.revocable-memory-selection-threshold`
-
-> -   **Type:** `data size`
-> -   **Default value:** `512 MB`
->
-> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke.
-
-### `experimental.prioritize-larger-spilts-memory-revoke`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `true`
->
-> Enables to prioritize splits with larger revocable memory.
-
-### `experimental.spill-non-blocking-orderby`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables order by operator to use asynchronous mechanism to spill, i.e it can accumulate input even when a spill is in progress and initiate a secondary spill when the secondary data accumulate exceeds a threshold or when the primary spill is completed, the default value of the threshold is the minimum between 20MB and 5% of available free memory. This property must be used in conjunction with the `experimental.spill-enabled` property.
->
->
->
-> This config property can be overridden by the `spill_non_blocking_orderby` session property.
-
-### `experimental.spiller-spill-to-hdfs`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spilling into HDFS. When this property is set to `true` the property `experimental.spiller-spill-profile` must be set and also `experimental.spiller-spill-path` must contain only a single path.
-
-### `experimental.spiller-spill-profile`
-
-> -   **Type:** `string`
-> -   **No default value.** Must be set when spilling to hdfs is enabled
->
->
-> This property defines the [filesystem](../develop/filesystem.md) profile used to spill. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `experimental.spiller-spill-profile=spill-hdfs`, a profile describing this filesystem `spill-hdfs.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable, refer [filesystem](../develop/filesystem.md) for details).
->
-> This property is required when `experimental.spiller-spill-to-hdfs` is set to `true`. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the path declared in `experimental.spiller-spill-path` folder in the specified file system.
-
 ## Exchange Properties
 
 Exchanges transfer data between openLooKeng nodes for different stages of a query. Adjusting these properties may help to resolve inter-node communication issues or improve network utilization.
@@ -378,6 +275,8 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
 >
+>
+>
 > Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
 
 ### `sink.max-buffer-size`
@@ -387,113 +286,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
 
-## Failure Recovery handling Properties
-
-### Failure Retry Policies
-
-### `failure.recovery.retry.profile`
-
-> -   **Type:** `String`
-> -   **Default value:** `default`
->
-> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
-> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
-> The file `test.properties` must contain `failure.recovery.retry.type` specified.
-
-
-### `failure.recovery.retry.type`
-
-> -   **Type:** `String`
-> -   **Default value:** `timeout`
->
-> The failure detection mechanism in use. Default is timeout based failure detection.
->
-#### `timeout` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
-> Additional properties `max.error.duration` can be defined for this type of failure detection.
->
-#### `max-retry` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
-> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection. 
-> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
-
-### `max.error.duration`
-> -   **Type:** `duration`
-> -   **Default value:** `300s`
->
-> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
-
-
-### `max.retry.count`
-
-> -   **Type:** `integer`
-> -   **Default value:** `100`
->
-> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. 
-> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster. 
-> This property is used only for `max-retry` based failure detection profiles. 
-> The minimum value for this parameter is 100.
-
-### Gossip Protocol Configurations for Failure Detection
-
-### `failure-detection-protocol`
-
->- **Type:** String
->- **Default value:** `heartbeat`
->
-> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
-> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
-> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
-
-### `failure-detector.heartbeat-interval`
-
->- **Type:** Duration
->- **Default value:** `500ms` (500 miliseconds)
->
-> This is the interval of gossip between two nodes in the cluster.
-> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
-> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
-> In workers, this property can be left to use the default value.
->
-### `failure-detector.worker-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.coordinator-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
->
-### `failure-detector.coordinator-gossip-collate-interval`
->
-> - **Type:** Duration
->- **Default value:** `2s` (2 seconds)
->
-> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
-> This property has to be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.gossip-group-size`
->
-> - **Type:** Integer
->- **Default value:** `Integer.MAX_VALUE`
->
-> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
-> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
-> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
-> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
-
 ## Task Properties
 
 ### `task.concurrency`
@@ -695,8 +487,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Use Reuse Exchange to cache data in memory if the query contains tables or Common Table Expressions(CTE) which are present more than one time with the same projections and filters on them. Enabling this feature will reduce the time taken to execute the query by caching data in memory and avoiding reading from disk multiple times.
 > This can also be specified on a per-query basis using the `reuse_table_scan` session property.
->
-> Note: when `cte_reuse_enabled` or `optimizer.cte-reuse-enabled` is enabled reuse exchange will be disabled.
 
 ### `optimizer.cte-reuse-enabled`
 
@@ -707,25 +497,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 > This will help to improve query execution performance when same CTE is used more than once.
 > This can also be specified on a per-query basis using the `cte_reuse_enabled` session property.
 
-### `optimizer.sort-based-aggregation-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Sort based aggregation is used when underlying source is in pre-sorted order, this is used instead of Hash aggregation which take more footprint to build hash tables.
-> Sort based aggregation used less memory foot print when compared to hash aggregation.
-> Conditions when Sort based aggregation in case of Hive
-> - 1) Grouping columns should be same or less than sorted columns and it should be in the same order.
-> - 2) Joins case probe side table should be sorted and join criteria should be same or less than sorted columns and it should be in the same order.
-> - 3) bucket_count is 1 bucketed_by columns should be same or less than Grouping columns and it should be in the same order.
-> - 4) bucket_count is more than 1 bucketed_by columns should be same as Grouping columns and it should be in the same order.
-> - 5) In case of partition table, Grouping columns should contain all partitions in same order following by the subset of sorted by columns.
-> - 6) When distinct is used, Grouping columns followed by a distinct column should be subset of sorted by columns.
->
-> This can also be specified on a per-query basis using the `sort_based_aggregation_enabled` session property.
->
-> **Note:** This is supported only for Hive connector.
-
 ## Regular Expression Function Properties
 
 The following properties allow tuning the [regexp](../functions/regexp.md).
@@ -878,7 +649,7 @@ helps with cache affinity scheduling.
 > Auto-Vacuum enables the system to automatically manage vacuum jobs by constantly monitoring the tables which needs vacuum in order to maintain optimal performance.
 > Engine gets the tables from data sources that are eligible for vacuum and trigger vacuum operation for those tables.
 
-### `auto-vacuum.enabled`
+### `auto-vacuum.enabled:`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
@@ -927,118 +698,39 @@ helps with cache affinity scheduling.
 >
 > **Note:** This should be configured in all workers.
 
-## Sort Base aggregation Properties
-
-### `sort.prcnt-drivers-for-partial-aggr`
-
-> -   **Type:** `int`
-> -   **Default value:** `5`
->
-> In Sort based aggregation percentage of number of drivers that are used for unfinalized/partial values.
-> This can also be specified on a per-query basis using the `prcnt_drivers_for_partial_aggr` session property.
->
-> **Note:** This should be configured on all nodes .
-
-
-## Query Manager
-
-### `query.remote-task.max-error-duration`
-
-> -   **Type:** `duration`
-> -   **Default value:** `5m`
->
-> The maximum time coordinator waits for remote-task related error to be resolved before it's considered a failure.
->
-> Note:
-> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
-
-## Query Recovery
-
-### `recovery_enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
+## Distributed Snapshot
 
 ### `snapshot_enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
 >
-> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
+> This session property is used to enable or disable the distributed snapshot functionality.
 
 ### `hetu.experimental.snapshot.profile`
 
 > -   **Type:** `string`
 >
-> This property defines the [filesystem](../develop/filesystem.md) profile used to stored snapshots. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `hetu.experimental.snapshot.profile=snapshot-hdfs1`, a profile describing this filesystem `snapshot-hdfs1.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable). Please refer to the [filesystem](../develop/filesystem.md) section for details.
+> This property defines the file system profile used to stored snapshots. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `hetu.experimental.snapshot.profile=snapshot-hdfs1`, a profile describing this filesystem `snapshot-hdfs1.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable).
 >
 > This property is required if any query is executed with distributed snapshot turned on. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the `/tmp/hetu/snapshot` folder in the specified file system.
 >
 > This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
 
-### `hetu.recovery.maxRetries`
+### `hetu.snapshot.maxRetries`
 
 > -   **Type:** `int`
 > -   **Default value:** `10`
 >
-> This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
+> This property defines the maxinum number of error recovery attempts for a query. When the limit is reached, the query fails.
 >
-> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
+> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
 
-### `hetu.recovery.retryTimeout`
+### `hetu.snapshot.retryTimeout`
 
 > -   **Type:** `duration`
 > -   **Default value:** `10m` (10 minutes)
 >
-> This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
+> This property defines the maxinum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
 >
-> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
-
-### `hetu.snapshot.useKryoSerialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for snapshot, instead of default java serializer.
-
-### `experimental.eliminate-duplicate-spill-files`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables elimination of duplicate spill files storage as part of snapshot capture.
-
-
-
-## HTTP Client Configurations
-
-### `http.client.idle-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `30s` (30 seconds)
->
-> This property defines the time for which a given http client shall stay connected without any operations performed over it.
-> After the specified time elapse with no activity, then the client connection is closed and related resources are released.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-### `http.client.request-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `10s` (10 seconds)
->
-> This property defines the time threshold for a given http client for which response should be received.
-> After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-## Connector Properties configuration
-
-### `case-insensitive-name-matching`
->
-> -    **Type:** `boolean`
-> -    **Default value:** `false`
->
-> Case-insensitive matching between database and collection names. The default is case sensitive.
+> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.

hetu-docs/en/admin/reliable-execution.md

@@ -11,33 +11,34 @@ To achieve better performance while maintaining execution reliability, the *dist
 
 As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
 
-## Enable Recovery framework
+## Enable Distributed Snapshot
 
-Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
+Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
 
 ## Requirements
 
 To be able to resume execution from a previously saved snapshot, there must be a sufficient number of workers available so that all previous tasks can be restored. To enable distributed snapshot for a query, the following is required:
 - at least 2 workers
-- at least 50% more available cluster-wide memory resources with tolerance for worker node failure. If constrained by memory, not all queries may be able to recover (see [I44RMW](https://e.gitee.com/open_lookeng/issues/list?issue=I44RMW))
-- at least 80% (rounded down) of previously available workers still active for the resume to be successful. If not enough workers are available, the query will not be able to
-  resume from any previous snapshot, so the query reruns from the beginning.
+- at least 80% of previously available nodes still active for the resume to be successful. If not enough workers are available, the query reruns from the beginning.
 
 ## Limitations
 
 - **Supported Statements**: only `INSERT` and `CREATE TABLE AS SELECT` types of statements are supported
    - This does *not* include statements like `INSERT INTO CUBE`
-- **Source tables**: can only read from tables in `Hive` catalog
+- **Source tables**: can only read from tables in `Hive`, `TPCDS`, and `TPCH` catalogs
 - **Target table**: can only write to tables in `Hive` catalogs, with `ORC` format
 - **Interaction with other features**: distributed snapshot does not yet work with the following features:
    - Reuse exchange, i.e. `optimizer.reuse-table-scan`
    - Reuse common table expression (CTE), i.e. `optimizer.cte-reuse-enabled`
+   - Spill, i.e. `experimental.spill-enabled`
 
 When a query that does not meet the above requirements is submitted with distributed snapshot enabled, the query will be executed as if the distributed snapshot feature is _not_ turned on.
 
 ## Detection
 
-Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
+Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the `query.remote-task.max-error-duration` configuration.
+
+Another relevant configuration is `exchange.max-error-duration`, which affects inter-task communication errors. It is recommended that this property is configured with a duration longer than `query.remote-task.max-error-duration`, to increase the chance of worker failure recovery.
 
 ## Storage Considerations
 
@@ -47,7 +48,7 @@ Snapshot data is stored in a file system as specified using the `hetu.experiment
 
 Snapshot files are stored under `/tmp/hetu/snapshot/` folder of the file system. All workers must be authorized to read and write to this folder.
 
-Snapshots reflect states in query execution, potentially becoming very large in size and varying significantly from query to query. For example, queries that need to buffer large amounts of data (typically involving ordering, window, join, aggregation, etc. operations), may result in snapshots that include data from an entire table. Ensure that the cluster has enough memory to process the snapshots and the shared file system has sufficient disk space available to store these snapshots before proceeding.
+Snapshots reflect states in query execution, potentially becoming very large in size and varying significantly from query to query. For example, queries that need to buffer large amounts of data (typically involving ordering, window, join, aggregation, etc. operations), may result in snapshots that include data from an entire table. Ensure that the shared file system has sufficient space available to save these snapshots before proceeding.
 
 Each query execution may produce multiple snapshots. Contents of these snapshots may overlap. Currently they are stored as separate files. In the future, "incremental snapshots" feature may be introduced to save storage space.
 
@@ -55,20 +56,8 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
 
 The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
 
-It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
-
-## Snapshot statistics
-
-Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
-
-Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query.  These statistics are displayed for all snapshots and for last snapshot separately.
-
-Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots.  Restore statistics are displayed only when there is restore(recovery) happened during the query.
-
-Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details 
-
-![](../images/snapshot_statistics.png)
+It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
 
 ## Configurations
 
-Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).
+Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).

hetu-docs/en/admin/resource-groups.md

@@ -13,14 +13,6 @@ resource-groups.config-file=etc/resource-groups.json
 
 Change the value of `resource-groups.config-file` to point to a JSON config file, which can be an absolute path, or a path relative to the openLooKeng data directory.
 
-### Additional Configuration
-
-In addition to above properties in `etc/resource-groups.properties`, below two properties can be configured, which gets used along with kill policy (details of same as part of killPolicy)
-
-`resource-groups.memory-margin-percent` (optional)- This is the allowed percentage of memory variation between two queries considered to be same. In this case queries does not get ordered based on memory usage instead they get ordered based on query execution progress provided query progress difference is more than configured. Default value is 10%.
-
-`resource-groups.query-progress-margin-percent`(optional)- This is the allowed percentage of query execution progress between two queries considered to be same. In this query does not get ordered based on query execution progress. Default value is 5%.
-
 ## Resource Group Properties
 
 
@@ -30,22 +22,14 @@ In addition to above properties in `etc/resource-groups.properties`, below two p
 -   `softMemoryLimit` (required): maximum amount of distributed memory this group may use before new queries become queued. May be specified as an absolute value (i.e. `1GB`) or as a percentage (i.e. `10%`) of the cluster\'s memory.
 -   `softCpuLimit` (optional): maximum amount of CPU time this group may use in a period (see `cpuQuotaPeriod`) before a penalty will be applied to the maximum number of running queries. `hardCpuLimit` must also be specified.
 -   `hardCpuLimit` (optional): maximum amount of CPU time this group may use in a period.
--   `schedulingPolicy` (optional): specifies how queued queries are selected to run, and how sub-groups become eligible to start their queries. May be one of three values. When High Availability (muliple coordinators) mode is enabled, only `fair` scheduling policy will be supported:
+-   `schedulingPolicy` (optional): specifies how queued queries are selected to run, and how sub-groups become eligible to start their queries. May be one of three values:
     - `fair` (default): queued queries are processed first-in-first-out, and sub-groups must take turns starting new queries (if they have any queued).
     - `weighted_fair`: sub-groups are selected based on their `schedulingWeight` and the number of queries they are already running concurrently. The expected share of running queries for a sub-group is computed based on the weights for all currently eligible sub-groups. The sub-group with the least concurrency relative to its share is selected to start the next query.
-    - `weighted`: queued queries are selected stochastically in proportion to their priority (specified via the `query_priority` [session property](../sql/set-session.md)). Sub groups are selected to start new queries in proportion to their `schedulingWeight`.
+- `weighted`: queued queries are selected stochastically in proportion to their priority (specified via the `query_priority` [session property](../sql/set-session.md)). Sub groups are selected to start new queries in proportion to their `schedulingWeight`.
     - `query_priority`: all sub-groups must also be configured with `query_priority`. Queued queries will be selected strictly according to their priority.
 -   `schedulingWeight` (optional): weight of this sub-group. See above. Defaults to `1`.
 -   `jmxExport` (optional): If true, group statistics are exported to JMX for monitoring. Defaults to `false`.
 -   `subGroups` (optional): list of sub-groups.
--   `killPolicy`(optional): Specifies how running queries are selected to kill If overall memory usage exceed **softMemoryLimit** after queries being submitted to worker. 
-
-    - `no_kill` (default): Queries will not be killed.
-    - `recent_queries`: This means queries in the reverse order of execution will be killed. 
-    - `oldest_queries`: Queries in the order of execution will get killed.
-    - `high_memory_queries` : Queries in the order of memory usage will be killed. Query having higher memory usage will get killed first so that with minimum number of query kill, more memory gets freed.
-    As part of this policy, we try to balance memory usage and percentage completion. So if two queries memory usage are within 10% (configurable as resource-groups.memory-margin-percent) of limit, then we pick the query which has progressed (% of execution) less. Incase these two queries difference in terms of percentage of completion are within 5% (configurable as resource-groups.query-progress-margin-percent) then we chose based on memory itself.
-    - `finish_percentage_queries`:  Query in the order of percentage of query execution will be killed. Query with least percentage of execution will be killed first.
 
 ## Selector Rules
 
@@ -82,25 +66,6 @@ Client tags can be set as follows:
 - CLI: use the `--client-tags` option.
 - JDBC: set the `ClientTags` client info property on the `Connection` instance.
 
-## Throttling and Kill Queries
-
-It may happen that once query got submitted to worker, then memory limit exceeded, in that case we need to handle these running queries using below mechanism:
-
-- Throttle Queries
-- Kill Queries
-
-### Throttle Queries
-
-Throttling of new split schedule is done to stop further increase in memory at worker. If memory usage by current query resource group has already exceeded **softReservedMemory** then further new splits will not get scheduled till memory usage comes below **softReservedMemory**.
-
-**softReservedMemory** is recommended to configure lesser than **softMemoryLimit**.
-
-User can also chose to **disable throttling** by omitting **softReservedMemory** configuration for a group.
-
-### Kill Queries
-
-If query could not get throttle and memory usage exceed softMemoryLimit, then query will be killed (made to fail) as per the kill policy configured. Only queries running from leaf group are considered to kill.
-
 ## Example
 
 

hetu-docs/en/admin/spill.md

@@ -12,7 +12,7 @@ Properties related to spilling are described in `tuning-spilling`.
 
 ## Memory Management and Spill
 
-By default, openLooKeng kills queries if the memory requested by the query execution exceeds session properties `query_max_memory` or `query_max_memory_per_node`. This mechanism ensures fairness in allocation of memory to queries and prevents deadlock caused by memory allocation. It is efficient when there are lots of small queries in the cluster, but leads to killing large queries that don\'t stay within the limits.
+By default, openLooKeng kills queries if the memory requested by the query execution exceeds session properties `query_max_memory` or `query_max_memory_per_node`. This mechanism ensures fairness in allocation of memory to queries and prevents deadlock caused by memory allocation. It is efficient when there is a lot of small queries in the cluster, but leads to killing large queries that don\'t stay within the limits.
 
 To overcome this inefficiency, the concept of revocable memory was introduced. A query can request memory that does not count toward the limits, but this memory can be revoked by the memory manager at any time. When memory is revoked, the query runner spills intermediate data from memory to disk and continues to process it later.
 
@@ -34,10 +34,6 @@ saturation of the configured spill paths.
 
 openLooKeng treats spill paths as independent disks (see [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD)), so there is no need to use RAID for spill.
 
-
-## Spill To HDFS
-Spilling directly into HDFS is also possible for that `experimental.spiller-spill-to-hdfs` needs to be set to `true`, `experimental.spiller-spill-profile` needs to be set and `spiller-spill-path` must contain only a single directory when we intend to spill into HDFS. (refer `experimental.spiller-spill-to-hdfs` and  `experimental.spiller-spill-profile` properties for more details )
-
 ## Spill Compression
 
 
@@ -65,21 +61,18 @@ When the build table is partitioned, the spill-to-disk mechanism can decrease th
 
 With this mechanism, the peak memory used by the join operator can be decreased to the size of the largest build table partition. Assuming no data skew, this will be `1 / task.concurrency` times the size of the whole build table.
 
-Note: spill-to-disk is not supported for Cross Join.
-
 ### Aggregations
 
 Aggregation functions perform an operation on a group of values and return one value. If the number of groups you\'re aggregating over is large, a significant amount of memory may be needed. When spill-to-disk
-is enabled, if there is not enough memory, intermediate accumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
+is enabled, if there is not enough memory, intermediate cumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
 
 ### Order By
 
-If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
-Generally when a spill is in progress the operator is blocked from taking inputs, but when `experimental.spill-non-blocking-orderby` is set to `true` order by uses asynchronous mechanism to spill (see`experimental.spill-non-blocking-orderby`).
+If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intemediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
 
 ### Window functions
 
-Window Functions perform an operators over a window of rows and return one value for each row. If this window of rows is large, a significant amount of memory may be needed. When spill to disk for window functions is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged when memory is available. There is a current limitation that spill will not work in all cases such as when a single window is very large.
+Window Functions perform an operators over a window of rows and return one value for each row. If this window of rows is large, a significant amount of memory may be needed. When spill to disk for window functions is enabled, if there is not enough memory, intemediate sorted results are written to disk. They are loaded back and merged when memory is available. There is a current limitation that spill will not work in all cases such as when a single window is very large.
 
 ### Reuse Exchange