hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.7.0-SNAPSHOT</version>
+    <version>1.4.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,7 +108,8 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this(metastore, null);
+            this.metastore = metastore;
+            this.schemaName = null;
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -232,6 +233,7 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
+        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,17 +73,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putShorts(inputRowId, count, value);
+            columnVector.putShorts(rowId, count, value);
         }
     }
 
@@ -98,17 +97,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putInts(inputRowId, count, value);
+            columnVector.putInts(rowId, count, value);
         }
     }
 
@@ -123,17 +121,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putLongs(inputRowId, count, value);
+            columnVector.putLongs(rowId, count, value);
         }
     }
 
@@ -148,12 +145,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -168,17 +164,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putDoubles(inputRowId, count, value);
+            columnVector.putDoubles(rowId, count, value);
         }
     }
 
@@ -201,12 +196,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -229,17 +223,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putNull(counter++);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putNulls(inputRowId, count);
+            columnVector.putNulls(rowId, count);
         }
     }
 
@@ -326,72 +319,66 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -127,16 +127,15 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        Path localOutPutPath = paramOutPutPath;
-        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
+        this.outPutPath = requireNonNull(outPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            localOutPutPath = new Path(properties.getProperty("location"));
+            outPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -212,7 +211,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -227,25 +226,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path finalOutPutPath = this.outPutPath;
-        Properties finalProperties = this.properties;
-        JobConf finalConfiguration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
+        Path outPutPath = this.outPutPath;
+        Properties properties = this.properties;
+        JobConf configuration = this.configuration;
+        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
-                        finalProperties, Reporter.NULL);
+                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
+                        properties, Reporter.NULL);
     }
 
     @Override
@@ -286,7 +285,7 @@ public class CarbondataFileWriter
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter finalRecordWriter = null;
+        FileSinkOperator.RecordWriter recordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -335,7 +334,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -352,7 +351,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            finalRecordWriter = this.recordWriter;
+            recordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -366,8 +365,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (finalRecordWriter != null) {
-                finalRecordWriter.write(serDe.serialize(row, tableInspector));
+            if (recordWriter != null) {
+                recordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,7 +48,6 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
-    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuFilterUtil.java

@@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
             if (rawData instanceof Slice) {
                 String value = ((Slice) rawData).toStringUtf8();
                 if (type.getTypeInfo() instanceof CharTypeInfo) {
-                    StringBuilder padding = new StringBuilder();
+                    String padding = "";
                     int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
                     int truncatedLength = value.length();
                     for (int i = 0; i < paddedLength - truncatedLength; i++) {
-                        padding.append(" ");
+                        padding += " ";
                     }
                     return value + padding;
                 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuOutputFormat.java

@@ -103,6 +103,7 @@ public class CarbondataHetuOutputFormat<T>
         OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
         JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
         carbonOutputCommitter.setupJob(jobContext);
+        CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
         org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
         return new FileSinkOperator.RecordWriter()
         {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataLocationService.java

@@ -80,6 +80,8 @@ public class CarbondataLocationService
     {
         // TODO: check and make it compatible for cloud scenario
 
+        HdfsEnvironment.HdfsContext context =
+                new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
         Path targetPath = new Path(table.getStorage().getLocation());
 
         return new LocationHandle(targetPath, targetPath, true,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java

@@ -306,13 +306,13 @@ public class CarbondataMetadata
 
     private void setupCommitWriter(Properties hiveSchema, Path outputPath, Configuration initialConfiguration, boolean isOverwrite) throws PrestoException
     {
-        CarbonLoadModel finalCarbonLoadModel;
+        CarbonLoadModel carbonLoadModel;
         TaskAttemptID taskAttemptID = TaskAttemptID.forName(initialConfiguration.get("mapred.task.id"));
         try {
             ThreadLocalSessionInfo.setConfigurationToCurrentThread(initialConfiguration);
-            finalCarbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
-            finalCarbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
-            CarbonTableOutputFormat.setLoadModel(initialConfiguration, finalCarbonLoadModel);
+            carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
+            carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
+            CarbonTableOutputFormat.setLoadModel(initialConfiguration, carbonLoadModel);
         }
         catch (IOException ex) {
             LOG.error("Error while creating carbon load model", ex);
@@ -360,13 +360,13 @@ public class CarbondataMetadata
         this.user = session.getUser();
         return hdfsEnvironment.doAs(user, () -> {
             SchemaTableName tableName = parent.getSchemaTableName();
-            Optional<Table> finalTable =
+            Optional<Table> table =
                     metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-            if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+            if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
                 throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
             }
 
-            this.table = finalTable;
+            this.table = table;
             Path outputPath =
                     new Path(parent.getLocationHandle().getJsonSerializableTargetPath());
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -382,7 +382,7 @@ public class CarbondataMetadata
             }
 
             /* Create committer object */
-            setupCommitWriter(finalTable, outputPath, initialConfiguration, isOverwrite);
+            setupCommitWriter(table, outputPath, initialConfiguration, isOverwrite);
 
             return new CarbondataInsertTableHandle(parent.getSchemaName(),
                     parent.getTableName(),
@@ -416,13 +416,13 @@ public class CarbondataMetadata
         currentState = State.UPDATE;
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -430,8 +430,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -470,13 +470,13 @@ public class CarbondataMetadata
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         List<HiveColumnHandle> inputColumns = parent.getInputColumns().stream().filter(HiveColumnHandle::isRequired).collect(toList());
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -484,8 +484,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -643,7 +643,7 @@ public class CarbondataMetadata
 
         return hdfsEnvironment.doAs(session.getUser(), () -> {
             Properties hiveSchema = MetastoreUtil.getHiveSchema(this.table.get());
-            CarbonTable finalCarbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
+            CarbonTable carbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
                     carbondataVacuumTableHandle.getTableName(),
                     hiveSchema,
                     initialConfiguration);
@@ -705,7 +705,7 @@ public class CarbondataMetadata
                     SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath()));
                     String source;
                     for (String currPartitionName : partitionNames) {
-                        source = finalCarbonTable.getTablePath() + "/" + currPartitionName;
+                        source = carbonTable.getTablePath() + "/" + currPartitionName;
                         moveFromTempFolder(source + "/" + carbonLoadModel.getSegmentId() + "_" + timeStamp + ".tmp", source);
                     }
                     segmentFilesToBeUpdatedLatest.add(new Segment(carbonLoadModel.getSegmentId(), segmentFileName));
@@ -719,7 +719,7 @@ public class CarbondataMetadata
                 for (CarbondataSegmentInfoUtil segmentInfo : newMergedSegmentInfoUtilList) {
                     String mergedLoadNumber = segmentInfo.getDestinationSegment();
                     try {
-                        String segmentFileName = SegmentFileStore.writeSegmentFile(finalCarbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
+                        String segmentFileName = SegmentFileStore.writeSegmentFile(carbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
                     }
                     catch (IOException e) {
                         throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while merging segment files", e);
@@ -900,9 +900,9 @@ public class CarbondataMetadata
     private LocationHandle getCarbonDataTableCreationPath(ConnectorSession session, ConnectorTableMetadata tableMetadata, HiveWriteUtils.OpertionType opertionType) throws PrestoException
     {
         Path targetPath = null;
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String tableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         Optional<String> location = getCarbondataLocation(tableMetadata.getProperties());
         LocationHandle locationHandle;
         FileSystem fileSystem;
@@ -914,32 +914,32 @@ public class CarbondataMetadata
                     throw new PrestoException(NOT_SUPPORTED, format("Setting %s property is not allowed", LOCATION_PROPERTY));
                 }
                 /* if path not having prefix with filesystem type, than we will take fileSystem type from core-site.xml using below methods */
-                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, finalSchemaName), new Path(location.get()));
+                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, schemaName), new Path(location.get()));
                 targetLocation = fileSystem.getFileStatus(new Path(location.get())).getPath().toString();
-                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, finalSchemaName, tableName), targetLocation, false);
+                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, schemaName, tableName), targetLocation, false);
             }
             else {
-                updateEmptyCarbondataTableStorePath(session, finalSchemaName);
+                updateEmptyCarbondataTableStorePath(session, schemaName);
                 targetLocation = carbondataTableStore;
-                targetLocation = targetLocation + File.separator + finalSchemaName + File.separator + tableName;
+                targetLocation = targetLocation + File.separator + schemaName + File.separator + tableName;
                 targetPath = new Path(targetLocation);
             }
         }
         catch (IllegalArgumentException | IOException e) {
             throw new PrestoException(NOT_SUPPORTED, format("Error %s store path %s ", e.getMessage(), targetLocation));
         }
-        locationHandle = locationService.forNewTable(metastore, session, finalSchemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
+        locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
         return locationHandle;
     }
 
     @Override
     public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
     {
-        SchemaTableName localSchemaTableName = tableMetadata.getTable();
-        String localSchemaName = localSchemaTableName.getSchemaName();
-        String tableName = localSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = localSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE;
         List<String> partitionedBy = new ArrayList<String>();
         List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@@ -947,29 +947,29 @@ public class CarbondataMetadata
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
 
-        metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
         Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
 
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                        new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
+                        new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                         new Path(locationHandle.getJsonSerializableTargetPath())));
 
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             try {
                 Map<String, String> serdeParameters = initSerDeProperties(tableName);
-                Table localTable = buildTableObject(
+                Table table = buildTableObject(
                         session.getQueryId(),
-                        localSchemaName,
+                        schemaName,
                         tableName,
                         session.getUser(),
                         columnHandles,
@@ -981,11 +981,11 @@ public class CarbondataMetadata
                         true, // carbon table is set as external table
                         prestoVersion,
                         serdeParameters);
-                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
-                HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
+                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
+                HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
                 metastore.createTable(
                         session,
-                        localTable,
+                        table,
                         principalPrivileges,
                         Optional.empty(),
                         ignoreExisting,
@@ -1092,8 +1092,8 @@ public class CarbondataMetadata
     public CarbondataTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
     {
         requireNonNull(tableName, "tableName is null");
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent()) {
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!table.isPresent()) {
             return null;
         }
 
@@ -1102,14 +1102,14 @@ public class CarbondataMetadata
             throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, "Unexpected table present in Hive metastore: " + tableName);
         }
 
-        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(finalTable.get()), finalTable.get().getParameters());
+        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(table.get()), table.get().getParameters());
 
         return new CarbondataTableHandle(
                 tableName.getSchemaName(),
                 tableName.getTableName(),
-                finalTable.get().getParameters(),
-                getPartitionKeyColumnHandles(finalTable.get()),
-                HiveBucketing.getHiveBucketHandle(finalTable.get()));
+                table.get().getParameters(),
+                getPartitionKeyColumnHandles(table.get()),
+                HiveBucketing.getHiveBucketHandle(table.get()));
     }
 
     private Optional<ConnectorOutputMetadata> finishUpdateAndDelete(ConnectorSession session,
@@ -1133,12 +1133,12 @@ public class CarbondataMetadata
 
         hdfsEnvironment.doAs(user, () -> {
             if (blockUpdateDetailsList.size() > 0) {
-                CarbonTable finalCarbonTable = getCarbonTable(tableHandle.getSchemaName(),
+                CarbonTable carbonTable = getCarbonTable(tableHandle.getSchemaName(),
                         tableHandle.getTableName(),
                         MetastoreUtil.getHiveSchema(table.get()),
                         initialConfiguration);
 
-                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(finalCarbonTable);
+                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(carbonTable);
                 SegmentUpdateDetails[] segementDetailsList = statusManager.getUpdateStatusDetails();
                 for (SegmentUpdateDetails segementDetails : segementDetailsList) {
                     segementDetails.getDeletedRowsInBlock();
@@ -1179,26 +1179,26 @@ public class CarbondataMetadata
                                             List<HiveColumnHandle> columnHandles,
                                             Map<String, String> tableProperties)
     {
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
 
         partitionedBy.addAll(CarbondataTableProperties.getPartitionedBy(tableMetadata.getProperties()));
         sortBy.addAll(CarbondataTableProperties.getSortedBy(tableMetadata.getProperties()));
         Optional<HiveBucketProperty> bucketProperty = Optional.empty();
         columnHandles.addAll(getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator));
-        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName)));
+        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, schemaName, tableName)));
     }
 
     @Override
     public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
     {
         // get the root directory for the database
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
+        String schemaName = schemaTableName.getSchemaName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = finalSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE_AS;
 
         List<String> partitionedBy = new ArrayList<String>();
@@ -1206,7 +1206,7 @@ public class CarbondataMetadata
         List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
-        metastore.getDatabase(finalSchemaName).orElseThrow(() -> new SchemaNotFoundException(finalSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         // to avoid type mismatch between HiveStorageFormat & Carbondata StorageFormat this hack no option
         HiveStorageFormat tableStorageFormat = HiveStorageFormat.valueOf("CARBON");
@@ -1222,29 +1222,29 @@ public class CarbondataMetadata
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE_AS);
         Path targetPath = locationService.getTableWriteInfo(locationHandle, false).getTargetPath();
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(finalSchemaName, finalTableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
 
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                                new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName),
+                                new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                                 new Path(locationHandle.getJsonSerializableTargetPath())));
             // Create Carbondata metadata folder and Schema file
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             Path outputPath = new Path(locationHandle.getJsonSerializableTargetPath());
-            Properties schema = readSchemaForCarbon(finalSchemaName, finalTableName, targetPath, columnHandles, partitionColumns);
+            Properties schema = readSchemaForCarbon(schemaName, tableName, targetPath, columnHandles, partitionColumns);
             // Create committer object
             setupCommitWriter(schema, outputPath, initialConfiguration, false);
         });
         try {
             CarbondataOutputTableHandle result = new CarbondataOutputTableHandle(
-                    finalSchemaName,
-                    finalTableName,
+                    schemaName,
+                    tableName,
                     columnHandles,
-                    metastore.generatePageSinkMetadata(new HiveIdentity(session), finalSchemaTableName),
+                    metastore.generatePageSinkMetadata(new HiveIdentity(session), schemaTableName),
                     locationHandle,
                     tableStorageFormat,
                     partitionStorageFormat,
@@ -1255,7 +1255,7 @@ public class CarbondataMetadata
                     EncodedLoadModel, jobContext.getConfiguration().get(LOAD_MODEL)));
 
             LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
-            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), finalSchemaTableName);
+            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
             return result;
         }
         catch (RuntimeException ex) {
@@ -1386,7 +1386,7 @@ public class CarbondataMetadata
         List<Segment> segmentFilesToBeUpdated = blockUpdateDetailsList.stream()
                 .map(SegmentUpdateDetails::getSegmentName)
                 .map(Segment::new).collect(Collectors.toList());
-        List<Segment> finalSegmentFilesToBeUpdatedLatest = new ArrayList<>();
+        List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
         List<Segment> segmentFilesToBeDeleted = blockUpdateDetailsList.stream()
                 .filter(segmentUpdateDetails -> segmentUpdateDetails.getSegmentStatus() != null &&
                         segmentUpdateDetails.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE))
@@ -1396,12 +1396,12 @@ public class CarbondataMetadata
         for (Segment segment : segmentFilesToBeUpdated) {
             String file =
                     SegmentFileStore.writeSegmentFile(carbonTable, segment.getSegmentNo(), timeStamp.toString());
-            finalSegmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
+            segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
         }
         if (!(updateSegmentStatusSuccess &&
                 CarbonUpdateUtil.updateTableMetadataStatus(new HashSet<>(segmentFilesToBeUpdated),
                         carbonTable, timeStamp.toString(), true, segmentFilesToBeDeleted,
-                        finalSegmentFilesToBeUpdatedLatest, ""))) {
+                        segmentFilesToBeUpdatedLatest, ""))) {
             CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timeStamp.toString());
         }
     }
@@ -1463,10 +1463,11 @@ public class CarbondataMetadata
                 Properties hiveschema = MetastoreUtil.getHiveSchema(table);
                 Configuration configuration = jobContext.getConfiguration();
                 configuration.set(SET_OVERWRITE, "false");
-                CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
-                LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
-                loadModel.setSegmentId(loadMetadataDetails.getLoadName());
-                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
+                CarbonLoadModel carbonLoadModel =
+                        HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
+                LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
+                carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
             }
             catch (IOException e) {
                 LOG.error("Error occurred while committing the insert job.", e);
@@ -1553,14 +1554,14 @@ public class CarbondataMetadata
         try {
             hdfsEnvironment.doAs(session.getUser(), () -> {
                 metastore.dropTable(session, handle.getSchemaName(), handle.getTableName());
-                Configuration finalInitialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
+                Configuration initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
                         .getConfiguration(new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
                                 handle.getTableName()), new Path(this.tableStorageLocation.get())));
 
                 Properties schema = MetastoreUtil.getHiveSchema(target.get());
                 schema.setProperty("tablePath", this.tableStorageLocation.get());
                 this.carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(),
-                        schema, finalInitialConfiguration);
+                        schema, initialConfiguration);
                 takeLocks(State.DROP_TABLE);
                 AbsoluteTableIdentifier identifier = this.carbonTable.getAbsoluteTableIdentifier();
                 if (SegmentStatusManager.isLoadInProgressInTable(carbonTable)) {
@@ -1569,7 +1570,7 @@ public class CarbondataMetadata
                 try {
                     //Simultaneous case after acquiring locks we should check table exist.
                     //if table is not there clean the lock folders
-                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, finalInitialConfiguration);
+                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, initialConfiguration);
                 }//CarbonFileException
                 catch (RuntimeException e) {
                     try {
@@ -1866,8 +1867,8 @@ public class CarbondataMetadata
     {
         String tableName = absoluteTableIdentifier.getTableName();
         String databaseName = absoluteTableIdentifier.getDatabaseName();
-        TableInfo finalTableInfo = carbonTable.getTableInfo();
-        List<SchemaEvolutionEntry> evolutionEntryList = finalTableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
+        TableInfo tableInfo = carbonTable.getTableInfo();
+        List<SchemaEvolutionEntry> evolutionEntryList = tableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
         Long updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).getTimeStamp();
         LOG.info("Reverting changes for " + databaseName + "." + tableName);
         List<ColumnSchema> addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getAdded();
@@ -1879,7 +1880,7 @@ public class CarbondataMetadata
                     break;
                 }
                 case DROP_COLUMN: {
-                    finalTableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
+                    tableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
                         if (cols.isInvisible() && removedCols.getColumnUniqueId().equals(cols.getColumnUniqueId())) {
                             cols.setInvisible(false);
                         }
@@ -2026,38 +2027,38 @@ public class CarbondataMetadata
     @Override
     protected ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName)
     {
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent() || finalTable.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
             throw new TableNotFoundException(tableName);
         }
 
-        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(finalTable.get(), typeManager);
+        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
         ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             columns.add(metadataGetter.apply(columnHandle));
         }
 
         // External location property
         ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
-        properties.put(LOCATION_PROPERTY, finalTable.get().getStorage().getLocation());
+        properties.put(LOCATION_PROPERTY, table.get().getStorage().getLocation());
 
         // Storage format property
         properties.put(HiveTableProperties.STORAGE_FORMAT_PROPERTY, CarbondataStorageFormat.CARBON);
 
         // Partitioning property
-        List<String> partitionedBy = finalTable.get().getPartitionColumns().stream()
+        List<String> partitionedBy = table.get().getPartitionColumns().stream()
                 .map(Column::getName)
                 .collect(toList());
         if (!partitionedBy.isEmpty()) {
             properties.put(HiveTableProperties.PARTITIONED_BY_PROPERTY, partitionedBy);
         }
 
-        Optional<String> comment = Optional.ofNullable(finalTable.get().getParameters().get(TABLE_COMMENT));
+        Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));
 
         // add partitioned columns into immutableColumns
         ImmutableList.Builder<ColumnMetadata> immutableColumns = ImmutableList.builder();
 
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             if (columnHandle.getColumnType().equals(HiveColumnHandle.ColumnType.PARTITION_KEY)) {
                 immutableColumns.add(metadataGetter.apply(columnHandle));
             }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -191,7 +191,7 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
+        SemiTransactionalHiveMetastore metastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
@@ -200,7 +200,7 @@ public class CarbondataMetadataFactory
                         this.hiveTransactionHeartbeatInterval,
                         this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
 
-        return new CarbondataMetadata(semiTransactionalHiveMetastore,
+        return new CarbondataMetadata(metastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
                 this.writesToNonManagedTablesEnabled,
@@ -212,8 +212,8 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
+                new MetastoreHiveStatisticsProvider(metastore, statsCache, samplePartitionCache),
+                this.accessControlMetadataFactory.create(metastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        finalHdfsEnvironment.doAs(session.getUser(), () -> {
+        hdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,7 +163,6 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
-    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int columnBatchSize = 0;
+        int batchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    columnBatchSize = columnarBatch.numRows();
-                    if (columnBatchSize == 0) {
+                    batchSize = columnarBatch.numRows();
+                    if (batchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(columnBatchSize, blocks);
+            Page page = new Page(batchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,17 +90,16 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
-        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            finalName = name.substring(0, name.length() - 4).trim();
+            name = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            finalName = name.substring(0, name.length() - 5).trim();
+            name = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(finalName, order);
+        return new SortingColumn(name, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -133,7 +133,6 @@ public class CarbondataWriterFactory
     {
     }
 
-    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,15 +143,14 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDecimal(inputRowId, value, precision);
+                columnVector.putDecimal(rowId, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -186,9 +185,8 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(inputRowId++, value);
+            columnVector.putByteArray(rowId++, value);
         }
     }
 
@@ -305,90 +303,84 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putFloat(inputRowId, src[i]);
+                columnVector.putFloat(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putShort(inputRowId, src[i]);
+                columnVector.putShort(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putInt(inputRowId, src[i]);
+                columnVector.putInt(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putLong(inputRowId, src[i]);
+                columnVector.putLong(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDouble(inputRowId, src[i]);
+                columnVector.putDouble(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
-                columnVector.putNull(inputRowId);
+            if (nullBitSet.get(rowId)) {
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putByte(inputRowId, src[i]);
+                columnVector.putByte(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,9 +58,8 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIdx++] == 1);
+            type.writeBoolean(builder, src[srcIndex++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,9 +76,8 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(id++, value, precision);
+            putDecimal(rowId++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,9 +58,8 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(id++, value);
+            putInt(rowId++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java

@@ -15,10 +15,12 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
+import com.esotericsoftware.minlog.Log;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import io.prestosql.hive.$internal.au.com.bytecode.opencsv.CSVReader;
 import io.prestosql.spi.PrestoException;
+import io.prestosql.spi.StandardErrorCode;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -51,6 +53,7 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.text.DateFormat;
@@ -64,8 +67,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
+import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
@@ -1424,7 +1429,7 @@ public class TestCarbonAllDataType
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtabledrop", false), false);
 
         } catch (IOException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1469,7 +1474,7 @@ public class TestCarbonAllDataType
                     i++;
                 }
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
 
             // Step 3: convert format level TableInfo to code level TableInfo
@@ -1550,7 +1555,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1564,7 +1569,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1573,7 +1578,7 @@ public class TestCarbonAllDataType
                 return   "date '" + data + "'";
             }
             case "varchar":
-            {
+            {//'china'
                 return "'" + data + "'";
             }
             case "timestamp":
@@ -1587,7 +1592,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1598,7 +1603,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1609,7 +1614,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1620,7 +1625,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1632,10 +1637,9 @@ public class TestCarbonAllDataType
                 return dateString;
             }
             case "smallint":  {
+                // smallint '12'
                 return "smallint '" + data + "'";
             }
-            default:
-                break;
         }
         return data;
     }
@@ -1693,7 +1697,7 @@ public class TestCarbonAllDataType
             hetuServer.execute(inserData);
         }
         catch(Exception e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1732,7 +1736,7 @@ public class TestCarbonAllDataType
 
         }
         catch (IOException | InterruptedException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1781,15 +1785,14 @@ public class TestCarbonAllDataType
     */
     private boolean checkStatusFileForDeleteMarked(String tableName, int updateNumber, int segmentNumber) throws SQLException
     {
-        BufferedReader reader = null;
         try {
             File dir = new File(storePath + "/carbon.store/testdb/" + tableName + "/Metadata");
             File[] tableUpdateStatusFiles = dir.listFiles((d, name) -> name.startsWith("tableupdatestatus"));
             Arrays.sort(tableUpdateStatusFiles);
             Gson gson = new Gson();
-            reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
+            BufferedReader reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
             SegmentUpdateDetails[] segmentUpdateDetails = gson.fromJson(reader, SegmentUpdateDetails[].class);
-            File tableStatusFile = new File(dir.getCanonicalPath() + "/tablestatus");
+            File tableStatusFile = new File(dir.getAbsolutePath() + "/tablestatus");
             reader = new BufferedReader(new FileReader(tableStatusFile));
             LoadMetadataDetails loadMetadataDetails = gson.fromJson(reader, LoadMetadataDetails[].class)[segmentNumber];
             if ((segmentUpdateDetails[0].getSegmentStatus() != null && segmentUpdateDetails[0].getSegmentStatus().toString().equals("Marked for Delete")) &&
@@ -1800,16 +1803,6 @@ public class TestCarbonAllDataType
             hetuServer.execute("drop table if exists testdb." + tableName);
             Assert.fail("Failed to read status files");
         }
-        finally {
-            if (reader != null) {
-                try {
-                    reader.close();
-                }
-                catch (IOException e) {
-                    logger.error(e.getMessage());
-                }
-            }
-        }
         return false;
     }
 
@@ -1829,7 +1822,7 @@ public class TestCarbonAllDataType
                     "/carbon.store/testdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
@@ -1853,7 +1846,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
@@ -1886,7 +1879,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException | InterruptedException e) {
             hetuServer.execute("DROP TABLE if exists testdb.myectable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.myectable");
@@ -1911,7 +1904,7 @@ public class TestCarbonAllDataType
                 FileFactory.mkdirs( storePath + "/carbon.store/mytestDb");
             }
         } catch (IOException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         String location = "'" + "file:///" + storePath + "/carbon.store/mytestDb" + "')" ;
@@ -2494,44 +2487,91 @@ public class TestCarbonAllDataType
     @Test
     public void test_writer_count() throws SQLException
     {
+        hetuServer.execute("drop table if exists testdb.testorders1");
+        hetuServer.execute("drop table if exists testdb.testorders_bak");
         hetuServer.execute("set session task_writer_count=32");
+        hetuServer.execute("set session implicit_conversion=true");
+        hetuServer.execute("CREATE TABLE testdb.testorders1(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
 
-        hetuServer.execute("CREATE TABLE testdb.testWriterCount_32(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("CREATE TABLE testdb.testorders_bak(orderkey bigint, orderstatus varchar(7), totalprice double, orderdate date)");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
 
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 6);
+        hetuServer.execute("set session task_writer_count=1");
+        List<Map<String, Object>>  actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
 
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 12);
-
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 24);
-
-        String filePath = storePath + "/carbon.store/testdb/testwritercount_32";
-        try {
-            hetuServer.execute("VACUUM TABLE testdb.testWriterCount_32 AND WAIT");
-            assertTrue(FileFactory.isFileExist(filePath + "/Fact/Part0/Segment_0.1"));
-        }
-        catch (IOException e) {
-            assertTrue(false, "Unable to read file from table path");
-        }
-        finally {
-            hetuServer.execute("drop table if exists testdb.testWriterCount_32");
-            hetuServer.execute("set session task_writer_count=1");
-        }
-    }
-
-    private void verifyRowCount(String tableName, int rowCount) throws SQLException
-    {
-        String query = String.format("SELECT COUNT(*) AS result FROM %s", tableName);
-        List<Map<String, Object>>  actualResult = hetuServer.executeQuery(query);
         List<Map<String, Object>>  expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("result", rowCount); }});
+            add(new HashMap<String, Object>() {{    put("RESULT", 3); }});
         }};
         Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        // checking correct number of files created in Fact
+        File files1 = null;
+        files1 = new File(storePath + "/carbon.store/testdb/testorders_bak/Fact");
+        File[] fileListPart0 = files1.listFiles();//Part0
+
+        if (fileListPart0.length == 1) {
+            File[] Segment_0 = fileListPart0[0].listFiles();
+            if (Segment_0.length == 1) {
+                File[] fileListSegment_0 = Segment_0[0].listFiles();
+                if (fileListSegment_0.length == 6) {
+                    assertEquals("true", "true");
+                }
+                assertEquals(fileListSegment_0.length, 6);
+            }
+            assertEquals(Segment_0.length, 1);
+        }
+        assertEquals(fileListPart0.length, 1);
+
+        //test with 16
+        hetuServer.execute("set session task_writer_count=16");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 6); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        //test with 8
+        hetuServer.execute("set session task_writer_count=8");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 9); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        //test with 2
+        hetuServer.execute("set session task_writer_count=2");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 12); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        try {
+            hetuServer.execute("set session task_writer_count=32");
+            hetuServer.execute("VACUUM TABLE testdb.testorders_bak AND WAIT");
+            assertEquals(FileFactory.isFileExist(storePath +
+                    "/carbon.store/testdb/testorders_bak/Fact/Part0/Segment_0.1", false), true);
+        } catch (IOException e) {
+        }
+
+        hetuServer.execute("drop table if exists testdb.testorders1");
+        hetuServer.execute("drop table if exists testdb.testorders_bak");
+        hetuServer.execute("set session task_writer_count=1");
     }
 
     private TableInfo getTableInfoFromSchemaFile(String tableName)

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -40,7 +40,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
 
 public class TestCarbonAutoVacuum
 {
@@ -100,6 +99,7 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
+        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -145,12 +145,11 @@ public class TestCarbonAutoVacuum
 
         try {
             CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
-            assertNotNull(catalog);
             connector = catalog.getConnector(catalog.getConnectorCatalogName());
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -216,7 +215,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -62,6 +62,7 @@ public class TestCarbondataAutoCleanup
     public void setup() throws Exception
     {
         logger.info("Setup begin: " + this.getClass().getSimpleName());
+        String dataPath = rootPath + "/src/test/resources/alldatatype.csv";
 
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
@@ -130,7 +131,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -159,7 +160,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -189,7 +190,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -221,7 +222,7 @@ public class TestCarbondataAutoCleanup
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-                logger.debug(exception.getMessage());
+
             }
         }
         finally {
@@ -254,7 +255,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -285,7 +286,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -314,7 +315,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -344,7 +345,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -374,7 +375,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -403,7 +404,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -15,7 +15,6 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -37,7 +36,6 @@ import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
-    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -116,7 +114,7 @@ public class TestsWithHiveConnector
         assertEquals(FileFactory.isFileExist(storePath +
                 "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
         hetuServer.execute("DROP TABLE hive.default.parttable");
     }
@@ -143,7 +141,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
         hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
@@ -152,7 +150,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2014", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE hive.default.parttable2");
@@ -180,7 +178,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "/hive.store/default/parttable3/year=2013", false), true);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
         hetuServer.execute("DROP TABLE hive.default.parttable3");
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -123,20 +123,14 @@ public class HetuTestServer
     public List<Map<String, Object>> executeQuery(String query) throws SQLException
     {
         logger.info(">>>>> Executing Query: " + query);
-        ResultSet rs = null;
         try {
-            rs = statement.executeQuery(query);
+            ResultSet rs = statement.executeQuery(query);
             return convertResultSetToList(rs);
         }
         catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
-        finally {
-            if (rs != null) {
-                rs.close();
-            }
-        }
     }
 
     private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
@@ -192,7 +186,7 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
@@ -203,7 +197,7 @@ public class HetuTestServer
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
         }
         catch (RuntimeException e) {

hetu-clickhouse/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-clickhouse</artifactId>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -345,9 +345,8 @@ public class ClickHouseClient
     }
 
     @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
+    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
     {
-        String newColumnName = inputNewColumnName;
         try (Connection connection = connectionFactory.openConnection(identity)) {
             if (connection.getMetaData().storesUpperCaseIdentifiers()) {
                 newColumnName = newColumnName.toUpperCase(ENGLISH);

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -41,7 +41,6 @@ public class ClickHouseApplyRemoteFunctionPushDown
     /**
      * rewrite the remote function to a executable function in the data source.
      */
-    @Override
     public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
     {
         if (!isConnectorSupportedRemoteFunction(callExpression)) {

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -37,9 +37,8 @@ public class ClickHouseSqlStatementWriter
     }
 
     @Override
-    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
+    public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
     {
-        String functionName = inputFunctionName;
         if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
             functionName = "varPop";
         }

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -205,7 +205,7 @@ public final class ClickHouseServerTest
     {
         String actualTable = tablePattern;
 
-        for (String table : tables) {
+        for (String table : tables) { //tableName + _ + UUID
             int lastIndex = table.lastIndexOf("_");
             if (lastIndex == -1) {
                 continue;

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,6 +47,16 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
+
+        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
+        // as per link above, the default SSLContext will be constructed using the default KeyManager and
+        // default TrustManager. Those can be configured using the following system properties:
+        // javax.net.ssl.keyStore
+        // javax.net.ssl.keyStorePassword
+        // javax.net.ssl.keyStoreType
+        // javax.net.ssl.trustStore
+        // javax.net.ssl.trustStorePassword
+        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,7 +13,6 @@
  */
 package io.hetu.core.common.util;
 
-import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -30,8 +29,6 @@ import java.util.Optional;
 
 public class TrustStore
 {
-    private static final Logger LOGGER = Logger.get(TrustStore.class);
-
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -51,7 +48,6 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
-            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
+            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
+            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-cube</artifactId>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -36,7 +36,8 @@ public class CubeFilter
 
     public CubeFilter(String sourceTablePredicate)
     {
-        this(sourceTablePredicate, null);
+        this.sourceTablePredicate = sourceTablePredicate;
+        this.cubePredicate = null;
     }
 
     public String getSourceTablePredicate()

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -140,13 +140,13 @@ public class CubeStatement
             return this;
         }
 
-        public Builder groupByAddString(String column)
+        public Builder groupBy(String column)
         {
             this.groupBy.add(column);
             return this;
         }
 
-        public Builder groupByAddStringList(String... columns)
+        public Builder groupBy(String... columns)
         {
             this.groupBy.addAll(Arrays.asList(columns));
             return this;

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -34,8 +34,8 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
-                .groupByAddString("address")
-                .groupByAddStringList("name", "nationkey")
+                .groupBy("address")
+                .groupBy("name", "nationkey")
                 .build();
 
         assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,7 +55,6 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
-    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -56,11 +56,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this(catalogName,
-                requireNonNull(schemaName, "schemaName is null"),
-                requireNonNull(tableName, "tableName is null"),
-                requireNonNull(limit, "limit is null"),
-                "");
+        this.catalogName = catalogName;
+        this.schemaName = requireNonNull(schemaName, "schemaName is null");
+        this.tableName = requireNonNull(tableName, "tableName is null");
+        this.limit = requireNonNull(limit, "limit is null");
+        this.pushDownSql = "";
     }
 
     /**
@@ -125,7 +125,6 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
-    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -179,7 +179,7 @@ public class DataCenterPlanOptimizer
             List<RowExpression> pushable = new ArrayList<>();
             List<RowExpression> nonPushable = new ArrayList<>();
 
-            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
+            for (RowExpression conjunct : logicalRowExpressions.extractConjuncts(node.getPredicate())) {
                 try {
                     conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
                     pushable.add(conjunct);

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterQueryGenerator.java

@@ -116,20 +116,6 @@ public class DataCenterQueryGenerator
                     .setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
                     .setSelections(selections)
                     .setFrom(Optional.of(table.toString()));
-
-            String catalogName = dcTableHandle.getCatalogName();
-            if (catalogName != null) {
-                contextBuilder.setRemoteCatalogName(catalogName);
-            }
-            String schemaName = dcTableHandle.getSchemaName();
-            if (schemaName != null) {
-                contextBuilder.setRemoteSchemaName(schemaName);
-            }
-            String tableName = dcTableHandle.getTableName();
-            if (tableName != null) {
-                contextBuilder.setRemoteTablename(tableName);
-            }
-
             // If LIMIT has been push down, add it to context
             if (dcTableHandle.getLimit().isPresent()) {
                 contextBuilder.setLimit(dcTableHandle.getLimit());

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1392,24 +1392,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner distributedQueryRunner = null;
+        DistributedQueryRunner queryRunner = null;
         try {
-            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            distributedQueryRunner.installPlugin(new DataCenterPlugin());
-            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            distributedQueryRunner.installPlugin(new TpchPlugin());
-            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
+            queryRunner.installPlugin(new DataCenterPlugin());
+            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            queryRunner.installPlugin(new TpchPlugin());
+            queryRunner.createCatalog("tpch", "tpch", properties);
 
-            return distributedQueryRunner;
+            return queryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, distributedQueryRunner);
+            closeAllSuppress(e, queryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
+openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,17 +24,15 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-The following is a detailed description of audit logging properties:
+Other audit logging properties include: 
 
-`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
+`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
 
-`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
+`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
 
 Example configuration file:
 
@@ -43,6 +41,7 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
+hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
+hetu.event.listener.audit.filecount=1
+hetu.event.listener.audit.limit=100000
 ```

hetu-docs/en/admin/extension-execution-planner.md

@@ -1,25 +0,0 @@
-#Extension Physical Execution Planner
-This section describes how to add an extension physical execution planner in openLooKeng. With the extension physical execution planner, openLooKeng can utilize other operator acceleration libraries to speed up the execution of SQL statements.
-
-##Configuration
-To enable extension physical execution feature, the following configs must be added in
-`config.properties`：
-
-``` properties
-extension_execution_planner_enabled=true
-extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
-extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
-```
-
-The above attributes are described below:
-
-- `extension_execution_planner_enabled`: Enable extension physical execution feature.
-- `extension_execution_planner_jar_path`: Set the file path of the extension physical execution jar package.
-- `extension_execution_planner_class_path`: Set the package path of extension physical execution generated class in jar。
-
-
-##Usage
-The below command can control the enablement of extension physical execution feature in WebUI or Cli while running openLooKeng:
-```
-set session extension_execution_planner_enabled=true/false
-```

hetu-docs/en/admin/properties.md

@@ -118,20 +118,6 @@ This section describes the most important config properties that may be used to
 >
 > This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
 
-### `query.suspend-query-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables running query temporary suspension when system is in low resource situation.
-
-### `query.max-suspended-queries`
-
-> -   **Type:** `integer`
-> -   **Default value:** `10`
->
-> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
-
 ## Spilling Properties
 
 ### `experimental.spill-enabled`
@@ -175,30 +161,6 @@ This section describes the most important config properties that may be used to
 >
 > This config property can be overridden by the `spill_window_operator` session property.
 
-
-### `experimental.spill-build-for-outer-join-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spill feature for right-outer and full-outer join operations.
->
->
->
-> This config property can be overridden by the `spill_build_for_outer_join_enabled` session property.
-
-### `experimental.inner-join-spill-filter-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables bloom filter based build-side spill matching for probe side spill decision.
->
->
->
-> This config property can be overridden by the `inner_join_spill_filter_enabled` session property.
-
-
 ### `experimental.spill-reuse-tablescan`
 
 > -   **Type:** `boolean`
@@ -217,7 +179,7 @@ This section describes the most important config properties that may be used to
 >
 > Directory where spilled content will be written. It can be a comma separated list to spill simultaneously to multiple directories, which helps to utilize multiple drives installed in the system.
 >
-> When `experimental.spiller-spill-to-hdfs` is to `true`, `experimental.spiller-spill-path` must contain only  a single directory.
+>
 >
 > It is not recommended to spill to system drives. Most importantly, do not spill to the drive on which the JVM logs are written, as disk overutilization might cause JVM to pause for lengthy periods, causing queries to fail.
 
@@ -278,71 +240,6 @@ This section describes the most important config properties that may be used to
 >
 > Enables using a randomly generated secret key (per spill file) to encrypt and decrypt data spilled to disk
 
-### `experimental.spill-direct-serde-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables to serialize/read the page directly to/from the stream.
-
-### `experimental.spill-prefetch-read-pages`
-
-> -   **Type:** `integer`
-> -   **Default value:** `1`
->
-> Sets number of pages prefetched while reading from spilled files.
-
-
-### `experimental.spill-use-kryo-serialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for spill to disk, instead of default java serializer.
-
-
-### `experimental.revocable-memory-selection-threshold`
-
-> -   **Type:** `data size`
-> -   **Default value:** `512 MB`
->
-> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke.
-
-### `experimental.prioritize-larger-spilts-memory-revoke`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `true`
->
-> Enables to prioritize splits with larger revocable memory.
-
-### `experimental.spill-non-blocking-orderby`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables order by operator to use asynchronous mechanism to spill, i.e it can accumulate input even when a spill is in progress and initiate a secondary spill when the secondary data accumulate exceeds a threshold or when the primary spill is completed, the default value of the threshold is the minimum between 20MB and 5% of available free memory. This property must be used in conjunction with the `experimental.spill-enabled` property.
->
->
->
-> This config property can be overridden by the `spill_non_blocking_orderby` session property.
-
-### `experimental.spiller-spill-to-hdfs`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spilling into HDFS. When this property is set to `true` the property `experimental.spiller-spill-profile` must be set and also `experimental.spiller-spill-path` must contain only a single path.
-
-### `experimental.spiller-spill-profile`
-
-> -   **Type:** `string`
-> -   **No default value.** Must be set when spilling to hdfs is enabled
->
->
-> This property defines the [filesystem](../develop/filesystem.md) profile used to spill. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `experimental.spiller-spill-profile=spill-hdfs`, a profile describing this filesystem `spill-hdfs.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable, refer [filesystem](../develop/filesystem.md) for details).
->
-> This property is required when `experimental.spiller-spill-to-hdfs` is set to `true`. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the path declared in `experimental.spiller-spill-path` folder in the specified file system.
-
 ## Exchange Properties
 
 Exchanges transfer data between openLooKeng nodes for different stages of a query. Adjusting these properties may help to resolve inter-node communication issues or improve network utilization.
@@ -378,8 +275,19 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
 >
+>
+>
 > Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
 
+### `exchange.max-error-duration`
+
+> -   **Type:** `duration`
+> -   **Minimum value:** `1m`
+> -   **Default value:** `7m`
+>
+> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a failure.
+
+
 ### `sink.max-buffer-size`
 
 > -   **Type:** `data size`
@@ -387,113 +295,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
 
-## Failure Recovery handling Properties
-
-### Failure Retry Policies
-
-### `failure.recovery.retry.profile`
-
-> -   **Type:** `String`
-> -   **Default value:** `default`
->
-> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
-> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
-> The file `test.properties` must contain `failure.recovery.retry.type` specified.
-
-
-### `failure.recovery.retry.type`
-
-> -   **Type:** `String`
-> -   **Default value:** `timeout`
->
-> The failure detection mechanism in use. Default is timeout based failure detection.
->
-#### `timeout` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
-> Additional properties `max.error.duration` can be defined for this type of failure detection.
->
-#### `max-retry` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
-> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection. 
-> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
-
-### `max.error.duration`
-> -   **Type:** `duration`
-> -   **Default value:** `300s`
->
-> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
-
-
-### `max.retry.count`
-
-> -   **Type:** `integer`
-> -   **Default value:** `100`
->
-> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. 
-> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster. 
-> This property is used only for `max-retry` based failure detection profiles. 
-> The minimum value for this parameter is 100.
-
-### Gossip Protocol Configurations for Failure Detection
-
-### `failure-detection-protocol`
-
->- **Type:** String
->- **Default value:** `heartbeat`
->
-> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
-> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
-> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
-
-### `failure-detector.heartbeat-interval`
-
->- **Type:** Duration
->- **Default value:** `500ms` (500 miliseconds)
->
-> This is the interval of gossip between two nodes in the cluster.
-> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
-> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
-> In workers, this property can be left to use the default value.
->
-### `failure-detector.worker-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.coordinator-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
->
-### `failure-detector.coordinator-gossip-collate-interval`
->
-> - **Type:** Duration
->- **Default value:** `2s` (2 seconds)
->
-> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
-> This property has to be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.gossip-group-size`
->
-> - **Type:** Integer
->- **Default value:** `Integer.MAX_VALUE`
->
-> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
-> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
-> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
-> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
-
 ## Task Properties
 
 ### `task.concurrency`
@@ -878,7 +679,7 @@ helps with cache affinity scheduling.
 > Auto-Vacuum enables the system to automatically manage vacuum jobs by constantly monitoring the tables which needs vacuum in order to maintain optimal performance.
 > Engine gets the tables from data sources that are eligible for vacuum and trigger vacuum operation for those tables.
 
-### `auto-vacuum.enabled`
+### `auto-vacuum.enabled:`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
@@ -948,25 +749,15 @@ helps with cache affinity scheduling.
 > -   **Default value:** `5m`
 >
 > The maximum time coordinator waits for remote-task related error to be resolved before it's considered a failure.
->
-> Note:
-> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
 
-## Query Recovery
-
-### `recovery_enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
+## Distributed Snapshot
 
 ### `snapshot_enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
 >
-> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
+> This session property is used to enable or disable the distributed snapshot functionality.
 
 ### `hetu.experimental.snapshot.profile`
 
@@ -978,67 +769,20 @@ helps with cache affinity scheduling.
 >
 > This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
 
-### `hetu.recovery.maxRetries`
+### `hetu.snapshot.maxRetries`
 
 > -   **Type:** `int`
 > -   **Default value:** `10`
 >
 > This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
 >
-> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
+> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
 
-### `hetu.recovery.retryTimeout`
+### `hetu.snapshot.retryTimeout`
 
 > -   **Type:** `duration`
 > -   **Default value:** `10m` (10 minutes)
 >
 > This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
 >
-> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
-
-### `hetu.snapshot.useKryoSerialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for snapshot, instead of default java serializer.
-
-### `experimental.eliminate-duplicate-spill-files`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables elimination of duplicate spill files storage as part of snapshot capture.
-
-
-
-## HTTP Client Configurations
-
-### `http.client.idle-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `30s` (30 seconds)
->
-> This property defines the time for which a given http client shall stay connected without any operations performed over it.
-> After the specified time elapse with no activity, then the client connection is closed and related resources are released.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-### `http.client.request-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `10s` (10 seconds)
->
-> This property defines the time threshold for a given http client for which response should be received.
-> After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-## Connector Properties configuration
-
-### `case-insensitive-name-matching`
->
-> -    **Type:** `boolean`
-> -    **Default value:** `false`
->
-> Case-insensitive matching between database and collection names. The default is case sensitive.
+> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.

hetu-docs/en/admin/reliable-execution.md

@@ -11,9 +11,9 @@ To achieve better performance while maintaining execution reliability, the *dist
 
 As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
 
-## Enable Recovery framework
+## Enable Distributed Snapshot
 
-Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
+Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
 
 ## Requirements
 
@@ -37,7 +37,7 @@ When a query that does not meet the above requirements is submitted with distrib
 
 ## Detection
 
-Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
+Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration) configuration.
 
 ## Storage Considerations
 
@@ -55,20 +55,8 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
 
 The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
 
-It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
-
-## Snapshot statistics
-
-Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
-
-Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query.  These statistics are displayed for all snapshots and for last snapshot separately.
-
-Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots.  Restore statistics are displayed only when there is restore(recovery) happened during the query.
-
-Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details 
-
-![](../images/snapshot_statistics.png)
+It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
 
 ## Configurations
 
-Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).
+Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).

hetu-docs/en/admin/spill.md

@@ -34,10 +34,6 @@ saturation of the configured spill paths.
 
 openLooKeng treats spill paths as independent disks (see [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD)), so there is no need to use RAID for spill.
 
-
-## Spill To HDFS
-Spilling directly into HDFS is also possible for that `experimental.spiller-spill-to-hdfs` needs to be set to `true`, `experimental.spiller-spill-profile` needs to be set and `spiller-spill-path` must contain only a single directory when we intend to spill into HDFS. (refer `experimental.spiller-spill-to-hdfs` and  `experimental.spiller-spill-profile` properties for more details )
-
 ## Spill Compression
 
 
@@ -65,17 +61,14 @@ When the build table is partitioned, the spill-to-disk mechanism can decrease th
 
 With this mechanism, the peak memory used by the join operator can be decreased to the size of the largest build table partition. Assuming no data skew, this will be `1 / task.concurrency` times the size of the whole build table.
 
-Note: spill-to-disk is not supported for Cross Join.
-
 ### Aggregations
 
 Aggregation functions perform an operation on a group of values and return one value. If the number of groups you\'re aggregating over is large, a significant amount of memory may be needed. When spill-to-disk
-is enabled, if there is not enough memory, intermediate accumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
+is enabled, if there is not enough memory, intermediate cumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
 
 ### Order By
 
 If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
-Generally when a spill is in progress the operator is blocked from taking inputs, but when `experimental.spill-non-blocking-orderby` is set to `true` order by uses asynchronous mechanism to spill (see`experimental.spill-non-blocking-orderby`).
 
 ### Window functions
 

hetu-docs/en/admin/web-interface.md

@@ -33,54 +33,4 @@ and statistics about the query is available by clicking the *JSON* link. These v
 > -   **Allowed values:** `true`, `false`
 > -   **Default value:** `false`
 >
-> Insecure authentication over HTTP is disabled by default. This could be overridden via `hetu.queryeditor-ui.allow-insecure-over-http` property of `etc/config.properties` (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
-
-### `hetu.queryeditor-ui.execution-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `100 DAYS`
->
-> UI Execution timeout is set to 100 days as default. This could be overridden via `hetu.queryeditor-ui.execution-timeout` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.max-result-count`
-
-> - **Type:** `int`
-> - **Default value:** `1000`
->
-> UI max result count is set to 1000 as default. This could be overridden via `hetu.queryeditor-ui.max-result-count` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.max-result-size-mb`
-
->- **Type:** `size`
->- **Default value:** `1GB`
->
-> UI max result size is set to 1 GB as default. This could be overridden via `hetu.queryeditor-ui.max-result-size-mb` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.session-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `1 DAYS`
->
-> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
-
-### `hetu.queryhistory.max-count`
-
-> -   **Type:** `int`
-> -   **Default value:** `1000`
->
-> The maximum number of query history stored by openLooKeng. This could be overridden via "hetu.queryhistory.max-count" of "etc/config.properties".
-
-### `hetu.collectionsql.max-count`
-
-> -   **Type:** `int`
-> -   **Default value:** `100`
->
-> The Maximum number of SQL collected by each user. This could be overridden via "hetu.collectionsql.max-count" of "etc/config.properties".
-
-## Remarks
-
-The max length of the favorite SQL is 600 by default. You can modify it through the following steps:
-
-1. Login MySQL database according to the JDBC configuration of `hetu-metastore.properties`
-
-2. Select table hetu_favorite, execute script `alter table hetu_favorite modify query varchar(2000) not null;` to modify the max length of the favorite SQL.
+> Insecure authentication over HTTP is disabled by default. This could be overridden via "hetu.queryeditor-ui.allow-insecure-over-http" property of "etc/config.properties" (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).

hetu-docs/en/connector/Hudi.md

@@ -1,8 +1,5 @@
 # Hudi  Connector
 
-### Release Notes
-Currently Hudi only supports version 0.7.0.
-
 ### Hudi Introduction
 
 Apache Hudi is a fast growing data lake storage system that helps organizations build and manage petabyte-scale data lakes. Hudi enables storing vast amounts of data on top of existing DFS compatible storage while also enabling stream processing in addition to typical batch-processing. This is made possible by providing two new primitives. Specifically,

hetu-docs/en/connector/elasticsearch.md

@@ -192,4 +192,4 @@ The array fields of this structure can be defined by using the following command
     
 ## Limitations
 1. openLooKeng does not support to query table in Elasticsearch which has duplicated columns, such as column "name" and "NAME";
-2. openLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.
+2. opneLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.

hetu-docs/en/connector/hive.md

@@ -620,36 +620,6 @@ The following operations are not supported when `avro_schema_url` is set:
     columns are not supported in `CREATE TABLE`.
 -   `ALTER TABLE` commands modifying columns are not supported.
 
-## Drop Column Behavior
-
-Syntax supported for Drop Column is as follows:
-
-```sql
-ALTER TABLE 'name' DROP COLUMN 'column_name'
-```
-
-In case of Hive connector, DROP COLUMN drops column which is at the end of existing columns. Hive doesn't support DROP COLUMN syntax, however closest DDL supported by Hive which enumerates the above is REPLACE COLUMNS. REPLACE COLUMNS removes all existing columns and adds the new set of columns whereas DROP COLUMN does remove all existing columns and add the same set of columns excluding column specified in the query, manifesting as it dropped the column.
-
-For example, consider a table with columns **a**, **b** and **c**.
-
-```sql
-lk:default> SELECT * FROM hive_table;
- a | b  |  c
----+----+-----
- 1 | 10 | 100
-(1 row)
-```
-
- On dropping column **a**:
-
-```sql
-lk:default> SELECT * FROM hive_table;
-  b | c
----+----
- 1 | 10
-(1 row)
-```
-
 ## Procedures
 
 
@@ -746,7 +716,7 @@ Drop a schema:
 DROP SCHEMA hive.web
 ```
 
-## Metastore Cache
+## Metastore Cache:
 
 Hive connector maintains a metastore cache to service the metastore request faster to various operations. Loading, reloading and retention times of the cache entries can be configured in `hive.properties`.
 
@@ -770,7 +740,7 @@ REFRESH META CACHE
 Additionally, metadata cache refresh command can be used to reload the metastore cache by user.
 
 
-## Performance tuning notes
+## Performance tuning notes:
 
 #### INSERT
 

hetu-docs/en/connector/kafka-tutorial.md

@@ -565,4 +565,4 @@ lk:default> SELECT created_at, raw_date FROM (
 (5 rows)
 ```
 
-The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or milliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.
+The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or miilliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.

hetu-docs/en/connector/memory.md

@@ -37,7 +37,7 @@ memory.spill-path=/opt/hetu/data/spill
   hetu.metastore.cache.type=local
   ```
   ##### Multi-Node Setup
-  - This section will give an example configuration for Memory Connector and a cluster with more than one node.
+  - This section will give an example configuration for Memory Connector an a cluster with more than one node.
   - Create a file `etc/catalog/memory.properties` with the following information:
   ``` properties
   connector.name=memory
@@ -80,10 +80,10 @@ memory.spill-path=/opt/hetu/data/spill
 **Note:**
 - `spill-path` should be set to a directory with enough free space to hold
  the table data.
-- See [**Configuration Properties**](#configuration-properties) section for additional properties and
+- See **Configuration Properties** section for additional properties and
  details.
-- In `etc/config.properties` ensure that `task.writer-count` is set
- `>=` to number of nodes in the cluster running openLooKeng. This will help
+- In `etc/config.properties` ensure that `task.writer-count` is set to
+ `>=` number of nodes in the cluster running openLooKeng. This will help
   distribute the data uniformly between all the workers.
 
 Examples
@@ -112,45 +112,17 @@ Create a table using the Memory Connector with sorting, indices and spill compre
     CREATE TABLE memory.default.nation
     WITH (
         sorted_by=array['nationkey'],
-        partitioned_by=array['regionkey'],
-        index_columns=array['name'],
+        index_columns=array['name', 'regionkey'],
         spill_compression=true
     )
     AS SELECT * from tpch.tiny.nation;
 
 After table creation completes, the Memory Connector will start building indices and sorting data in the background. Once the processing is complete any queries using the sort or index columns will be faster and more efficient.
 
-For now, `sorted_by` and `partitioned_by` only accepts a single column.
+For now, `sorted_by` only accepts a single column.
 
-Memory and Disk Usage via JMX
------------------------------
-JMX can be used to show memory and disk usage of memory connector tables
-Please refer to [JMX Connector](./jmx.md) for setup
 
-The `io.prestosql.plugin.memory.data:name=MemoryTableManager` table of `jmx.current` contains information on all the tables' memory and disk usage size in bytes
-
-    SELECT * FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
-
-```
- currentbytes | alltablesdiskbyteusage | alltablesmemorybyteusage |   node   |                       object_name                       
---------------+------------------------+--------------------------+----------+---------------------------------------------------------
-           23 |                   3456 |                       23 | example1 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
-          253 |                   8713 |                      667 | example2 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
-```
-
-Not all tables will be in memory since some may have been spilled to disk. `currentbytes` column will show the current memory occupied by tables which are in memory now.
-
-The usage for each node is shown as a separate row, aggregation functions can be utilized to show total usage across the cluster. For example, to view total disk or memory usage on all nodes, run:
-
-    SELECT sum(alltablesdiskbyteusage) as totaldiskbyteusage, sum(alltablesmemorybyteusage) as totalmemorybyteusage FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
-
-```
-totaldiskbyteusage | totalmemorybyteusage
--------------------+---------------------
-             12169 |                  690
-```
-
-# Configuration Properties
+Configuration Properties
 ------------------------
 
 | Property Name                         | Default Value   | Required| Description               |
@@ -161,50 +133,31 @@ totaldiskbyteusage | totalmemorybyteusage
 | `memory.max-page-size                `  | 512KB         | No      | Memory limit for each page. Default value is recommended.|
 | `memory.logical-part-processing-delay`  | 5s            | No      | The delay between when the table is created/updated and LogicalPart processing starts. Default value is recommended.|
 | `memory.thread-pool-size             `  | Half of threads available to the JVM | No      | Maximum threads to allocate for background processing (e.g. sorting, index creation, cleanup, etc)|
-| `memory.table-statistics-enabled`       | False         | No      | When enabled, user can run analyze to collect statistics and leverage that information for accelerating queries.|
 
-Path whitelist:  `["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
+Path whitelist：`["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
 
 Additional WITH properties
---------------------------
+--------------
 Use these properties when creating a table with the Memory Connector to make queries faster.
 
 | Property Name            | Argument type             | Requirements                     | Description|
 |--------------------------|---------------------------|----------------------------------|------------|
 | sorted_by                | `array['col']`            | Maximum of one column. Column type must be comparable.  | Sort and create indexes on the given column|
-| partitioned_by           | `array['col']`            | Maximum of one column. | Partition the table on the given column|
 | index_columns            | `array['col1', 'col2']`   | None                             | Create indexes on the given column|
 | spill_compression        | `boolean`           | None                             | Compress data when spilling to disk|
 
 
 Index Types
 --------------
-These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`.
-If a query operator is not supported by a particular index, you can still use that operator, but the query will not benefit from the index.
+These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`. If a query operator is not supported by a particular index you can still
+use that operator, but the query will not benefit from the index.
 
 | Index ID                          |Built for Columns In | Supported query operators             |
 |-----------------------------------|----------------------------------------|---------------------------------------|
-| Bloom   | `index_columns`                                 | `=` `IN`                             |                   
+| Bloom   | `sorted_by,index_columns`                                 | `=` `IN`                             |                   
 | MinMax  | `sorted_by,index_columns`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 | Sparse  | `sorted_by`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 
-
-Using statistics
------------------
-If the statistic configuration is enabled, you can refer to the example below to use it.
-
-Create a table using the Memory Connector:
-
-    CREATE TABLE memory.default.nation AS
-    SELECT * from tpch.tiny.nation;
-
-Run Analyze to collect the statistic information:
-
-    ANALYZE memory.default.nation;
-
-And then run the queries. Note that currently we do not support automatic statistic update, so you will need to run ANALYZE again if the table is updated.
-
-
 Developer Information
 ----------------------------
 
@@ -213,11 +166,20 @@ This section outlines the overall design of the Memory Connector, as shown in th
 ![Memory Connector Overall Design](../images/memory-connector-design.png)
 
 ### Scheduling Process
-The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng. In the Memory Connector, each worker has several LogicalParts. During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion. Table data will be automatically spilled to disk as part of a background process as well.  If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule. HetuMetastore is used to persist table metadata. At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
+The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng.
+In the Memory Connector, each worker has a number of LogicalParts.
+During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion.
+Table data will be automatically spilled to disk as part of a background process as well. 
+If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule.
+HetuMetastore is used to persist table metadata.
+At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
 
 ### LogicalPart
-As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data. The sorting and indexing are handled in a background process allowing faster querying,
-but the table is still queriable during processing. LogicalParts have a maximum configurable size (default 256 MB).  New LogicalParts are created once the previous one is full.
+As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data.
+The sorting and indexing are handled in a background process allowing faster querying,
+but the table is still queriable during processing.
+LogicalParts have a maximum configurable size (default 256 MB). 
+New LogicalParts are created once the previous one is full.
 
 
 ### Indices
@@ -251,5 +213,4 @@ Limitations and known Issues
 
 - Without State Store and Hetu Metastore with global cache, after `DROP TABLE`, memory is not released immediately on the workers. It is released on the next `CREATE TABLE` operation.
 - Currently only a single column in ascending order is supported by `sorted_by`
-- If a CTAS (CREATE TABLE AS) query fails or is cancelled, an invalid table will remain. This table must be dropped manually.
-- And we support BOOLEAN, All INT Types, CHAR, VARCHAR, DOUBLE, REAL, DECIMAL, DATE, TIME, UUID types as partition keys.
+- If a CTAS (CREATE TABLE AS) query fails or is cancelled, an invalid table will remain. This table must be dropped manually.

hetu-docs/en/connector/omnidata.md

@@ -1,103 +0,0 @@
-
-OmniData Connector
-==============
-
-## Overview
-
-The OmniData connector allows querying data stored in the remote Hive data warehouse.
-It pushes the operators of openLooKeng down to the storage node to achieve near-data calculation, thereby reducing the amount of network transmission data and improving computing performance.
-
-For more information, please see: [OmniData](https://www.hikunpeng.com/en/developer/boostkit/big-data?accelerated=3) and [OmniData connector](https://github.com/kunpengcompute/omnidata-openlookeng-connector).
-
-## Supported File Types
-
-
-The following file types are supported for the OmniData connector:
-
--   ORC
--   Parquet
--   Text
-
-## Configuration
-
-Create `etc/catalog/omnidata.properties` with the following configurations, replacing `example.net:9083` with the correct host and port for your Hive metastore Thrift service:
-
-``` properties
-connector.name=omnidata-openlookeng
-hive.metastore.uri=thrift://example.net:9083
-```
-
-### HDFS Configuration
-
-For basic setups, openLooKeng configures the HDFS client automatically and does not require any configuration files. In some cases, such as when using federated HDFS or NameNode high availability, it is necessary to specify additional HDFS client options in order to access your HDFS cluster. To do so, add the `hive.config.resources` property to reference your HDFS config files:
-
-``` properties
-hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop/conf/hdfs-site.xml
-```
-
-Only specify additional configuration files if necessary for your setup. We also recommend reducing the configuration files to have the minimum set of required properties, as additional properties may cause problems.
-
-The configuration files must exist on all openLooKeng nodes. If you are referencing existing Hadoop config files, make sure to copy them to any openLooKeng nodes that are not running Hadoop.
-
-## OmniData Configuration Properties
-
-| Property Name                   | Description                                                  | Default |
-| ------------------------------- | ------------------------------------------------------------ | ------- |
-| hive.metastore                  | The type of Hive metastore                                   | thrift  |
-| hive.config.resources           | An optional comma-separated list of HDFS configuration files. These files must exist on the machines running openLooKeng. Only specify this if absolutely necessary to access HDFS. Example: `/etc/hdfs-site.xml` |         |
-| hive.omnidata-enabled           | Allows push-down operators to execute on the storage side. If disabled, all operators will not be pushed down. | true    |
-| hive.min-offload-row-number     | If the number of rows in the table is less than the threshold, all operators of the table will not be pushed down. | 500     |
-| hive.filter-offload-enabled     | Allows the filter operator to be pushed down to the storage side. If disabled, the filter operator will not be pushed down. | true    |
-| hive.filter-offload-factor      | Only when the selection rate of the filter operator is less than the threshold, it will be pushed down. | 0.25    |
-| hive.aggregator-offload-enabled | Allows the aggregator operator to be pushed down to the storage side. If disabled, the aggregator operator will not be pushed down. | true    |
-| hive.aggregator-offload-factor  | Only when the aggregation rate of the aggregator operator is less than the threshold, it will be pushed down. | 0.25    |
-
-For more configuration, please refer to the [Hive Configuration Properties](./hive.md#Hive Configuration Properties) chapter.
-
-### Querying OmniData
-
-The SQL query plan after some operators are pushed down:
-
-```sql
-lk:tpch_flat_orc_date_1000> explain select sum(l_extendedprice * l_discount) as revenue
-				 		 -> from
-				 		 -> lineitem
-				 		 -> where
-				 		 -> l_shipdate >= DATE '1993-01-01'
-				 		 -> and l_shipdate < DATE '1994-01-01'
-				 		 -> and l_discount between 0.06 - 0.01 and 0.06 + 0.01
-				 		 -> and l_quantity < 25;
-				 							Query Plan
-------------------------------------------------------------------------------------------------------
-Output[revenue]
-│ Layout: [sum:double]
-│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
-│ revenue := sum
-└─ Aggregate(FINAL)
-│ Layout: [sum:double]
-│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
-│ sum := sum(sum_4)
-└─ LocalExchange[SINGLE] ()
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
-└─ RemoteExchange[GATHER]
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
-└─ Aggregate(PARTIAL)
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 0B}
-│ sum_4 := sum(expr)
-└─ ScanProject[table = hive:tpch_flat_orc_date_1000:lineitem offload={ filter=[AND(AND(BETWEEN(l_discount, 0.05, 0.07), LESS_THAN(l_quantity, 25.0)), AND(GREATER_THAN_OR_EQUAL(l_shipdate, 8401), LESS_THAN(l_shipdate, 8766)))]} ]
- Layout: [expr:double]
- Estimates: {rows: 5999989709 (50.29GB), cpu: 100.58G, memory: 0B, network: 0B}/{rows: 5999989709 (50.29GB), cpu: 150.87G, memory: 0B, network: 0B}
- expr := (l_extendedprice) * (l_discount)
- l_extendedprice := l_extendedprice:double:5:REGULAR
- l_discount := l_discount:double:6:REGULAR
-```
-
-
-## OmniData Connector Limitations
-
-
-- The OmniData service needs to be deployed on the storage node.
-- Only the pushdown of Filter, Aggregator, and Limit operators are supported.

hetu-docs/en/connector/opengauss.md

@@ -45,95 +45,6 @@ Finally, you can access the `hetutb` table in the `public` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `opengauss` in the above examples.
 
-## openGauss Update/Delete Support
-
-### Create openGauss Table
-
-Example：
-
-```sql
-CREATE TABLE opengauss_table (
-    id int,
-    name varchar(255));
-```
-
-### INSERT on openGauss tables
-
-Example：
-
-```sql
-INSERT INTO opengauss_table
-  VALUES
-     (1, 'Jack'),
-     (2, 'Bob');
-```
-
-### UPDATE on openGauss tables
-
-Example：
-
-```sql
-UPDATE opengauss_table
-  SET name='Tim'
-  WHERE id=1;
-```
-
-Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
-
-SELECT result before UPDATE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
-id | name
-----+------
-  1 | Jack
-  2 | Bob
-(2 rows)
-```
-
-SELECT result after UPDATE
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
-id | name
-----+------
-2 | Bob
-1 | Tim
-(2 rows)
-```
-
-### DELETE on openGauss tables
-
-Example：
-
-```sql
-DELETE FROM opengauss_table
-  WHERE id=2;
-```
-
-Above example delete the rows with column `id` having value `2`.
-
-SELECT result before DELETE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
- id | name
-----+------
-  2 | Bob
-  1 | Tim
-(2 rows)
-```
-
-SELECT result after DELETE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
- id | name
-----+------
-  1 | Tim
-(1 row)
-```
-
 ****Note:****
 
 > - When the compatibility type of the openGuass database is O (DBCOMPATIBILITY = A), the `Date` data type is not supported.
@@ -153,4 +64,4 @@ openGauss Connector Limitations
 
 The following SQL statements are not yet supported:
 
-[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/postgresql.md

@@ -46,98 +46,9 @@ Finally, you can access the `clicks` table in the `web` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `postgresql` in the above examples.
 
-## PostgreSQL Update/Delete Support
-
-### Create PostgreSQL Table
-
-Example：
-
-```sql
-CREATE TABLE postgresql_table (
-    id int,
-    name varchar(255));
-```
-
-### INSERT on PostgreSQL tables
-
-Example：
-
-```sql
-INSERT INTO postgresql_table
-  VALUES
-     (1, 'Jack'),
-     (2, 'Bob');
-```
-
-### UPDATE on PostgreSQL tables
-
-Example：
-
-```sql
-UPDATE postgresql_table
-  SET name='Tim'
-  WHERE id=1;
-```
-
-Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
-
-SELECT result before UPDATE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
-id | name
-----+------
-  1 | Jack
-  2 | Bob
-(2 rows)
-```
-
-SELECT result after UPDATE
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
-id | name
-----+------
-2 | Bob
-1 | Tim
-(2 rows)
-```
-
-### DELETE on PostgreSQL tables
-
-Example：
-
-```sql
-DELETE FROM postgresql_table
-  WHERE id=2;
-```
-
-Above example delete the rows with column `id` having value `2`.
-
-SELECT result before DELETE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
- id | name
-----+------
-  2 | Bob
-  1 | Tim
-(2 rows)
-```
-
-SELECT result after DELETE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
- id | name
-----+------
-  1 | Tim
-(1 row)
-```
-
 PostgreSQL Connector Limitations
 --------------------------------
 
 The following SQL statements are not yet supported:
 
-[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/redis.md

@@ -1,173 +0,0 @@
-Redis Connector
-====================
-Overview
---------
-this connector allows the use of Redis key/value pair is presented as a single row in openLooKeng.
-
-**Note**
-
-*In Redis,key/value pair can only be mapped to string or hash value types.keys can be stored in a zset,then keys can split into multiple slice*
-
-*Support Redis 2.8.0 or higher*
-
-Configuration
--------------
-To configure the Redis connector, create a catalog properties file `etc/catalog/redis.properties` with the following contents, replacing the properties as appropriate:
-``` properties
-connector.name=redis
-redis.table-names=schema1.table1,schema1.table2
-redis.nodes=host1:port
-```
-### Multiple Redis Servers
-You can have as many catalogs as you need. If you have additional
-Redis servers, simply add another properties file to ``etc/catalog``
-with a different name, making sure it ends in ``.properties``.
-For example, if you name the property file `sales.properties`, openLooKeng will create a catalog named `sales` using the configured connector.
-
-Configuration properties
-------------------------
-The following configuration properties are available:
-
-| Property Name                                                    | Description                                                                                                           |
-|:-----------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
-| `redis.table-names`                                              | List of all tables provided by the catalog                                                                            |
-| `redis.default-schema`                                           | Default schema name for tables  (default `default`)                                                                   |
-| `redis.nodes`                                                    | List of nodes in the Redis server                                                                                     |
-| `redis.connect-timeout`                                          | Timeout for connecting to the  Redis server (ms) (default 2000)                                                       |
-| `redis.scan-count`                                               | The number of keys obtained from each scan for string and hash value types (default 100)                              |
-| `redis.key-prefix-schema-table`                                  | Redis keys have schema-name:table-name prefix   (default  false)                                                      |
-| `redis.key-delimiter`                                            | Delimiter separating schema_name and table_name if redis.key-prefix-schema-table is used (default `:`)                |
-| `redis.table-description-dir`                                    | Directory containing table description files (default `etc/redis/`)                                                   |
-| `redis.hide-internal-columns`                                    | Whether internal columns are shown in table metadata or not. (default true)                                           |
-| `redis.database-index`                                           | Redis database index  (default 0)                                                                                     |
-| `redis.password`                                                 | Redis server password  (default null)                                                                                 |
-| `redis.table-description-interval`                               | the interval of flush description files (ms) (default no flush,table description will be memoized without expiration) |
-
-
-Internal columns
-----------------
-
-| Column name        | Type    | Description                                                                                                                              |
-|:-------------------| :------ |:-----------------------------------------------------------------------------------------------------------------------------------------|
-| `_key`             | VARCHAR  | Redis key.                                                                                                                               |
-| `_value`           | VARCHAR   | Redis value corresponding to the key                                                                                                     |
-| `_key_length`   | BIGINT  | Number of bytes in the key.                                                                                                              |
-| `_key_corrupt`     | BOOLEAN  | True if the decoder could not decode the key for this row. When true, data columns mapped from the key should be treated as invalid.     |
-| `_value_corrupt`   | BOOLEAN | True if the decoder could not decode the value for this row. When true, data columns mapped from the value should be treated as invalid. |
-
-
-Table Definition Files
-----------------------
-For openLooKeng, every key/value pair  must be mapped into columns to allow queries against the data. It is like kafka conntector,so you can refer to kafka-tutorial
-
-A table definition file consists of a JSON definition for a table. The name of the file can be arbitrary but must end in `.json`.
-
-for example,there is a nation.json
-``` json
-{
-    "tableName": "nation",
-    "schemaName": "tpch",
-    "key": {
-        "dataFormat": "raw",
-        "fields": [
-            {
-                "name": "redis_key",
-                "type": "VARCHAR(64)",
-                "hidden": "true"
-            }
-        ]
-    },
-    "value": {
-        "dataFormat": "json",
-        "fields": [
-            {
-                "name": "nationkey",
-                "mapping": "nationkey",
-                "type": "BIGINT"
-            },
-            {
-                "name": "name",
-                "mapping": "name",
-                "type": "VARCHAR(25)"
-            },
-                        {
-                "name": "regionkey",
-                "mapping": "regionkey",
-                "type": "BIGINT"
-            },
-            {
-                "name": "comment",
-                "mapping": "comment",
-                "type": "VARCHAR(152)"
-            }
-       ]
-    }
-}
-```
-In redis,such data exists
-```shell
-127.0.0.1:6379> keys tpch:nation:*
- 1) "tpch:nation:2"
- 2) "tpch:nation:4"
- 3) "tpch:nation:16"
- 4) "tpch:nation:18"
- 5) "tpch:nation:10"
- 6) "tpch:nation:17"
- 7) "tpch:nation:1"
-```
-```shell
-127.0.0.1:6379> get tpch:nation:1
-"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
-```
-Now we can use redis connector get data from redis,(redis_key don't show,because we set "hidden": "true" ) 
-```shell
-lk> select * from redis.tpch.nation;
- nationkey |      name      | regionkey |                                                      comment                                                       
------------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
-         3 | CANADA         |         1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold              
-         9 | INDONESIA      |         2 |  slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 
-        19 | ROMANIA        |         3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account    
-         2 | BRAZIL         |         1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special   
-```
-**Note**
-
-*if redis.key-prefix-schema-table is false (default is false),all keys in redis will be mapped to table's key,no matching occurs*
-
-Please refer to the `kafka-tutorial` for the description of the ``dataFormat`` as well as various available decoders.
-
-In addition to the above Kafka types, the Redis connector supports ``hash`` type for the ``value`` field which represent data stored in the Redis hash.
-Redis connector use `hgetall key` to get data.
-``` json
-      {
-      "tableName": ...,
-      "schemaName": ...,
-      "value": {
-        "dataFormat": "hash",
-        "fields": [
-          ...
-        ]
-      }
-    }
-```
-
-the Redis connector supports ``zset`` type for the ``key`` field which represent key stored in the Redis zset. 
-if and only if ``zset`` is used as key datafomart,the split is truly supported , because we can use `zrange zsetkey split.start split.end` to get keys of a split.
-``` json
-      {
-      "tableName": ...,
-      "schemaName": ...,
-      "key": {
-        "dataFormat": "zset",
-        "name": "zsetkey", //zadd zsetkey score member
-        "fields": [
-            ...
-        ]
-      }
-    }
-```
-
-Redis Connector Limitations
----------------------------
-only support read operation,don't support write operation.
-
-

hetu-docs/en/develop/_index.md

@@ -1,5 +1,5 @@
 
 # Developer Guide
 
-openLooKeng is based on Trino 316(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
+openLooKeng is based on Trino(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
 

hetu-docs/en/develop/connectors.md

@@ -21,10 +21,10 @@ This interface is too big to list in this documentation, but if you are interest
 connector. If your underlying data source supports schemas, tables and columns, this interface should be straightforward to implement. If you are attempting to adapt something that is not a relational database (as
 the Example HTTP connector does), you may need to get creative about how you map your data source to openLooKeng\'s schema, table, and column concepts.
 
-### ConnectorSplitManager
+### ConnectorSplitManger
 
 The split manager partitions the data for a table into the individual chunks that openLooKeng will distribute to workers for processing. For example, the Hive connector lists the files for each Hive partition and creates
-one or more split per file. For data sources that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.
+one or more split per file. For data so urces that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.
 
 ### ConnectorRecordSetProvider
 

hetu-docs/en/develop/externalfunction-registration-pushdown.md

@@ -3,7 +3,7 @@ External Function Registration and Push Down
 
 Introduction
 ------------
-The connector can register `external function` into openLooKeng. In Jdbc connector, openLooKeng can push them down to data source which support to execute those functions.
+The connector can register `external function` into openLooKeng. In Jdbc connector, openLookeng can push them down to data source which support to execute those functions.
 
 Function Registration in Connector
 ----------------------------------

hetu-docs/en/develop/functions.md

@@ -216,7 +216,7 @@ An in-depth look at the various annotations relevant to writing an aggregation f
     
 -   `@CombineFunction`:
 
-    The `@CombineFunction` annotation declares the function used to combine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
+    The `@CombineFunction` annotation declares the function used to ombine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
     the results into the first one (in the above example, just by adding them together).
     
 -   `@OutputFunction`:

hetu-docs/en/develop/getting-started.md

@@ -5,7 +5,7 @@
 
 * Mac OS X or Linux
 * Java 8 Update 161 or higher (8u161+), 64-bit. Both Oracle JDK and OpenJDK are supported.
-* AArch64 ([Bisheng JDK 1.8.262 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
+* AArch64 ([Bisheng JDK 11 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
 * Maven 3.3.9+ (for building)
 * Python 2.4+ (for running with the launcher script)
 

hetu-docs/en/develop/star-tree-cube.md

@@ -0,0 +1,148 @@
+# Star-Tree 
+
+Star tree cubing is a pre-aggregation technique to achieve low latency runtime for iceberg queries. An iceberg query computes an aggregate function over 
+an attribute ( or set of attributes) in order to find aggregate values above a specified threshold. Using this technique, User is provided with an option to 
+create a cube with necessary aggregations and dimensions. Then, when aggregation queries are executed, the cube is used to executing the query instead of the 
+original table. The actual performance gain is achieved during the TableScan operation as cubes are pre-computed and pre-aggregated. 
+
+For this reason, the cubing technique is highly effective when the group by cardinality results in lesser rows than the original table.
+
+## Supported functions
+    COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
+
+## Enabling and Disabling Star-tree
+To enable:
+```sql 
+SET SESSION enable_star_tree_index=true;
+```
+To disable:
+```sql 
+SET SESSION enable_star_tree_index=false;
+```
+
+## Configuration Properties
+| Property Name                                     | Default Value       | Required| Description|
+|---------------------------------------------------|---------------------|---------|--------------|
+| optimizer.enable-star-tree-index                  | false               | No      | Enables star-tree index|
+| cube.metadata-cache-size                          | 5                   | No      | The maximum number of metadata for star-trees that could be loaded into cache before eviction happens|
+| cube.metadata-cache-ttl                           | 1h                  | No      | The maximum time to live of star-trees that are be loaded into cache before eviction happens |
+
+## Examples
+
+Creating a star-tree cube:
+```sql 
+CREATE CUBE nation_cube 
+ON nation 
+WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
+GROUP=(nationkey),
+format='orc', partitioned_by=ARRAY['nationkey']);
+```
+Next, to add data to the cube:
+```sql 
+INSERT INTO CUBE nation_cube WHERE nationkey >= 5;
+```
+Creating a star-tree cube with CLI with WHERE clause:
+```sql 
+CREATE CUBE nation_cube 
+ON nation 
+WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
+GROUP=(nationkey),
+format='orc', partitioned_by=ARRAY['nationkey'])
+WHERE nationkey >= 5;
+```
+
+To use the new cube, just query the original table using aggregations that were included in the cube:
+```sql 
+SELECT count(*) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
+SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
+```
+
+Since the data inserted into the cube was for nationkey >= 5, only queries matching this condition will utilize the cube. 
+Queries not matching the condition will still work as usual.
+
+## Optimizer Changes
+
+The star tree aggregation rule is an Iterative optimizer that optimizes the logical plan by replacing the original aggregation sub-tree
+and original table scan with pre-aggregation table scan. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can
+be supported by the Cubes. The exact rows are not queried to check if Cube is applicable or not.
+
+## Dependencies
+
+Star Tree index relies on Hetu metastore to store the cube related metadata.
+Please check [Hetu Metastore](../admin/meta-store.md) for more information.
+
+### Building Cube for Large dataset
+One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
+Processing large number rows requires more memory than cluster is configured with. This results in query failing with message "Query exceeded per-node user memory 
+limit". To overcome this issue, "Insert into cube" query support was added. The user has ability to build a cube for larger data by executing multiple 
+insert into cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into cube.
+
+This section explains the steps to build a cube for larger dataset. 
+
+Let us take an example of TPCDS dataset and `store_sales` table. The table has 10 years worth of data 
+and user wants to build a cube for year 2001 and due to the cluster memory limit, the entire data set for year 2001 cannot be processed at once.
+
+```sql
+
+CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk));
+
+SELECT min(d_date_sk) as year_start, max(d_date_sk) as year_end FROM date_dim WHERE d_year = 2001;
+ year_start | year_end 
+------------+----------
+    2451911 |  2452275 
+(1 row)
+
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275; 
+-- This could result in query failure if the number of rows need to be processed is huge and the query memory exceeds the configured limit.
+
+To overcome this issue, multiple insert statements can be used into process rows and insert into cube and the number of rows canbe controlled by using where clause;
+
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2452010;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk >= 2452011 AND ss_sold_date_sk <= 2452110;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452111 AND 2452210;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452211 AND 2452275;
+
+Alternative to overcome this issue, use CLI if expression is Between Predicate or Comparison Expression. CLI internally queries multiple insert statements.
+
+CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk)) WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275;
+
+Internally the system will rewrite and merge all continuous range predicates into a single predicate;
+
+SHOW CUBES;
+
+           Cube Name             |         Table Name         | Status |         Dimensions          |                     Aggregations                      |                                     Where Clause                                     
+---------------------------------+----------------------------+--------+-----------------------------+-------------------------------------------------------+-------------------------------------------------------+------------------------------
+ hive.tpcds_sf1.store_sales_cube | hive.tpcds_sf1.store_sales | Active | ss_sold_date_sk,ss_store_sk | sum(ss_sales_price),sum(ss_net_paid),sum(ss_quantity) | (("ss_sold_date_sk" >= BIGINT '2451911') AND ("ss_sold_date_sk" < BIGINT '2452276')) 
+
+Note:
+1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together. 
+   All continous predicates will be merged into a single range predicate and remainining predicates are untouched.
+   Only the following types are supported and can be merged together. 
+   Integer, TinyInt, SmallInt, BigInt, Date;
+   For other types, its difficult to identify if two predicates are continous therefore they cannot be merged together. And because of this issue, there is 
+   possibility that particular cube may not be used during query optimisation even if the cube has all the required data. For example,
+   
+   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
+   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
+   
+   Here these two predicates cannot be merged into store_id BETWEEN 'A01' AND 'A20'; So the cube won't be used 
+   for queries that are spanning over two the predicates;
+   
+   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube won't be used for optimizing this query. This is a limitation as of now.
+
+2. Only Single column predicates can be merged. 
+```
+
+#CLI changes
+
+The CLI supports the create cube statement with where clause. The where clause used in the create cube statement is the range of data to be inserted into the cube.
+Once the user runs create cube statement with where clause then the cli internally runs the insert cube statements. This process improves the user experience and
+improves the memory footprint based on the cluster memory limits. As of now, only Between Predicate and Comparison Expressions are supported. We support only Integer and Long Literals.
+
+## Limitation
+
+1. Star tree cube is only effective when the group by cardinality is considerably lower than the number of rows in 
+   source table.
+2. A significant amount of user effort required in maintaining Cubes for large datasets.
+3. Only incremental insert into cube is supported. Cannot delete specific rows from Cube.
+4. For the create cube statement in CLI, we only support Between Predicate or Comparison Expression. We support only Integer and Long Literal.

hetu-docs/en/develop/types.md

@@ -13,7 +13,7 @@ Below is a high level overview of the `Type` interface, for more details see the
     
 -   Native encoding:
 
-    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the Unix epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
+    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the unix  epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
     
 -   Type signature:
 

hetu-docs/en/faq/faq.md

@@ -40,10 +40,6 @@
    >
    > openLooKeng Bilibili channel:  https://space.bilibili.com/627629884   
 
-8. Which version of Trino is openLooKeng developed on?
-
-   > Based on Trino 316 version development.
-
 ## Functions
 
 1. What connectors does the openLooKeng support?

hetu-docs/en/functions/conversion.md

@@ -106,7 +106,7 @@ Obviously, not all data types are compatible with each other, below table lists
 
 **Note:**
 
-- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. Please refer to below item.
+- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. please refer to below item.
 - N: standard for not support implicit convert
 
 (1): BOOLEAN-\>NUMBER the converted result can be only 0 or 1
@@ -123,23 +123,23 @@ Obviously, not all data types are compatible with each other, below table lists
 
 (7): VARCHAR-\>BOOLEAN only \'0\',\'1\',\'TRUE\',\'FALSE\' can be converted. Others will be failed
 
-(8): VARCHAR-\>DECIMAL conversion will fail when it's not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
+(8): VARCHAR-\>DECIMAL conversion will fail when its not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
 
 (9): VARCHAR-\>CHAR if length of VARCHAR is larger than CHAR, it will be cut off.
 
-(10): VARCHAR-\>DATE The VARCHAR can only be formatted like: \'YYYY-MM-DD\', e.g. 2000-01-01
+(10): VARCHAR-\>DATE The VARCHAR can only be formatted like:\'YYYY-MM-DD\', e.g. 2000-01-01
 
-(11): VARCHAR-\>TIME The VARCHAR can only be formatted like: \'HH:MM:SS.XXX\'
+(11): VARCHAR-\>TIME The VARCHAR can only be formatted like:\'HH:MM:SS.XXX\'
 
-(12): VARCHAR-\>TIME ZONE The VARCHAR can only be formatted like: \'HH:MM:SS.XXX XXX\', e.g. 01:02:03.456 America/Los\_Angeles
+(12): VARCHAR-\>TIME ZONE The VARCHAR can only be formatted like:\'HH:MM:SS.XXX XXX\', e.g. 01:02:03.456 America/Los\_Angeles
 
-(13): VARCHAR-\>TIMESTAMP The VARCHAR can only be formatted like: YYYY-MM-DD HH:MM:SS.XXX
+(13): VARCHAR-\>TIMESTAMP The VARCHAR can only be formatted like:YYYY-MM-DD HH:MM:SS.XXX
 
-(14): DATE-\>TIMESTAMP will auto padding the time with 0. e.g. \'2010-01-01\' -> 2010-01-01 00:00:00.000
+(14): DATE-\>TIMESTAMP will auto padding the time with 0. e.g.\'2010-01-01\' -> 2010-01-01 00:00:00.000
 
 (15): TIME-\>TIME WITH TIME ZONE will auto padding the default time zone
 
-(16): TIME-\>TIMESTAMP will auto add the default date: 1970-01-01
+(16): TIME-\>TIMESTAMP will auto add the default date:1970-01-01
 
 Miscellaneous
 -------------

hetu-docs/en/functions/geospatial.md

@@ -245,7 +245,7 @@ Returns `true` if this Geometry is an empty geometrycollection, polygon, point e
 
 **ST\_IsSimple(Geometry)** -\> boolean
 
-Returns `true` if this Geometry has no anomalous geometric points, such as self-intersection or self-tangency.
+Returns `true` if this Geometry has no anomalous geometric points, such as self intersection or self tangency.
 
 **ST\_IsRing(Geometry)** -\> boolean
 

hetu-docs/en/functions/lambda.md

@@ -13,7 +13,7 @@ Lambda expressions are written with `->`:
     x -> CAST(x AS JSON)
     x -> x + TRY(1 / 0)
 
-Most SQL expressions can be used in a lambda body, with a few exceptions:
+Most SQL expressions can be used in a lambda body, with a fewexceptions:
 
 -   Subqueries are not supported. `x -> 2 + (SELECT 3)`
 -   Aggregations are not supported. `x -> max(y)`

hetu-docs/en/functions/window.md

@@ -45,7 +45,7 @@ Returns the rank of a value in a group of values. This is similar to `rank`, exc
 **ntile(n)** -\> bigint
 
 Divides the rows for each window partition into `n` buckets ranging from `1` to at most `n`. Bucket values will differ by at most `1`. If the number of rows in the partition does not divide evenly into the number
-of buckets, then the remainder values are distributed one per bucket, starting with the first bucket.
+of buckets, then the remainder values are distributed one per bucket,starting with the first bucket.
 
 For example, with `6` rows and `4` buckets, the bucket values would be as follows: `1` `1` `2` `2` `3` `4`
 

hetu-docs/en/index.md

@@ -46,7 +46,6 @@ headless: true
     - [Audit Log]({{< relref "./docs/admin/audit-log.md" >}})
     - [Reliable Execution]({{< relref "./docs/admin/reliable-execution.md" >}})
     - [JDBC Data Source Multi-Split Management]({{< relref "./docs/admin/multi-split-for-jdbc-data-source.md" >}})
-    - [Extension Physical Execution Planner]({{< relref "./docs/admin/extension-execution-planner.md" >}})
 
 - [Query Optimizer]("#")
     - [Table Statistics]({{< relref "./docs/optimizer/statistics.md" >}})
@@ -63,12 +62,7 @@ headless: true
     - [BTree Index]({{< relref "./docs/indexer/btree.md" >}})
     - [HIndex Statements]({{< relref "./docs/indexer/hindex-statements.md" >}})
     - [New Index]({{< relref "./docs/indexer/new-index.md" >}})
-    
-- [Star Tree Cubes](#)
-    - [Overview]({{< relref "./docs/preagg/overview.md" >}})
-    - [Join Support]({{< relref "./docs/preagg/join-queries.md" >}})
-    - [Statements]({{< relref "./docs/preagg/statements.md" >}})
-  
+
 - [Connectors]({{< relref "./docs/connector/_index.md" >}})
     - [Carbondata]({{< relref "./docs/connector/carbondata.md" >}})
     - [ClickHouse]({{< relref "./docs/connector/clickhouse.md" >}})
@@ -84,7 +78,6 @@ headless: true
     - [JMX]({{< relref "./docs/connector/jmx.md" >}})
     - [Kafka]({{< relref "./docs/connector/kafka.md" >}})
     - [Kafka Connector Tutorial]({{< relref "./docs/connector/kafka-tutorial.md" >}})
-    - [Redis] ({{< relref "./docs/connector/redis.md" >}})
     - [Local File]({{< relref "./docs/connector/localfile.md" >}})
     - [Memory]({{< relref "./docs/connector/memory.md" >}})
     - [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
@@ -99,8 +92,7 @@ headless: true
     - [TPCH]({{< relref "./docs/connector/tpch.md" >}})
     - [VDM]({{< relref "./docs/connector/vdm.md" >}})
     - [Kylin]({{< relref "./docs/connector/kylin.md" >}})
-    - [OmniData]({{< relref "./docs/connector/omnidata.md" >}})
-    
+
 - [Functions and Operators]("#")
     - [Logical Operators]({{< relref "./docs/functions/logical.md" >}})
     - [Comparison Functions and Operators]({{< relref "./docs/functions/comparison.md" >}})
@@ -140,6 +132,7 @@ headless: true
     - [CALL]({{< relref "./docs/sql/call.md" >}})
     - [COMMENT]({{< relref "./docs/sql/comment.md" >}})
     - [COMMIT]({{< relref "./docs/sql/commit.md" >}})
+    - [CREATE CUBE]({{< relref "./docs/sql/create-cube.md" >}})
     - [CREATE ROLE]({{< relref "./docs/sql/create-role.md" >}})
     - [CREATE SCHEMA]({{< relref "./docs/sql/create-schema.md" >}})
     - [CREATE TABLE]({{< relref "./docs/sql/create-table.md" >}})
@@ -151,6 +144,7 @@ headless: true
     - [DESCRIBE INPUT]({{< relref "./docs/sql/describe-input.md" >}})
     - [DESCRIBE OUTPUT]({{< relref "./docs/sql/describe-output.md" >}})
     - [DROP CACHE]({{< relref "./docs/sql/drop-cache.md" >}})
+    - [DROP CUBE]({{< relref "./docs/sql/drop-cube.md" >}})
     - [DROP ROLE]({{< relref "./docs/sql/drop-role.md" >}})
     - [DROP SCHEMA]({{< relref "./docs/sql/drop-schema.md" >}})
     - [DROP TABLE]({{< relref "./docs/sql/drop-table.md" >}})
@@ -162,6 +156,8 @@ headless: true
     - [GRANT ROLES]({{< relref "./docs/sql/grant-roles.md" >}})
     - [INSERT]({{< relref "./docs/sql/insert.md" >}})
     - [INSERT OVERWRITE]({{< relref "./docs/sql/insert-overwrite.md" >}})
+    - [INSERT CUBE]({{< relref "./docs/sql/insert-cube.md" >}})
+    - [INSERT OVERWRITE CUBE]({{< relref "./docs/sql/insert-overwrite-cube.md" >}})
     - [JMX]({{< relref "./docs/sql/jmx.md" >}})
     - [PREPARE]({{< relref "./docs/sql/prepare.md" >}})
     - [RESET SESSION]({{< relref "./docs/sql/reset-session.md" >}})
@@ -174,9 +170,9 @@ headless: true
     - [SHOW CACHE]({{< relref "./docs/sql/show-cache.md" >}})
     - [SHOW CATALOGS]({{< relref "./docs/sql/show-catalogs.md" >}})
     - [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
-    - [SHOW CREATE CUBE]({{< relref "./docs/sql/show-create-cube.md" >}})
     - [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
     - [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
+    - [SHOW CUBES]({{< relref "./docs/sql/show-cubes.md" >}})
     - [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
     - [SHOW EXTERNAL FUNCTION]({{< relref "./docs/sql/show-external-function.md" >}})
     - [SHOW GRANTS]({{< relref "./docs/sql/show-grants.md" >}})
@@ -210,6 +206,7 @@ headless: true
     - [Filesystem Access Utilities]({{< relref "./docs/develop/filesystem.md" >}})
     - [Hive ORC Cache]({{< relref "./docs/develop/hive-orc-cache.md" >}})
     - [External Function Registration and Push Down]({{< relref "./docs/develop/externalfunction-registration-pushdown.md" >}})
+    - [Star Tree Cube]({{< relref "./docs/develop/star-tree-cube.md" >}})
 
 - [openLooKeng REST API]({{< relref "./docs/rest/_index.md" >}})
     - [Node Resource]({{< relref "./docs/rest/node.md" >}})
@@ -219,11 +216,6 @@ headless: true
     - [Task Resource]({{< relref "./docs/rest/task.md" >}})
 
 - [Release Notes]("#")
-    - [1.6.1 (27 Apr 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.1.md" >}})
-    - [1.6.0 (30 Mar 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.0.md" >}})
-    - [1.5.0 (30 Dec 2021)]({{< relref "./docs/releasenotes/releasenotes-1.5.0.md" >}})
-    - [1.4.1 (12 Nov 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.1.md" >}})
-    - [1.4.0 (15 Oct 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.0.md" >}})
     - [1.3.0 (30 Jun 2021)]({{< relref "./docs/releasenotes/releasenotes-1.3.0.md" >}})
     - [1.2.0 (31 Mar 2021)]({{< relref "./docs/releasenotes/releasenotes-1.2.0.md" >}})
     - [1.1.0 (30 Dec 2020)]({{< relref "./docs/releasenotes/releasenotes-1.1.0.md" >}})

hetu-docs/en/indexer/hindex-statements.md

@@ -31,7 +31,7 @@ CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WITH ("bloom.
 CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WHERE p in (part1, part2, part3);
 ```
 
-**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), for BTree index, only index creation on the **first** level is supported (colA). Bloom, Bitmap and Minmax index creation on either (colA or colB) is supported.
+**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), only index creation on the **first** level is supported (colA).
 
 ## SHOW
 

hetu-docs/en/indexer/new-index.md

@@ -10,13 +10,17 @@ The `getID()` method in the `Index` interface returns the ID of this index type
 
 ### Level
 
-A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e.  it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();`  which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
+A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must 
+have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e. 
+it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();` 
+which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
 
 ## Interface overlook
 
 ### Indexing methods
 
-Apart from the methods mentioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete document on `Index` interface, please refer to the Java Doc of the source code.
+Apart from the methods metioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete 
+document on `Index` interface, please refer to the Java Doc of the source code.
 
 There are two main functionalities in the `Index` interface:
 
@@ -42,7 +46,7 @@ boolean addValues(Map<String, List<Object>> values) throws IOException;
 Index deserialize(InputStream in) throws IOException;
 
 void serialize(OutputStream out) throws IOException;
-```
+``` 
 
 The usage of them are pretty straightforward. A good example to help understand their usage is the source code of `MinMaxIndex`, where adding values is just to 
 update the `max` and `min` variables according to the input number, and `serialize()/deserialize()`

hetu-docs/en/installation/deployment-auto.md

@@ -3,13 +3,13 @@
 
 
 
-In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete the deployment faster and easier. The script is friendly to most of Linux OS. However, to Ubuntu, you need to manually install the following dependencies:
+In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete deployment faster and easier. The script is friendly to most of linux OS. However, to Ubuntu, you need to manually install the following dependencies:
 
 > sshpass1.06 or above
 
 ## Deploying openLooKeng on a Single Node
 
-Executing the below command can help you download the necessary packages and deploy openLooKeng server in one-click:
+Execute below command can help you download the necessary packages and deploy openLooKeng server in one-click:
 
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh)
@@ -21,7 +21,7 @@ or:
 wget -O - https://download.openlookeng.io/install.sh|bash
 ```
 
-Normally, you don\'t need to do anything, except waiting for the installation to complete. It will automatically start the service.
+Normally, you don\'t need to do any thing, except for the installation to complete. It will automatically start the service.
 
 Execute below command to stop openLooKeng service.:
 
@@ -51,13 +51,13 @@ or:
 bash <(wget -qO- https://download.openlookeng.io/install.sh) --multi-node
 ```
 
-First, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
+First of all, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
 
 Besides, jdk version is required to be greater than 1.8.0\_151. If not, jdk1.8.0\_201 will be installed in the cluster. It is recommended to manually install these dependencies before installing openLooKeng service.
 
 Secondly, the script will download openLooKeng-server tarball and copy that tarball to all the nodes in the cluster. Then install the openLooKeng-server by using this tarball.
 
-Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and for built-in catalogs like `tpch`, `tpcds`, `memory connector`.
+Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and also for build-in catalogs like `tpch`, `tpcds`, `memory connector`.
 
 By design, the script will check if there are existing configuration under directory:
 `/home/openlkadmin/.openlkadmin/cluster_node_info`
@@ -79,9 +79,9 @@ The general configurations for openLooKeng\'s coordinator, workers are taken fro
 `/home/openlkadmin/.openlkadmin/cluster_config_info` and configurations for connectors are taken from the directory `/home/openlkadmin/.openlkadmin/catalog` respectively. If these directories or any required configuration files are absent during the deploy script running, default configuration files will be generated
 automatically and deployed to all nodes.
 
-Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customize the deployment.
+Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customized the deployment.
 
-If all the above processes succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
+If above process all succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
 
 ```shell
 /opt/openlookeng/bin/stop.sh
@@ -108,7 +108,7 @@ bash <(wget -qO- https://download.openlookeng.io/install.sh) --file <cluster_nod
 ```
 
 
-For more help, execute below command to deploy single node cluster:
+For more help message,execute below command to deploy single node cluster:
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh) -h
 ```
@@ -150,9 +150,9 @@ execute below command to deploy the configurations to openLooKeng cluster:
 bash /opt/openlookeng/bin/configuration_deploy.sh
 ```
 
-Note, if you want to add more configurations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
+Note, if you want to add more configrations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
 
-The property format must be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it is a dynamic value. For example:
+The property format has to be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it it a dynamic value. For example:
 
 ``` properties
 http-server.http.port=<http-server.http.port>
@@ -174,7 +174,7 @@ It is very easy and straight forward to uninstall openLooKeng Service, simply ru
 bash /opt/openlookeng/bin/uninstall.sh
 ```
 
-This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you want to delete user and configuration files, you need to run the below command:
+This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you wan to delete user and configuration files, you need to run the below command:
 
 ```shell
 bash /opt/openlookeng/bin/uninstall.sh --all
@@ -190,7 +190,7 @@ If you can't access the download URL from the machine where you want to install
 
 1. Also save third party dependencies under `/opt/openlookeng/resource`. That is, download all files from either `https://download.openlookeng.io/auto-install/third-resource/x86/` or `https://download.openlookeng.io/auto-install/third-resource/aarch64/`, depending on the machine's architecture. This should include 1 `OpenJDK` file and 2 `sshpass` files.
 
-1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
+1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture, and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
 
 After all resources are available, execute below command to deploy single node cluster:
 
@@ -217,7 +217,7 @@ bash /opt/openlookeng/bin/install_offline.sh --help
 
 ## Adding Node to Cluster
 
-If you want to add node to make the cluster bigger, execute the below command:
+If you want to add node to make the cluster bigger,execute the below command:
 
 ```shell
 bash /opt/openlookeng/bin/add_cluster_node.sh -n <ip_address_1,……ip_address_N>
@@ -238,11 +238,11 @@ or:
 bash /opt/openlookeng/bin/add_cluster_node.sh --file <add_nodes_file_path>
 ```
 
-If there are multiple nodes, separated by commas (,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
+If there are multiple nodes, separated by commas(,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
 
 ## Removing Node to Cluster
 
-If you want to remove node to make the cluster smaller, execute the below command:
+If you want to remove node to make the cluster smaller,execute the below command:
 
 ```shell
 bash /opt/openlookeng/bin/remove_cluster_node.sh -n <ip_address_1,……ip_address_N>
@@ -263,7 +263,7 @@ or:
 bash /opt/openlookeng/bin/remove_cluster_node.sh --file <remove_nodes_file_path>
 ```
 
-If there are multiple nodes, separate them with commas (,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
+If there are multiple nodes, separate them with commas(,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
 
 ## See Also
 

hetu-docs/en/installation/deployment-ha.md

@@ -25,7 +25,7 @@ The above properties are described below:
 - `hetu.multiple-coordinator.enabled`: Enable multiple coordinators.
 - `hetu.embedded-state-store.enabled`: Enable coordinators to start embedded state store. 
 
-Note: It is suggested to enable embedded state store on all coordinators (or at least 3) to guarantee the high availability of service when node/network is down.
+Note: It is suggested to enable embedded state store on all coordinators(or at least 3) to guarantee the high availability of service when node/network is down.
 
 ###Configuring State Store
 Please refer to the section [State Store](../admin/state-store.md) to configure state store.

hetu-docs/en/installation/odbc.md

@@ -61,7 +61,7 @@ Before an application uses the openLooKeng ODBC driver, the data source DSN must
 
 ### Opening the ODBC Data Source Administrator (64-bit)
 
-1. Click **Start** and choose **Control Panel**.
+1. Click **Start**, and choose **Control Panel**.
 
 2. In **Control Panel**, click **System and Security**, and then click **Administrative Tools**.
 
@@ -171,6 +171,6 @@ You can obtain the details about data types by calling **SQLGetTypInfo** in **Ca
 
 The openLooKeng ODBC driver supports **both ANSI and Unicode** applications. The default connection character set is the system default character set for ANSI applications and utf8 for Unicode applications. If the character set used by the application is different from the above-mentioned character set,  it may cause garbled characters. For this, the user should specify the connection character set to adapt to the character set required by the application. The corresponding configuration of the connection character set is described as follows.
 
-When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by default, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
+When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by deafult, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
 
 While configuring data source all connection character sets supported by the openLooKeng ODBC driver can be set in the **Character Set** drop-down box on the page 3 of the User interface. User can select the connection character from the drop-down box after the **Test DSN** is success.

hetu-docs/en/preagg/join-queries.md

@@ -1,65 +0,0 @@
-## Join Query Support
-StarTree Cube can help optimize aggregation over join queries as well. The optimizer looks for aggregation subtree pattern in the logical plan that typically looks like following.
-```
-  AggregationNode
-  |- ProjectNode[Optional]
-  .  |- ProjectNode[Optional]
-  .  .  |- FilterNode[Optional]
-  .  .  .  |- JoinNode
-  .  .  .  .  [More Joins]
-  .  .  .  .  .
-  .  .  .  .  |- ProjectNode[Optional] - Left
-  .  .  .  .  .  |- TableScanNode [Fact Table]
-  .  .  .  .  |- ProjectNode[Optional] - Right
-  .  .  .  .  .  |- TableScanNode [Dim Table]
-```
-
-If the query matches the pattern, the optimizer rewrites the logical plan by replacing the Fact TableScanNode with Cube TableScanNode. This is similar to the single 
-table rewrite.
-
-### Star Schema Support
-Join Query optimizer supports star schema only. A star schema is a data warehousing architecture model where one fact table references multiple dimension tables, which, when viewed as a diagram, 
-looks like a star with the fact table in the center and the dimension tables radiating from it. All kinds of joins are supported.
-
-![star-schema](../images/star-schema.png "star schema")
-
-### Cube Management
-`Create Cube` can be still be used to define Cubes to optimize Join queries as well. The difficult part is identifying GROUP construct while building the Cubes. With single table 
-queries, the GROUP BY clause will contain columns only from same the table. But with join queries, especially star schema queries, the GROUP BY contain columns from Dimension tables and not the Fact table. 
-Let's analyze more with following query
-```sql
-SELECT SUM(lo_revenue) AS lo_revenue, d_year, p_brand
-FROM lineorder
-LEFT JOIN dates ON lo_orderdate = d_datekey
-LEFT JOIN part on lo_partkey = p_partkey
-LEFT JOIN supplier on lo_suppkey = s_suppkey
-WHERE p_category = 'MFGR#12' AND s_region = 'AMERICA'
-GROUP BY d_year, p_brand
-ORDER BY d_year, p_brand;
-```
-
-Here `lineorder` is the Fact table and `dates`, `part`, `supplier` are the Dimension tables. Cubes will be defined on the `lineorder` table. The group by columns `d_year`, `p_brand` are part 
-of the dimension tables `dates` and `part` appropriately. They cannot be used directly in `CREATE CUBE` statement. The proper solution is to use the foreign key columns of `lineorder` table in 
-the GROUP construct while building Cubes.
-```sql
-CREATE CUBE lineorder_cube ON lineorder WITH(
-AGGREGATIONS = (sum(lo_revenue)),
-GROUP = (lo_orderdate, lo_partkey, lo_suppkey));
-```
-The optimizer parses the join conditions and uses those columns to identify the matching Cubes. The performance gain is realized if Cube size is smaller than fact table.
-
-### Limitations
-* Only star schema is supported.
-* Count distinct not supported because Cube does not store actual dimension values.
-* Queries won't be optimized if Cubes are defined on both Fact and Dimension as the optimizer does not have capability to differentiate between two.
-* If Cubes are defined on more than one table of the Join query - then Optimizer does not work. Assumption is that Cubes are defined only the Fact table.
-* Supports only simple aggregation like SUM, COUNT, AVG, MIN, MAX - defined on Single column. Cube does not support SUM(revenue - supplycost) aggregation. The following query cannot be optimized using Cube.
-```
-   SELECT sum(lo_extendedprice * lo_discount) AS revenue
-   FROM lineorder
-   WHERE toYear(lo_orderdate) = 1993 AND lo_discount BETWEEN 1 AND 3 AND lo_quantity < 25;
-```
-
-### Future
-* Support for snowflake schema
-* Building a single cube over multiple tables

hetu-docs/en/preagg/overview.md

@@ -1,261 +0,0 @@
-# StarTree Cube
-## Introduction
-StarTree Cubes are materialized pre-aggregation results stored as tables. This technique is built to optimize low latency iceberg queries. 
-Iceberg queries are a special case of SQL queries involving **GROUP BY** and **HAVING** clauses, wherein the answer set is small relative to the data scanned size. 
-Queries can be characterized by their huge input-small output. 
-
-This technique allows user to build Cubes on an existing table with aggregates and dimensions that are intended to optimize specific queries. 
-Cubes are rollup pre-aggregations that have fewer dimensions and rows compared to the original table. Smaller number of rows means the time spent 
-on table scan is significantly reduced which in turn reduces query latency. If a query is a subset of dimensions and measures of the pre-aggregated table, 
-then Cube can be used to calculate the query without accessing the original table. 
-
-Few of the Cube properties are
- - Cubes are stored in tabular format
- - Generally speaking, Cubes can be created for any table in any connector and stored in another connector
- - Query latency is reduced by rewriting the logical plan to use Cube instead of the original table.
-
-## Cube Optimizer Rule
-As part of logical plan optimization, Cube optimizer rule analyzes and optimizes the aggregation subtree of the logical plan with Cubes. 
-The rule looks for the aggregation subtree that typically looks like the following
-
-```
-AggregationNode
-|- ProjectNode[Optional]
-  |- ProjectNode[Optional]
-    |- FilterNode[Optional]
-      |- ProjectNode[Optional]
-        |- TableScanNode
-```
-
-The rule parses through the subtree and identifies the table name, aggregate functions, where clause, group by clause that is matched with Cube metadata 
-to identify any Cube that can help optimize the query. In case of multiple match, recently created Cube is selected for optimization. If any match found, entire 
-aggregation subtree is rewritten using the Cube. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can be supported by the 
-Cubes.
-
-The following picture depicts the change in the logical plan after the optimization. 
-
-![img](../images/cube-logical-plan-optimizer.png)
-
-## Recommended Usage
-1. Cubes are most useful for iceberg queries that takes huge input and produces small output.
-2. Query performance is best when size of the Cube is less that on the actual table on which Cube was built.
-3. Cubes need to be rebuilt if the source table is updated. 
-
-**Note:** 
-If the source table is updated once the Cubes are built, Cube optimizer ignores the set of Cubes created on the table. Reason being, any 
-operation on the update is considered as a change in the existing data even if only new rows are inserted on the original table. Since inserts and updates
-can't be differentiated, Cubes can't be used as it might result in incorrect result. We are working on a solution to overcome this limitation.
-
-## Supported Connectors
-Star Tree Cube can be stored in following Connectors
-1. Hive
-2. Memory
-   
-Tables from following Connectors can be used as source to build a StarTree Cube.
-1. Hive
-2. Memory
-3. Clickhouse
-
-## Future Work
-1. Support for more JDBC connectors
-2. Simplify Cube management 
-
-   2.1. Overcome the limitation of Creating Cube for larger dataset.
-
-## Enabling and Disabling StarTree Cube
-To enable:
-```sql 
-SET SESSION enable_star_tree_index=true;
-```
-To disable:
-```sql 
-SET SESSION enable_star_tree_index=false;
-```
-
-## Configuration Properties
-| Property Name                                     | Default Value       | Required| Description|
-|---------------------------------------------------|---------------------|---------|--------------|
-| optimizer.enable-star-tree-index                  | false               | No      | Enables StarTree Cube |
-| cube.metadata-cache-size                          | 50                  | No      | The maximum number of metadata for StarTree Cubes that could be loaded into cache before eviction happens |
-| cube.metadata-cache-ttl                           | 1h                  | No      | The maximum time to live of StarTree Cubes that are be loaded into cache before eviction happens |
-
-## Dependencies
-
-StarTree Cube relies on Hetu Metastore to store the Cube related metadata.
-Please check [Hetu Metastore](../admin/meta-store.md) for more information.
-
-## Examples
-
-Creating a StarTree Cube:
-```sql 
-CREATE CUBE nation_cube 
-ON nation 
-WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
-GROUP=(nationkey),
-format='orc', partitioned_by=ARRAY['nationkey']);
-```
-Next, to add data to the Cube:
-```sql 
-INSERT INTO CUBE nation_cube WHERE nationkey >= 5;
-```
-Creating a StarTree Cube with WHERE clause:
-Please note that the following query is only supported via the CLI
-
-```sql 
-CREATE CUBE nation_cube 
-ON nation 
-WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
-GROUP=(nationkey),
-format='orc', partitioned_by=ARRAY['nationkey'])
-WHERE nationkey >= 5;
-```
-
-To use the new Cube, just query the original table using aggregations that were included in the Cube:
-
-```sql 
-SELECT count(*) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
-SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
-```
-
-Since the data inserted into the Cube was for `nationkey >= 5`, only queries matching this condition will utilize the Cube.
-Queries not matching the condition would continue to work but won't use the Cube.
-
-If the source table of a Cube gets updated, the corresponding Cube gets expired automatically. In order to overcome
-this issue, we have added support in openLooKeng CLI by introducing **RELOAD CUBE** command. The user will have the
-ability to manually reload a cube if the status of the Cube becomes INACTIVE or EXPIRED. The syntax to reload the
-Cube nation_cube is as follows,
-
-```sql 
-RELOAD CUBE nation_cube
-```
-Please note that this feature is only supported via the CLI. During this reload process if an unexpected error occurs, the user will get to see the original SQL statement
-to recreate the cube manually.
-
-## Building Cube for Large Dataset
-One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
-Processing large number of rows requires more memory than cluster is configured with. This results in query failing with message **Query exceeded per-node user memory
-limit**. To overcome this issue, **INSERT INTO CUBE** SQL support was added. The user has ability to build a Cube for larger data by executing multiple
-insert into Cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into Cube.
-
-This section explains the steps to build a Cube for larger dataset.
-
-Let us take an example of TPCDS dataset and `store_sales` table. The table has 10 years worth of data and user wants to build a Cube for year 2001 
-and due to the cluster memory limit, the entire data set for year 2001 cannot be processed at once.
-
-```sql
-CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk));
-
-SELECT min(d_date_sk) as year_start, max(d_date_sk) as year_end FROM date_dim WHERE d_year = 2001;
- year_start | year_end 
-------------+----------
-    2451911 |  2452275 
-(1 row)
-```
-The following query could result in a failure if the number of rows need to be processed is huge and the query memory exceeds 
-the limit configured for the cluster.
-
-```sql
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275; 
-```
-
-### Solution 1)
-To overcome this issue, multiple insert statements can be used to process rows and insert into Cube and the number of rows can be limited by using where clause;
-
-```sql
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2452010;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk >= 2452011 AND ss_sold_date_sk <= 2452110;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452111 AND 2452210;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452211 AND 2452275;
-```
-
-### Solution 2)
-CLI has been modified to support creating Cubes for larger dataset and without need for multiple insert statements. CLI internally handles this process.
-Once the user runs create Cube statement with where clause, the CLI takes care of creating the Cube as well as inserting the data into it. This process improves the user experience and
-improves the memory footprint based on the cluster memory limits. CLI internally parses the converts the statement into one create Cube statement followed by 
-one or more insert statements. This change is only works if user executes the command from CLI and not via any other means i.e. JDBC, etc...
-
-```sql
-CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk)) WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275;
-```
-
-Internally the system will rewrite and merge all continuous range predicates into a single predicate;
-
-```sql
-SHOW CUBES;
-
-           Cube Name             |         Table Name         | Status |         Dimensions          |                     Aggregations                      |                                     Where Clause                                     
----------------------------------+----------------------------+--------+-----------------------------+-------------------------------------------------------+-------------------------------------------------------+------------------------------
- hive.tpcds_sf1.store_sales_cube | hive.tpcds_sf1.store_sales | Active | ss_sold_date_sk,ss_store_sk | sum(ss_sales_price),sum(ss_net_paid),sum(ss_quantity) | (("ss_sold_date_sk" >= BIGINT '2451911') AND ("ss_sold_date_sk" < BIGINT '2452276')) 
-```
-
-**Note:**
-1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together.
-   All continuous predicates will be merged into a single range predicate and remaining predicates are untouched.
-
-   Only the following types are supported and can be merged together.
-   `Integer, TinyInt, SmallInt, BigInt, Date, String`
-
-   For String data type, predicate merge logic functionally works only if the Strings are ending with a digit and all are of same length. 
-   For example,
-
-```sql
-   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
-   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
-```
-After the insertion, the two predicates will be merged into `'A01' AND 'A20'`
-
-```sql
-   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube would be used for this query.
-```
-
-   Consider the following example where `store_id` values are not of same length.
-
-```sql
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A1';
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A2' 
-```   
-   store_id predicate will be rewritten as `store_id >= 'A1' and store < 'A3'` as per the varchar predicate merge logic;
-
-```sql   
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A10' 
-```
-   The above query would fail because `A10` is subset of the range `store_id >= 'A1' and store < 'A3'`. So Users should be wary of this issue. 
-
-For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is
-possibility that particular Cube may not be used during query optimization even if the Cube has all the required data.
-
-2. Predicate rewrite has some limitations as well. Consider the following query
-
-```sql   
-   INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
-```
-The predicate is rewritten as ss_sold_date_sk >= 2451912 to support merging continuous predicates.
-Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to
-optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
-
-```sql   
-   SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
-```   
-3. Only single column predicates can be merged.
-
-
-## Open issues and Limitations
-1. StarTree Cube is only effective when the group by cardinality is considerably fewer than the number of rows in source table.
-2. A significant amount of user effort required in maintaining Cubes for large datasets.
-3. Only incremental insert into Cube is supported. Cannot delete specific rows from Cube.
-4. Cubes created on a transaction table may expire automatically even if the source table has not been updated. This is due to the compaction policy which
-   merges delta files into single large ORC file which in turn changes the last modified of time of the table. Cube status is determined by comparing last modified
-   timestamp of table when Cube was created with the last modified time of the table when queries are executed.
-5. OpenLooKeng CLI has been modified to ease the process of creating Cubes for larger datasets. But still there are limitations with this implementation
-   as the process involves merging multiple Cube predicates into one. Only Cube predicates defined on Integer, Long and Date types can be merged properly. Support for Char, 
-   String types still need to be implemented.
-6. Varchar predicates can be merged only if the values are of same length.  
-
-## Performance Optimizations on Star Tree
-1. Star Tree Query re-write optimization for same group by columns: If the group by columns of the cube and query matches, the query is 
-re-written internally to select the pre-aggregated data. If the group by columns does not matches, the additional aggregations are 
-internally applied on the re-written query.
-2. Star Tree table scan optimization for Average aggregation function: If the group by columns of the cube and query matches, the select query 
-is re-written internally to select the startree cube's pre-aggregated Average column data. If the group by columns does not match, 
-the select query is re-written internally to select the startree cube's pre-aggregated Sum and Count column data, from which the 
-average is later calculated.

hetu-docs/en/preagg/statements.md

@@ -1,195 +0,0 @@
-# Usage
-Cubes can be managed using any of the supported clients, such as hetu-cli located under the `bin` directory in the installation.
-
-## CREATE CUBE
-### Synopsis
-
-``` sql
-CREATE CUBE [ IF NOT EXISTS ]
-cube_name ON table_name WITH (
-   AGGREGATIONS = ( expression [, ...] ),
-   GROUP = ( column_name [, ...])
-   [, FILTER = (expression)]
-   [, ( property_name = expression [, ...] ) ] 
-)
-[WHERE predicate]
-```
-### Description
-Create a new, empty Cube with the specified group and aggregations. Use `INSERT INTO CUBE (see below)` to insert into data.
-
-The optional `IF NOT EXISTS` clause causes the error to be suppressed if the Cube already exists.
-The optional `property_name` section can be used to set properties on the newly created Cube. 
-
-To list all available table properties, run the following query:
-
-    SELECT * FROM system.metadata.table_properties
-
-**Note:** These properties are limited to the Connector which the Cube is being created for.
-
-### Examples
-Create a new Cube `orders_cube` on `orders`:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
-      GROUP = ( orderstatus, orderdate ),
-      format = 'ORC'
-    )
-
-Create a new partitioned Cube `orders_cube`:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
-      GROUP = ( orderstatus, orderdate ),
-      format = 'ORC',
-      partitioned_by = ARRAY['orderdate']
-    )
-
-Create a new Cube `orders_cube` with some source data filter:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    )
-
-Create a new Cube `orders_cube` with some additional predicate on Cube columns:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    ) WHERE orderstatus = 'PENDING';
-
-This is same as following:
-    
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    );
-    INSERT INTO CUBE orders_cube WHERE orderstatus = 'PENDING';
-
-The `FILTER` property can be used to filter out data from the source table while building the Cube. Cube is built on the data after 
-applying the `orderdate BETWEEN 2512450 AND 2512460` predicate on the source table. The columns used in the filter predicate must not be part the Cube.
-
-### Limitations
-- Cubes can be created with only following aggregation functions.  
-  In other words, Queries using the following functions can only be optimized using Cubes.
-  **COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG**
-- Different connector might support different data type, and different table/column properties.
-
-## INSERT INTO CUBE
-
-### Synopsis
-``` sql
-INSERT INTO CUBE cube_name [WHERE condition]
-```
-
-### Description
-`CREATE CUBE` statement creates Cube without any data. To insert data into Cube, use `INSERT INTO CUBE` SQL.
-The `WHERE` clause is optional. If predicate is provided, only data matching the given predicate are processed from the source table and inserted into the Cube. 
-Otherwise, entire data from the source table is processed and inserted into Cube.
-
-### Examples
-Insert data into the `orders_cube` Cube:
-
-    INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
-    INSERT INTO CUBE order_all_cube;
-
-### Limitations
-1. Subsequent inserts to the same Cube need to use same set of columns
-
-```sql
-   CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (count(*)), GROUP = (orderdate));
-   
-   INSERT INTO CUBE orders_cube WHERE orderdate BETWEEN date '1999-01-01' AND date '1999-01-05';
-   
-   -- This statement would fail because its possible the Cube already contain rows matching the given predicate.
-   INSERT INTO CUBE orders_cube WHERE location = 'Canada';
-```
-**Note:** This means that columns used in the first insert must be used in every insert predicate following the first to avoid inserting duplicate data.
-
-## INSERT OVERWRITE CUBE
-
-### Synopsis
-``` sql
-INSERT OVERWRITE CUBE cube_name [WHERE condition]
-```
-
-### Description
-Similar to `INSERT INTO CUBE` statement but with this statement the existing data is overwritten. Predicates
-are optional.`INSERT OVERWRITE CUBE` is not supported on partitioned cubes. Cubes are essentially stored as tables and so `INSERT OVERWRITE` only 
-replaces the matching partitions and does not overwrite the entire table. So this operation is blocked on partitioned cube. 
-Drop and recreate cube if needed.
-
-### Examples
-Insert data based on condition into the `orders_cube` Cube:
-
-    INSERT OVERWRITE CUBE orders_cube WHERE orderdate > date '1999-01-01';
-    INSERT OVERWRITE CUBE orders_cube;
-
-## SHOW CUBES
-
-### Synopsis
-```sql
-SHOW CUBES [ FOR table_name ];
-```
-
-### Description
-`SHOW CUBES` lists all Cubes. Adding the optional `table_name` lists only the Cubes for that table.
-
-### Examples
-
-Show all Cubes:
-```sql
-    SHOW CUBES;
-```
-
-Show Cubes for `orders` table:
-
-```sql
-    SHOW CUBES FOR orders;
-```
-## RELOAD CUBE
-
-### Synopsis
-
-``` sql
-RELOAD CUBE cube_name
-```
-
-### Description
-Reloads the Cube if the source table has been updated.
-
-### Examples
-
-If the source table `orders` of the cube `orders_cube` gets updated then the status of the cube `orders_cube` 
-gets EXPIRED. Use the command `RELOAD CUBE cube_name` to overcome this issue as follows:
-
-```sql
-    RELOAD CUBE orders_cube
-```
-## DROP CUBE
-
-### Synopsis
-
-``` sql
-DROP CUBE  [ IF EXISTS ] cube_name
-```
-
-### Description
-Drop an existing Cube.
-
-The optional `IF EXISTS` clause causes the error to be suppressed if the Cube does not exist.
-
-### Examples
-
-Drop the Cube `orders_cube`:
-
-    DROP CUBE orders_cube
-
-Drop the Cube `orders_cube` if it exists:
-
-    DROP CUBE IF EXISTS orders_cube
-
-

hetu-docs/en/releasenotes/releasenotes-1.4.1.md

@@ -1,14 +0,0 @@
-# Release 1.4.1 (12 Nov 2021)
-
-## Key Features
-
-This release mainly adds the introduction of OmniData Connector and jdk8 support under arm architecture.
-
-| Area                    | Feature                                                      | PR #s                                                        |
-| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Data Source Connector                | openLooKeng supports offload operators to the near data side for process through omnidata connector, reducing the transmission of invalid data on the network and effectively improving the performance of big data computing.                                               | 1219                                                          |
-| Arm architecture | Eliminate the mandatory requirements for Java version under arm architecture caused by JDK paused problem, and support jdk1.8.262 and above under arm architecture. | 1214                                                           |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.5.0.md

@@ -1,23 +0,0 @@
-# Release 1.5.0
-
-## Key Features
-
-| Area             | Feature                                                      |
-| ---------------- | ------------------------------------------------------------ |
-| Star Tree        | 1. Support for optimizing join queries such as star schema queries.<br/>2. Optimized the query plan by eliminating unnecessary aggregations on top of the cube since the cube already contains the rolled up results. The performance optimizations benefits queries whose group by clause exactly matches the cube’s group.<br/>3. Bug fixes to further enhance the usability, and robustness of cubes. |
-| Memory Connector | 1. Improved performance of memory connector by adding support for memory table partitioning to allow data skipping of entire partitions<br/>2. Collect statistics on memory table to support openLooKeng cost based optimizers. |
-| Task Recovery    | Fixed several important bugs to address data inconsistency issues, and query hanging issues that occasionally occur during high concurrency, and during worker failures. |
-| OLK-on-Yarn      | Support deploying an HA-enabled openLooKeng cluster instance on-yarn, that contains a reverse proxy (ngnix by default), and 2 or more coordinator nodes. The cluster can be horizontally scaled manually by adding and removing yarn containers to the coordinator and worker components.|
-| Spill to Disk    | Optimized the spill to disk mechansim to directly write Pages to disk instead of buffering it. Changed the strategy to spill those operators which can free up maximum memory. This resulted in improvement of spill to disk performance by 30% |
-
-## Known Issues
-
-| Category              | Description                                                  | Gitee issue                                                       |
-| --------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Task Recovery         |When the query process reaches stage1,a worker is killed result some values become smaller(occasionally).   |[I4M2LW](https://e.gitee.com/open_lookeng/issues/list?issue=I4M2LW) |
-| Memory Connector      |When a table with the index_columns parameter is queried, an error message is displayed：java.lang.NullPointerException. | [I4NVW3](https://e.gitee.com/open_lookeng/issues/list?issue=I4NVW3)|
-|                       |When drop and then create a same partitioned table with data type double, query result rows is greater than expected. | [I4LUDF](https://e.gitee.com/open_lookeng/issues/list?issue=I4LUDF) |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.6.0.md

@@ -1,25 +0,0 @@
-# Release 1.6.0
-
-## Key Features
-
-| Area                  | Feature                                                      |
-| --------------------- | ------------------------------------------------------------ |
-| Star Tree             | Support update cube command to allow admin to easily update an existing cube when the underlying data changes |
-| Bloom Index           | Hindex-Optimize Bloom Index Size-Reduce bloom index size by 10X+ times |
-| Task Recovery         | 1. Improve failure detection time: It need take 300s to determine a task is failed and resume after that. Improving this would improve the resume & also the overall query time<br/>2. snapshotting speed & size: When sql execute takes a snapshot, now use direct Java serialization which is slow and also takes more size. Using kryo serialization would reduce size and also increase speed there by increasing the overall throughput |
-| Spill to Disk         | 1. Spill to disk speed & size improvement: When spill happens during HashAggregation & GroupBy, the data serialized to disk is slow and also size is more. It can improve the overall performance by reducing size and also improving  the writing speed. Using kryo serialization improves both speed and reduces size<br/>2. Support spilling to hdfs: Currently data can spill to multiple disks, now support spill to hdfs to improve throughput<br/>3. Async spill/unspill: When revocable memory crosses threshold and spill is triggered, it blocks accepting the data from the downstream operators. Accepting this and adding to the existing spill would help to complete the pipeline faster<br/>4. Enable spill for right outer & full join for spilling: It don’t spill the build side data when the join type is right outer or full join as it needs the entire data in memory for lookup. This leads to out of  memory when the data size is more. Instead by enable spill and create a Bloom Filter to identify the data spilled and use it during join with probe side |
-| Connector Enhancement | Support data update and delete operator for PostgreSQL and openGauss |
-
-## Known Issues
-
-| Category      | Description                                                  | Gitee issue                                               |
-| ------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
-| Task Recovery | When a snapshot is enabled and a CTAS with transaction is executed, an error is reported in the SQL statement. | [I502KF](https://e.gitee.com/open_lookeng/issues/list?issue=I502KF) |
-|               | An error occurs occasionally when snapshot is enabled and exchange.is-timeout-failure-detection-enabled is disabled. | [I4Y3TQ](https://e.gitee.com/open_lookeng/issues/list?issue=I4Y3TQ) |
-| Star Tree     | In the memory connector, after the star tree is enabled, data inconsistency occurs during query. | [I4QQUB](https://e.gitee.com/open_lookeng/issues/list?issue=I4QQUB) |
-|               | When the reload cube command is executed for 10 different cubes at the same time, some cubes fail to be reloaded. | [I4VSVJ](https://e.gitee.com/open_lookeng/issues/list?issue=I4VSVJ) |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en)
-

hetu-docs/en/releasenotes/releasenotes-1.6.1.md

@@ -1,15 +0,0 @@
-# Release 1.6.1 (27 Apr 2022)
-
-## Key Features
-
-This release is mainly about modification and enhancement of some SPIs, which are used in more scenarios.
-
-| Area                    | Feature                                                      | PR #s                                                        |
-| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Data Source statistics                | The method of obtaining statistics is added so that statistics can be directly obtained from the Connector. Some operators can be pushed down to the Connector for calculation. You may need to obtain statistics from the Connector to display the amount of processed data.                                               | 1450                                                          |
-| Operator processing extension | Users can customize the physical execution plan of worker nodes. Users can use their own operator pipelines to replace the native implementation to accelerate operator processing. | 1436                                                           |
-| HIVE UDF extension | Adds the adaptation of HIVE UDF function namespace to support the execution of UDFs (including GenericUDF) written based on the HIVE UDF framework. | 1456                                                           |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en)

hetu-docs/en/security/built-in-system-access-control.md

@@ -1,3 +1,4 @@
+
 Built-in System Access Control
 ==============================
 
@@ -56,10 +57,7 @@ composed of the following fields:
 
 -   `user` (optional): regex to match against user name. Defaults to `.*`.
 -   `catalog` (optional): regex to match against catalog name. Defaults to `.*`.
--   ``allow`` (required): string indicating whether a user has access to the catalog.
-    This value can be ``all``, ``read-only`` or ``none``, and defaults to ``none``.
-    Setting this value to ``read-only`` has the same behavior as the ``read-only``
-    system access control plugin.
+-   `allow` (required): boolean indicating whether a user has access to the catalog
 
 
 **Note**
@@ -67,9 +65,7 @@ composed of the following fields:
 *By default, all users have access to the `system` catalog. You can override this behavior by adding a rule.*
 
 
-For example, if you want to allow only the user ``admin`` to access the``mysql`` and the ``system`` catalog, 
-allow all users to access the ``hive`` catalog, allow the user ``alice`` read-only access to the ``postgresql`` 
-catalog, and deny all other access, you can use the following rules:
+For example, if you want to allow only the user `admin` to access the `mysql` and the `system` catalog, allow all users to access the `hive` catalog, and deny all other access, you can use the following rules:
 
 ``` json
 {
@@ -77,20 +73,15 @@ catalog, and deny all other access, you can use the following rules:
     {
       "user": "admin",
       "catalog": "(mysql|system)",
-      "allow": all
+      "allow": true
     },
     {
       "catalog": "hive",
-      "allow": all
-    },
-    {
-      "user": "alice",
-      "catalog": "postgresql",
-      "allow": "read-only"
+      "allow": true
     },
     {
       "catalog": "system",
-      "allow": none
+      "allow": false
     }
   ]
 }
@@ -204,7 +195,6 @@ If you want to allow users to use the  extactly same name as their Kerberos prin
 ```
 
 ### Node State Rules
-
 These rules govern the node state info particular users can access. The user is granted access to update a node state based on the first matching rule read from top to bottom. If no rule matches, access is denied. Each rule is
 composed of the following fields:
 

hetu-docs/en/security/ranger-access-control.md

@@ -7,7 +7,7 @@ Overview
 
 Apache Ranger delivers a comprehensive approach to security for a Hadoop cluster. It provides a centralized platform to define, administer and manage security policies consistently across Hadoop components. Check [Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index) for detail introduction and user guide.
 
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is developed based on Ranger 2.1.0, which is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
+[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
 
 Build Process
 -------------------------

hetu-docs/en/sql/cache-table.md

@@ -38,7 +38,7 @@ Cache data with complex predicate string:
 Limitations
 -----------
 
-- Only Hive connector (ORC Format) support this functionality at this time. See connector documentation for more details.
+- Only Hive connector(ORC Format) support this functionality at this time. See connector documentation for more details.
 - Does not support `LIKE` in `WHERE` clause.
 - Does not support 'OR' operator in complex predicate.
 

hetu-docs/en/sql/create-cube.md

@@ -0,0 +1,69 @@
+CREATE CUBE
+============
+
+Synopsis
+--------
+
+``` sql
+CREATE CUBE [ IF NOT EXISTS ]
+cube_name ON table_name WITH (
+   AGGREGATIONS = ( expression [, ...] ),
+   GROUP = ( column_name [, ...])
+   [, FILTER = (expression)]
+   [, ( property_name = expression [, ...] ) ] 
+)
+```
+
+Description
+-----------
+
+Create a new, empty star-tree cube with the specified group and aggregations. Use `insert-into-cube` to insert data.
+
+The optional `IF NOT EXISTS` clause causes the error to be suppressed if the table already exists.
+
+The optional `property_name` section can be used to set properties on the newly created cube. To list all available table properties, run the following query:
+
+    SELECT * FROM system.metadata.table_properties
+
+Examples
+--------
+
+Create a new cube `orders_cube` on `orders`:
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
+      GROUP = ( orderstatus, orderdate ),
+      format = 'ORC'
+    )
+
+Create a new partitioned cube `orders_cube`:
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
+      GROUP = ( orderstatus, orderdate ),
+      format = 'ORC',
+      partitioned_by = ARRAY['orderdate']
+    )
+
+Create a new cube 'orders_cube' with some source filter
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
+      GROUP = ( orderstatus ),
+      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
+    )
+
+Filter is additional predicate that applied on the source table when building a cube. The columns used in the filter predicate must not be part the Cube.
+
+Limitations
+-----------
+
+- Supported aggregate functions:
+      COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
+- Only one group supported per Cube.  
+- Different connector might support different data type, and different table/column properties.
+- Can currently only create cubes in Hive connector, but the cubes can be created on a table from another connector. 
+
+See Also
+--------
+[INSERT INTO CUBE](./insert-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)

hetu-docs/en/sql/create-view.md

@@ -14,7 +14,7 @@ AS query
 Description
 -----------
 
-Create a new view of a [SELECT](./select.md) query. The view is a logical table that can be referenced by future queries. Views do not contain any data. Instead, the query stored by the view is executed every time the view is referenced by another query.
+Create a new view of a [SELECT](./select.md) query. The view is a logical table that can be referenced by future queries. Views do not contain any data. Instead, the query stored by the view is executed everytime the view is referenced by another query.
 
 The optional `OR REPLACE` clause causes the view to be replaced if it already exists rather than raising an error.
 

hetu-docs/en/sql/drop-cube.md

@@ -0,0 +1,33 @@
+
+DROP CUBE
+==========
+
+Synopsis
+--------
+
+``` sql
+DROP CUBE  [ IF EXISTS ] cube_name
+```
+
+Description
+-----------
+
+Drop an existing cube.
+
+The optional `IF EXISTS` clause causes the error to be suppressed if the cube does not exist.
+
+Examples
+--------
+
+Drop the cube `orders_cube`:
+
+    DROP CUBE orders_cube
+
+Drop the cube `orders_cube` if it exists:
+
+    DROP CUBE IF EXISTS orders_cube
+
+See Also
+--------
+
+[CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [INSERT INTO CUBE](./insert-cube.md)

hetu-docs/en/sql/insert-cube.md

@@ -0,0 +1,44 @@
+INSERT INTO CUBE
+======
+
+Synopsis
+--------
+
+``` sql
+INSERT INTO CUBE cube_name [WHERE condition]
+```
+
+Description
+-----------
+
+Insert data into a star-tree cube. Predicate information is optional. If predicate provided, only data matching 
+the given predicate are processed from the source table and inserted into the cube. Otherwise, entire 
+data from the source table is processed and inserted into Cube.
+
+Examples
+--------
+
+Insert data based on condition into the `orders_cube` cube:
+
+    INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
+    INSERT INTO CUBE order_all_cube;
+
+See Also
+--------
+
+[INSERT OVERWRITE CUBE](./insert-overwrite-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)
+
+
+Limitations
+----------
+1. Insert statement does not allow different columns to be used in the where clause for successive inserts.
+
+```sql
+   CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (count(*)), GROUP = (orderdate));
+   
+   INSERT INTO CUBE orders_cube WHERE orderdate BETWEEN date '1999-01-01' AND date '1999-01-05';
+   
+   -- This statement would fail because its possible the Cube already contain rows matching the given predicate.
+   INSERT INTO CUBE orders_cube WHERE location = 'Canada';
+```
+Note: this means that columns used in the first insert must be used in every insert predicate following the first to avoid inserting duplicate data.

hetu-docs/en/sql/insert-overwrite-cube.md

@@ -0,0 +1,28 @@
+INSERT INTO CUBE
+======
+
+Synopsis
+--------
+
+``` sql
+INSERT OVERWRITE CUBE cube_name [WHERE condition]
+```
+
+Description
+-----------
+
+Similar to INSERT INTO CUBE statement but with this statement the existing data is overwritten. Predicates 
+are optional.
+
+Examples
+--------
+
+Insert data based on condition into the `orders_cube` cube:
+
+    INSERT OVERWRITE CUBE orders_cube WHERE orderdate > date '1999-01-01';
+    INSERT OVERWRITE CUBE orders_cube;
+
+See Also
+--------
+
+[INSERT INTO CUBE](./insert-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)

hetu-docs/en/sql/insert-overwrite.md

@@ -28,7 +28,7 @@ Assume `orders` is not a partitioned table, and have 100 rows, then execute belo
 
     INSERT OVERWRITE orders VALUES (1, 'SUCCESS', '10.25', DATA '2020-01-01');
 
-Then the `orders` table will only have 1 row that is the data specified in the `VALUE` clause.
+Then the `orders` table will only have 1 rows, that is the data specified in the `VALUE` clause.
 
 Assume `users` has 3 columns: (`id`, `name`, `state`) and partitioned by `state`, and the existing data has follow rows:
 

hetu-docs/en/sql/select.md

@@ -508,7 +508,7 @@ _col0
 **INTERSECT**
 
 `INTERSECT` returns only the rows that are in the result sets of both the first and the second queries. The following is an example of one of the simplest possible `INTERSECT` clauses. It selects the values `13`
-and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results:
+and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results.:
 
     SELECT * FROM (VALUES 13, 42)
     INTERSECT
@@ -524,7 +524,7 @@ _col0
 **EXCEPT**
 
 `EXCEPT` returns the rows that are in the result set of the first query, but not the second. The following is an example of one of the simplest possible `EXCEPT` clauses. It selects the values `13` and `42` and
-combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result:
+combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result.:
 
     SELECT * FROM (VALUES 13, 42)
     EXCEPT
@@ -746,7 +746,7 @@ Joins allow you to combine data from multiple relations.
 
 ### CROSS JOIN
 
-A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explicit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
+A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
 
 Both of the following queries are equivalent:
 

hetu-docs/en/sql/show-create-cube.md

@@ -1,32 +0,0 @@
-
-SHOW CREATE CUBE
-=================
-
-Synopsis
---------
-
-``` sql
-SHOW CREATE CUBE cube_name
-```
-
-Description
------------
-
-Show the SQL statement that creates the specified cube.
-
-Examples
---------
-
-Create a cube `orders_cube` on `orders` table as follows
-
-    CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
-    GROUP = (custKEY, ORDERkey), format= 'orc')
-
-Use `SHOW CREATE CUBE` command to show the SQL statement that was used to create the cube `orders_cube`:
-
-    SHOW CREATE CUBE orders_cube;
-
-``` sql
-CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
-GROUP = (custKEY, ORDERkey), format= 'orc')
-```

hetu-docs/en/sql/show-cubes.md

@@ -0,0 +1,35 @@
+
+SHOW CUBES
+==========
+
+Synopsis
+--------
+
+``` sql
+SHOW CUBES [ FOR table_name ];
+```
+
+Description
+-----------
+
+`SHOW CUBES` lists all cubes. Adding the optional `table_name` lists only the cubes for that table.
+
+Examples
+--------
+
+Show all cubes:
+
+```sql
+    SHOW CUBES;
+```
+
+Show cubes for `orders` table:
+
+```sql
+    SHOW CUBES FOR orders;
+```
+
+See Also
+--------
+
+[CREATE CUBE](./create-cube.md), [DROP CUBE](./drop-cube.md), [INSERT INTO CUBE](./insert-cube.md)