!1531 Fixed 0 row count issue while recovering

Merge pull request !1531 from Kishore Kumar M V/0rows_after_recovery

hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.4.0</version>
+    <version>1.7.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,8 +108,7 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this.metastore = metastore;
-            this.schemaName = null;
+            this(metastore, null);
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -233,7 +232,6 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
-        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,16 +73,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putShorts(rowId, count, value);
+            columnVector.putShorts(inputRowId, count, value);
         }
     }
 
@@ -97,16 +98,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putInts(rowId, count, value);
+            columnVector.putInts(inputRowId, count, value);
         }
     }
 
@@ -121,16 +123,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putLongs(rowId, count, value);
+            columnVector.putLongs(inputRowId, count, value);
         }
     }
 
@@ -145,11 +148,12 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -164,16 +168,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putDoubles(rowId, count, value);
+            columnVector.putDoubles(inputRowId, count, value);
         }
     }
 
@@ -196,11 +201,12 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -223,16 +229,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putNull(counter++);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putNulls(rowId, count);
+            columnVector.putNulls(inputRowId, count);
         }
     }
 
@@ -319,66 +326,72 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -127,15 +127,16 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        this.outPutPath = requireNonNull(outPutPath, "path is null");
+        Path localOutPutPath = paramOutPutPath;
+        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            outPutPath = new Path(properties.getProperty("location"));
+            localOutPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -211,7 +212,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -226,25 +227,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path outPutPath = this.outPutPath;
-        Properties properties = this.properties;
-        JobConf configuration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
+        Path finalOutPutPath = this.outPutPath;
+        Properties finalProperties = this.properties;
+        JobConf finalConfiguration = this.configuration;
+        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
-                        properties, Reporter.NULL);
+                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
+                        finalProperties, Reporter.NULL);
     }
 
     @Override
@@ -285,7 +286,7 @@ public class CarbondataFileWriter
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter recordWriter = null;
+        FileSinkOperator.RecordWriter finalRecordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -334,7 +335,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -351,7 +352,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            recordWriter = this.recordWriter;
+            finalRecordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -365,8 +366,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (recordWriter != null) {
-                recordWriter.write(serDe.serialize(row, tableInspector));
+            if (finalRecordWriter != null) {
+                finalRecordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,6 +48,7 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
+    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuFilterUtil.java

@@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
             if (rawData instanceof Slice) {
                 String value = ((Slice) rawData).toStringUtf8();
                 if (type.getTypeInfo() instanceof CharTypeInfo) {
-                    String padding = "";
+                    StringBuilder padding = new StringBuilder();
                     int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
                     int truncatedLength = value.length();
                     for (int i = 0; i < paddedLength - truncatedLength; i++) {
-                        padding += " ";
+                        padding.append(" ");
                     }
                     return value + padding;
                 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuOutputFormat.java

@@ -103,7 +103,6 @@ public class CarbondataHetuOutputFormat<T>
         OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
         JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
         carbonOutputCommitter.setupJob(jobContext);
-        CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
         org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
         return new FileSinkOperator.RecordWriter()
         {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataLocationService.java

@@ -80,8 +80,6 @@ public class CarbondataLocationService
     {
         // TODO: check and make it compatible for cloud scenario
 
-        HdfsEnvironment.HdfsContext context =
-                new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
         Path targetPath = new Path(table.getStorage().getLocation());
 
         return new LocationHandle(targetPath, targetPath, true,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java

@@ -306,13 +306,13 @@ public class CarbondataMetadata
 
     private void setupCommitWriter(Properties hiveSchema, Path outputPath, Configuration initialConfiguration, boolean isOverwrite) throws PrestoException
     {
-        CarbonLoadModel carbonLoadModel;
+        CarbonLoadModel finalCarbonLoadModel;
         TaskAttemptID taskAttemptID = TaskAttemptID.forName(initialConfiguration.get("mapred.task.id"));
         try {
             ThreadLocalSessionInfo.setConfigurationToCurrentThread(initialConfiguration);
-            carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
-            carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
-            CarbonTableOutputFormat.setLoadModel(initialConfiguration, carbonLoadModel);
+            finalCarbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
+            finalCarbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
+            CarbonTableOutputFormat.setLoadModel(initialConfiguration, finalCarbonLoadModel);
         }
         catch (IOException ex) {
             LOG.error("Error while creating carbon load model", ex);
@@ -360,13 +360,13 @@ public class CarbondataMetadata
         this.user = session.getUser();
         return hdfsEnvironment.doAs(user, () -> {
             SchemaTableName tableName = parent.getSchemaTableName();
-            Optional<Table> table =
+            Optional<Table> finalTable =
                     metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-            if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+            if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
                 throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
             }
 
-            this.table = table;
+            this.table = finalTable;
             Path outputPath =
                     new Path(parent.getLocationHandle().getJsonSerializableTargetPath());
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -382,7 +382,7 @@ public class CarbondataMetadata
             }
 
             /* Create committer object */
-            setupCommitWriter(table, outputPath, initialConfiguration, isOverwrite);
+            setupCommitWriter(finalTable, outputPath, initialConfiguration, isOverwrite);
 
             return new CarbondataInsertTableHandle(parent.getSchemaName(),
                     parent.getTableName(),
@@ -416,13 +416,13 @@ public class CarbondataMetadata
         currentState = State.UPDATE;
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> table =
+        Optional<Table> finalTable =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = table;
+        this.table = finalTable;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -430,8 +430,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", table.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -470,13 +470,13 @@ public class CarbondataMetadata
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         List<HiveColumnHandle> inputColumns = parent.getInputColumns().stream().filter(HiveColumnHandle::isRequired).collect(toList());
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> table =
+        Optional<Table> finalTable =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = table;
+        this.table = finalTable;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -484,8 +484,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", table.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -643,7 +643,7 @@ public class CarbondataMetadata
 
         return hdfsEnvironment.doAs(session.getUser(), () -> {
             Properties hiveSchema = MetastoreUtil.getHiveSchema(this.table.get());
-            CarbonTable carbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
+            CarbonTable finalCarbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
                     carbondataVacuumTableHandle.getTableName(),
                     hiveSchema,
                     initialConfiguration);
@@ -705,7 +705,7 @@ public class CarbondataMetadata
                     SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath()));
                     String source;
                     for (String currPartitionName : partitionNames) {
-                        source = carbonTable.getTablePath() + "/" + currPartitionName;
+                        source = finalCarbonTable.getTablePath() + "/" + currPartitionName;
                         moveFromTempFolder(source + "/" + carbonLoadModel.getSegmentId() + "_" + timeStamp + ".tmp", source);
                     }
                     segmentFilesToBeUpdatedLatest.add(new Segment(carbonLoadModel.getSegmentId(), segmentFileName));
@@ -719,7 +719,7 @@ public class CarbondataMetadata
                 for (CarbondataSegmentInfoUtil segmentInfo : newMergedSegmentInfoUtilList) {
                     String mergedLoadNumber = segmentInfo.getDestinationSegment();
                     try {
-                        String segmentFileName = SegmentFileStore.writeSegmentFile(carbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
+                        String segmentFileName = SegmentFileStore.writeSegmentFile(finalCarbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
                     }
                     catch (IOException e) {
                         throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while merging segment files", e);
@@ -900,9 +900,9 @@ public class CarbondataMetadata
     private LocationHandle getCarbonDataTableCreationPath(ConnectorSession session, ConnectorTableMetadata tableMetadata, HiveWriteUtils.OpertionType opertionType) throws PrestoException
     {
         Path targetPath = null;
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String tableName = finalSchemaTableName.getTableName();
         Optional<String> location = getCarbondataLocation(tableMetadata.getProperties());
         LocationHandle locationHandle;
         FileSystem fileSystem;
@@ -914,32 +914,32 @@ public class CarbondataMetadata
                     throw new PrestoException(NOT_SUPPORTED, format("Setting %s property is not allowed", LOCATION_PROPERTY));
                 }
                 /* if path not having prefix with filesystem type, than we will take fileSystem type from core-site.xml using below methods */
-                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, schemaName), new Path(location.get()));
+                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, finalSchemaName), new Path(location.get()));
                 targetLocation = fileSystem.getFileStatus(new Path(location.get())).getPath().toString();
-                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, schemaName, tableName), targetLocation, false);
+                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, finalSchemaName, tableName), targetLocation, false);
             }
             else {
-                updateEmptyCarbondataTableStorePath(session, schemaName);
+                updateEmptyCarbondataTableStorePath(session, finalSchemaName);
                 targetLocation = carbondataTableStore;
-                targetLocation = targetLocation + File.separator + schemaName + File.separator + tableName;
+                targetLocation = targetLocation + File.separator + finalSchemaName + File.separator + tableName;
                 targetPath = new Path(targetLocation);
             }
         }
         catch (IllegalArgumentException | IOException e) {
             throw new PrestoException(NOT_SUPPORTED, format("Error %s store path %s ", e.getMessage(), targetLocation));
         }
-        locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
+        locationHandle = locationService.forNewTable(metastore, session, finalSchemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
         return locationHandle;
     }
 
     @Override
     public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
     {
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName localSchemaTableName = tableMetadata.getTable();
+        String localSchemaName = localSchemaTableName.getSchemaName();
+        String tableName = localSchemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = schemaName;
+        this.schemaName = localSchemaName;
         currentState = State.CREATE_TABLE;
         List<String> partitionedBy = new ArrayList<String>();
         List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@@ -947,29 +947,29 @@ public class CarbondataMetadata
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
 
-        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
+        metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
 
         BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
         Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
 
-        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                        new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
+                        new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
                         new Path(locationHandle.getJsonSerializableTargetPath())));
 
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             try {
                 Map<String, String> serdeParameters = initSerDeProperties(tableName);
-                Table table = buildTableObject(
+                Table localTable = buildTableObject(
                         session.getQueryId(),
-                        schemaName,
+                        localSchemaName,
                         tableName,
                         session.getUser(),
                         columnHandles,
@@ -981,11 +981,11 @@ public class CarbondataMetadata
                         true, // carbon table is set as external table
                         prestoVersion,
                         serdeParameters);
-                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
-                HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
+                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
+                HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
                 metastore.createTable(
                         session,
-                        table,
+                        localTable,
                         principalPrivileges,
                         Optional.empty(),
                         ignoreExisting,
@@ -1092,8 +1092,8 @@ public class CarbondataMetadata
     public CarbondataTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
     {
         requireNonNull(tableName, "tableName is null");
-        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!table.isPresent()) {
+        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!finalTable.isPresent()) {
             return null;
         }
 
@@ -1102,14 +1102,14 @@ public class CarbondataMetadata
             throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, "Unexpected table present in Hive metastore: " + tableName);
         }
 
-        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(table.get()), table.get().getParameters());
+        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(finalTable.get()), finalTable.get().getParameters());
 
         return new CarbondataTableHandle(
                 tableName.getSchemaName(),
                 tableName.getTableName(),
-                table.get().getParameters(),
-                getPartitionKeyColumnHandles(table.get()),
-                HiveBucketing.getHiveBucketHandle(table.get()));
+                finalTable.get().getParameters(),
+                getPartitionKeyColumnHandles(finalTable.get()),
+                HiveBucketing.getHiveBucketHandle(finalTable.get()));
     }
 
     private Optional<ConnectorOutputMetadata> finishUpdateAndDelete(ConnectorSession session,
@@ -1133,12 +1133,12 @@ public class CarbondataMetadata
 
         hdfsEnvironment.doAs(user, () -> {
             if (blockUpdateDetailsList.size() > 0) {
-                CarbonTable carbonTable = getCarbonTable(tableHandle.getSchemaName(),
+                CarbonTable finalCarbonTable = getCarbonTable(tableHandle.getSchemaName(),
                         tableHandle.getTableName(),
                         MetastoreUtil.getHiveSchema(table.get()),
                         initialConfiguration);
 
-                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(carbonTable);
+                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(finalCarbonTable);
                 SegmentUpdateDetails[] segementDetailsList = statusManager.getUpdateStatusDetails();
                 for (SegmentUpdateDetails segementDetails : segementDetailsList) {
                     segementDetails.getDeletedRowsInBlock();
@@ -1179,26 +1179,26 @@ public class CarbondataMetadata
                                             List<HiveColumnHandle> columnHandles,
                                             Map<String, String> tableProperties)
     {
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String finalTableName = finalSchemaTableName.getTableName();
 
         partitionedBy.addAll(CarbondataTableProperties.getPartitionedBy(tableMetadata.getProperties()));
         sortBy.addAll(CarbondataTableProperties.getSortedBy(tableMetadata.getProperties()));
         Optional<HiveBucketProperty> bucketProperty = Optional.empty();
         columnHandles.addAll(getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator));
-        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, schemaName, tableName)));
+        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName)));
     }
 
     @Override
     public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
     {
         // get the root directory for the database
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String finalTableName = finalSchemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = schemaName;
+        this.schemaName = finalSchemaName;
         currentState = State.CREATE_TABLE_AS;
 
         List<String> partitionedBy = new ArrayList<String>();
@@ -1206,7 +1206,7 @@ public class CarbondataMetadata
         List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
-        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
+        metastore.getDatabase(finalSchemaName).orElseThrow(() -> new SchemaNotFoundException(finalSchemaName));
 
         // to avoid type mismatch between HiveStorageFormat & Carbondata StorageFormat this hack no option
         HiveStorageFormat tableStorageFormat = HiveStorageFormat.valueOf("CARBON");
@@ -1222,29 +1222,29 @@ public class CarbondataMetadata
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE_AS);
         Path targetPath = locationService.getTableWriteInfo(locationHandle, false).getTargetPath();
-        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(finalSchemaName, finalTableName, UUID.randomUUID().toString()));
 
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                                new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
+                                new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName),
                                 new Path(locationHandle.getJsonSerializableTargetPath())));
             // Create Carbondata metadata folder and Schema file
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             Path outputPath = new Path(locationHandle.getJsonSerializableTargetPath());
-            Properties schema = readSchemaForCarbon(schemaName, tableName, targetPath, columnHandles, partitionColumns);
+            Properties schema = readSchemaForCarbon(finalSchemaName, finalTableName, targetPath, columnHandles, partitionColumns);
             // Create committer object
             setupCommitWriter(schema, outputPath, initialConfiguration, false);
         });
         try {
             CarbondataOutputTableHandle result = new CarbondataOutputTableHandle(
-                    schemaName,
-                    tableName,
+                    finalSchemaName,
+                    finalTableName,
                     columnHandles,
-                    metastore.generatePageSinkMetadata(new HiveIdentity(session), schemaTableName),
+                    metastore.generatePageSinkMetadata(new HiveIdentity(session), finalSchemaTableName),
                     locationHandle,
                     tableStorageFormat,
                     partitionStorageFormat,
@@ -1255,7 +1255,7 @@ public class CarbondataMetadata
                     EncodedLoadModel, jobContext.getConfiguration().get(LOAD_MODEL)));
 
             LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
-            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
+            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), finalSchemaTableName);
             return result;
         }
         catch (RuntimeException ex) {
@@ -1386,7 +1386,7 @@ public class CarbondataMetadata
         List<Segment> segmentFilesToBeUpdated = blockUpdateDetailsList.stream()
                 .map(SegmentUpdateDetails::getSegmentName)
                 .map(Segment::new).collect(Collectors.toList());
-        List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
+        List<Segment> finalSegmentFilesToBeUpdatedLatest = new ArrayList<>();
         List<Segment> segmentFilesToBeDeleted = blockUpdateDetailsList.stream()
                 .filter(segmentUpdateDetails -> segmentUpdateDetails.getSegmentStatus() != null &&
                         segmentUpdateDetails.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE))
@@ -1396,12 +1396,12 @@ public class CarbondataMetadata
         for (Segment segment : segmentFilesToBeUpdated) {
             String file =
                     SegmentFileStore.writeSegmentFile(carbonTable, segment.getSegmentNo(), timeStamp.toString());
-            segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
+            finalSegmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
         }
         if (!(updateSegmentStatusSuccess &&
                 CarbonUpdateUtil.updateTableMetadataStatus(new HashSet<>(segmentFilesToBeUpdated),
                         carbonTable, timeStamp.toString(), true, segmentFilesToBeDeleted,
-                        segmentFilesToBeUpdatedLatest, ""))) {
+                        finalSegmentFilesToBeUpdatedLatest, ""))) {
             CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timeStamp.toString());
         }
     }
@@ -1463,11 +1463,10 @@ public class CarbondataMetadata
                 Properties hiveschema = MetastoreUtil.getHiveSchema(table);
                 Configuration configuration = jobContext.getConfiguration();
                 configuration.set(SET_OVERWRITE, "false");
-                CarbonLoadModel carbonLoadModel =
-                        HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
-                LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
-                carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
-                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
+                CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
+                LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
+                loadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
             }
             catch (IOException e) {
                 LOG.error("Error occurred while committing the insert job.", e);
@@ -1554,14 +1553,14 @@ public class CarbondataMetadata
         try {
             hdfsEnvironment.doAs(session.getUser(), () -> {
                 metastore.dropTable(session, handle.getSchemaName(), handle.getTableName());
-                Configuration initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
+                Configuration finalInitialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
                         .getConfiguration(new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
                                 handle.getTableName()), new Path(this.tableStorageLocation.get())));
 
                 Properties schema = MetastoreUtil.getHiveSchema(target.get());
                 schema.setProperty("tablePath", this.tableStorageLocation.get());
                 this.carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(),
-                        schema, initialConfiguration);
+                        schema, finalInitialConfiguration);
                 takeLocks(State.DROP_TABLE);
                 AbsoluteTableIdentifier identifier = this.carbonTable.getAbsoluteTableIdentifier();
                 if (SegmentStatusManager.isLoadInProgressInTable(carbonTable)) {
@@ -1570,7 +1569,7 @@ public class CarbondataMetadata
                 try {
                     //Simultaneous case after acquiring locks we should check table exist.
                     //if table is not there clean the lock folders
-                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, initialConfiguration);
+                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, finalInitialConfiguration);
                 }//CarbonFileException
                 catch (RuntimeException e) {
                     try {
@@ -1867,8 +1866,8 @@ public class CarbondataMetadata
     {
         String tableName = absoluteTableIdentifier.getTableName();
         String databaseName = absoluteTableIdentifier.getDatabaseName();
-        TableInfo tableInfo = carbonTable.getTableInfo();
-        List<SchemaEvolutionEntry> evolutionEntryList = tableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
+        TableInfo finalTableInfo = carbonTable.getTableInfo();
+        List<SchemaEvolutionEntry> evolutionEntryList = finalTableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
         Long updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).getTimeStamp();
         LOG.info("Reverting changes for " + databaseName + "." + tableName);
         List<ColumnSchema> addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getAdded();
@@ -1880,7 +1879,7 @@ public class CarbondataMetadata
                     break;
                 }
                 case DROP_COLUMN: {
-                    tableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
+                    finalTableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
                         if (cols.isInvisible() && removedCols.getColumnUniqueId().equals(cols.getColumnUniqueId())) {
                             cols.setInvisible(false);
                         }
@@ -2027,38 +2026,38 @@ public class CarbondataMetadata
     @Override
     protected ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName)
     {
-        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!finalTable.isPresent() || finalTable.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
             throw new TableNotFoundException(tableName);
         }
 
-        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
+        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(finalTable.get(), typeManager);
         ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
             columns.add(metadataGetter.apply(columnHandle));
         }
 
         // External location property
         ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
-        properties.put(LOCATION_PROPERTY, table.get().getStorage().getLocation());
+        properties.put(LOCATION_PROPERTY, finalTable.get().getStorage().getLocation());
 
         // Storage format property
         properties.put(HiveTableProperties.STORAGE_FORMAT_PROPERTY, CarbondataStorageFormat.CARBON);
 
         // Partitioning property
-        List<String> partitionedBy = table.get().getPartitionColumns().stream()
+        List<String> partitionedBy = finalTable.get().getPartitionColumns().stream()
                 .map(Column::getName)
                 .collect(toList());
         if (!partitionedBy.isEmpty()) {
             properties.put(HiveTableProperties.PARTITIONED_BY_PROPERTY, partitionedBy);
         }
 
-        Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));
+        Optional<String> comment = Optional.ofNullable(finalTable.get().getParameters().get(TABLE_COMMENT));
 
         // add partitioned columns into immutableColumns
         ImmutableList.Builder<ColumnMetadata> immutableColumns = ImmutableList.builder();
 
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
             if (columnHandle.getColumnType().equals(HiveColumnHandle.ColumnType.PARTITION_KEY)) {
                 immutableColumns.add(metadataGetter.apply(columnHandle));
             }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -191,7 +191,7 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore metastore =
+        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
@@ -200,7 +200,7 @@ public class CarbondataMetadataFactory
                         this.hiveTransactionHeartbeatInterval,
                         this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
 
-        return new CarbondataMetadata(metastore,
+        return new CarbondataMetadata(semiTransactionalHiveMetastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
                 this.writesToNonManagedTablesEnabled,
@@ -212,8 +212,8 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(metastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(metastore),
+                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
+                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        hdfsEnvironment.doAs(session.getUser(), () -> {
+        finalHdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,6 +163,7 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
+    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int batchSize = 0;
+        int columnBatchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    batchSize = columnarBatch.numRows();
-                    if (batchSize == 0) {
+                    columnBatchSize = columnarBatch.numRows();
+                    if (columnBatchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(batchSize, blocks);
+            Page page = new Page(columnBatchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,16 +90,17 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
+        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            name = name.substring(0, name.length() - 4).trim();
+            finalName = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            name = name.substring(0, name.length() - 5).trim();
+            finalName = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(name, order);
+        return new SortingColumn(finalName, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -133,6 +133,7 @@ public class CarbondataWriterFactory
     {
     }
 
+    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,14 +143,15 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putDecimal(rowId, value, precision);
+                columnVector.putDecimal(inputRowId, value, precision);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -185,8 +186,9 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(rowId++, value);
+            columnVector.putByteArray(inputRowId++, value);
         }
     }
 
@@ -303,84 +305,90 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putFloat(rowId, src[i]);
+                columnVector.putFloat(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putShort(rowId, src[i]);
+                columnVector.putShort(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putInt(rowId, src[i]);
+                columnVector.putInt(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putLong(rowId, src[i]);
+                columnVector.putLong(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putDouble(rowId, src[i]);
+                columnVector.putDouble(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putByte(rowId, src[i]);
+                columnVector.putByte(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,8 +58,9 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIndex++] == 1);
+            type.writeBoolean(builder, src[srcIdx++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,8 +76,9 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(rowId++, value, precision);
+            putDecimal(id++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,8 +58,9 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
+        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(rowId++, value);
+            putInt(id++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java

@@ -15,12 +15,10 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import com.esotericsoftware.minlog.Log;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import io.prestosql.hive.$internal.au.com.bytecode.opencsv.CSVReader;
 import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.StandardErrorCode;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -53,7 +51,6 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.text.DateFormat;
@@ -67,10 +64,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
@@ -1429,7 +1424,7 @@ public class TestCarbonAllDataType
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtabledrop", false), false);
 
         } catch (IOException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1474,7 +1469,7 @@ public class TestCarbonAllDataType
                     i++;
                 }
             } catch (IOException e) {
-                e.printStackTrace();
+                logger.error(e.getMessage());
             }
 
             // Step 3: convert format level TableInfo to code level TableInfo
@@ -1555,7 +1550,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1569,7 +1564,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1578,7 +1573,7 @@ public class TestCarbonAllDataType
                 return   "date '" + data + "'";
             }
             case "varchar":
-            {//'china'
+            {
                 return "'" + data + "'";
             }
             case "timestamp":
@@ -1592,7 +1587,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1603,7 +1598,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1614,7 +1609,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1625,7 +1620,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1637,9 +1632,10 @@ public class TestCarbonAllDataType
                 return dateString;
             }
             case "smallint":  {
-                // smallint '12'
                 return "smallint '" + data + "'";
             }
+            default:
+                break;
         }
         return data;
     }
@@ -1697,7 +1693,7 @@ public class TestCarbonAllDataType
             hetuServer.execute(inserData);
         }
         catch(Exception e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1736,7 +1732,7 @@ public class TestCarbonAllDataType
 
         }
         catch (IOException | InterruptedException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1785,14 +1781,15 @@ public class TestCarbonAllDataType
     */
     private boolean checkStatusFileForDeleteMarked(String tableName, int updateNumber, int segmentNumber) throws SQLException
     {
+        BufferedReader reader = null;
         try {
             File dir = new File(storePath + "/carbon.store/testdb/" + tableName + "/Metadata");
             File[] tableUpdateStatusFiles = dir.listFiles((d, name) -> name.startsWith("tableupdatestatus"));
             Arrays.sort(tableUpdateStatusFiles);
             Gson gson = new Gson();
-            BufferedReader reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
+            reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
             SegmentUpdateDetails[] segmentUpdateDetails = gson.fromJson(reader, SegmentUpdateDetails[].class);
-            File tableStatusFile = new File(dir.getAbsolutePath() + "/tablestatus");
+            File tableStatusFile = new File(dir.getCanonicalPath() + "/tablestatus");
             reader = new BufferedReader(new FileReader(tableStatusFile));
             LoadMetadataDetails loadMetadataDetails = gson.fromJson(reader, LoadMetadataDetails[].class)[segmentNumber];
             if ((segmentUpdateDetails[0].getSegmentStatus() != null && segmentUpdateDetails[0].getSegmentStatus().toString().equals("Marked for Delete")) &&
@@ -1803,6 +1800,16 @@ public class TestCarbonAllDataType
             hetuServer.execute("drop table if exists testdb." + tableName);
             Assert.fail("Failed to read status files");
         }
+        finally {
+            if (reader != null) {
+                try {
+                    reader.close();
+                }
+                catch (IOException e) {
+                    logger.error(e.getMessage());
+                }
+            }
+        }
         return false;
     }
 
@@ -1822,7 +1829,7 @@ public class TestCarbonAllDataType
                     "/carbon.store/testdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
@@ -1846,7 +1853,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
@@ -1879,7 +1886,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException | InterruptedException e) {
             hetuServer.execute("DROP TABLE if exists testdb.myectable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.myectable");
@@ -1904,7 +1911,7 @@ public class TestCarbonAllDataType
                 FileFactory.mkdirs( storePath + "/carbon.store/mytestDb");
             }
         } catch (IOException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         String location = "'" + "file:///" + storePath + "/carbon.store/mytestDb" + "')" ;
@@ -2487,91 +2494,44 @@ public class TestCarbonAllDataType
     @Test
     public void test_writer_count() throws SQLException
     {
-        hetuServer.execute("drop table if exists testdb.testorders1");
-        hetuServer.execute("drop table if exists testdb.testorders_bak");
         hetuServer.execute("set session task_writer_count=32");
-        hetuServer.execute("set session implicit_conversion=true");
-        hetuServer.execute("CREATE TABLE testdb.testorders1(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
-        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
-        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
-        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
 
-        hetuServer.execute("CREATE TABLE testdb.testorders_bak(orderkey bigint, orderstatus varchar(7), totalprice double, orderdate date)");
-        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+        hetuServer.execute("CREATE TABLE testdb.testWriterCount_32(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
 
-        hetuServer.execute("set session task_writer_count=1");
-        List<Map<String, Object>>  actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
+        verifyRowCount("testdb.testWriterCount_32", 6);
 
-        List<Map<String, Object>>  expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("RESULT", 3); }});
-        }};
-        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
+        verifyRowCount("testdb.testWriterCount_32", 12);
 
-        // checking correct number of files created in Fact
-        File files1 = null;
-        files1 = new File(storePath + "/carbon.store/testdb/testorders_bak/Fact");
-        File[] fileListPart0 = files1.listFiles();//Part0
-
-        if (fileListPart0.length == 1) {
-            File[] Segment_0 = fileListPart0[0].listFiles();
-            if (Segment_0.length == 1) {
-                File[] fileListSegment_0 = Segment_0[0].listFiles();
-                if (fileListSegment_0.length == 6) {
-                    assertEquals("true", "true");
-                }
-                assertEquals(fileListSegment_0.length, 6);
-            }
-            assertEquals(Segment_0.length, 1);
-        }
-        assertEquals(fileListPart0.length, 1);
-
-        //test with 16
-        hetuServer.execute("set session task_writer_count=16");
-        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
-
-        hetuServer.execute("set session task_writer_count=1");
-        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
-
-        expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("RESULT", 6); }});
-        }};
-        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
-
-        //test with 8
-        hetuServer.execute("set session task_writer_count=8");
-        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
-
-        hetuServer.execute("set session task_writer_count=1");
-        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
-
-        expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("RESULT", 9); }});
-        }};
-        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
-
-        //test with 2
-        hetuServer.execute("set session task_writer_count=2");
-        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
-
-        hetuServer.execute("set session task_writer_count=1");
-        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
-
-        expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("RESULT", 12); }});
-        }};
-        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
+        verifyRowCount("testdb.testWriterCount_32", 24);
 
+        String filePath = storePath + "/carbon.store/testdb/testwritercount_32";
         try {
-            hetuServer.execute("set session task_writer_count=32");
-            hetuServer.execute("VACUUM TABLE testdb.testorders_bak AND WAIT");
-            assertEquals(FileFactory.isFileExist(storePath +
-                    "/carbon.store/testdb/testorders_bak/Fact/Part0/Segment_0.1", false), true);
-        } catch (IOException e) {
+            hetuServer.execute("VACUUM TABLE testdb.testWriterCount_32 AND WAIT");
+            assertTrue(FileFactory.isFileExist(filePath + "/Fact/Part0/Segment_0.1"));
         }
+        catch (IOException e) {
+            assertTrue(false, "Unable to read file from table path");
+        }
+        finally {
+            hetuServer.execute("drop table if exists testdb.testWriterCount_32");
+            hetuServer.execute("set session task_writer_count=1");
+        }
+    }
 
-        hetuServer.execute("drop table if exists testdb.testorders1");
-        hetuServer.execute("drop table if exists testdb.testorders_bak");
-        hetuServer.execute("set session task_writer_count=1");
+    private void verifyRowCount(String tableName, int rowCount) throws SQLException
+    {
+        String query = String.format("SELECT COUNT(*) AS result FROM %s", tableName);
+        List<Map<String, Object>>  actualResult = hetuServer.executeQuery(query);
+        List<Map<String, Object>>  expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("result", rowCount); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
     }
 
     private TableInfo getTableInfoFromSchemaFile(String tableName)

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -40,6 +40,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.testng.Assert.assertTrue;
+import static org.testng.AssertJUnit.assertNotNull;
 
 public class TestCarbonAutoVacuum
 {
@@ -99,7 +100,6 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
-        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -145,11 +145,12 @@ public class TestCarbonAutoVacuum
 
         try {
             CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
+            assertNotNull(catalog);
             connector = catalog.getConnector(catalog.getConnectorCatalogName());
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-
+            logger.debug(e.getMessage());
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -215,7 +216,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-
+            logger.debug(e.getMessage());
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -62,7 +62,6 @@ public class TestCarbondataAutoCleanup
     public void setup() throws Exception
     {
         logger.info("Setup begin: " + this.getClass().getSimpleName());
-        String dataPath = rootPath + "/src/test/resources/alldatatype.csv";
 
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
@@ -131,7 +130,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -160,7 +159,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -190,7 +189,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -222,7 +221,7 @@ public class TestCarbondataAutoCleanup
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-
+                logger.debug(exception.getMessage());
             }
         }
         finally {
@@ -255,7 +254,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -286,7 +285,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -315,7 +314,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -345,7 +344,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -375,7 +374,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -404,7 +403,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                e.printStackTrace();
+                logger.error(e.getMessage());
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -15,6 +15,7 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
+import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -36,6 +37,7 @@ import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
+    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -114,7 +116,7 @@ public class TestsWithHiveConnector
         assertEquals(FileFactory.isFileExist(storePath +
                 "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
         hetuServer.execute("DROP TABLE hive.default.parttable");
     }
@@ -141,7 +143,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2013", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
 
         hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
@@ -150,7 +152,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2014", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
 
         hetuServer.execute("DROP TABLE hive.default.parttable2");
@@ -178,7 +180,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "/hive.store/default/parttable3/year=2013", false), true);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
         hetuServer.execute("DROP TABLE hive.default.parttable3");
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -123,14 +123,20 @@ public class HetuTestServer
     public List<Map<String, Object>> executeQuery(String query) throws SQLException
     {
         logger.info(">>>>> Executing Query: " + query);
+        ResultSet rs = null;
         try {
-            ResultSet rs = statement.executeQuery(query);
+            rs = statement.executeQuery(query);
             return convertResultSetToList(rs);
         }
         catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
+        finally {
+            if (rs != null) {
+                rs.close();
+            }
+        }
     }
 
     private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
@@ -186,7 +192,7 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
@@ -197,7 +203,7 @@ public class HetuTestServer
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
         }
         catch (RuntimeException e) {

hetu-clickhouse/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.4.0</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-clickhouse</artifactId>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -345,8 +345,9 @@ public class ClickHouseClient
     }
 
     @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
+    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
     {
+        String newColumnName = inputNewColumnName;
         try (Connection connection = connectionFactory.openConnection(identity)) {
             if (connection.getMetaData().storesUpperCaseIdentifiers()) {
                 newColumnName = newColumnName.toUpperCase(ENGLISH);

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -41,6 +41,7 @@ public class ClickHouseApplyRemoteFunctionPushDown
     /**
      * rewrite the remote function to a executable function in the data source.
      */
+    @Override
     public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
     {
         if (!isConnectorSupportedRemoteFunction(callExpression)) {

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -37,8 +37,9 @@ public class ClickHouseSqlStatementWriter
     }
 
     @Override
-    public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
+    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
     {
+        String functionName = inputFunctionName;
         if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
             functionName = "varPop";
         }

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -205,7 +205,7 @@ public final class ClickHouseServerTest
     {
         String actualTable = tablePattern;
 
-        for (String table : tables) { //tableName + _ + UUID
+        for (String table : tables) {
             int lastIndex = table.lastIndexOf("_");
             if (lastIndex == -1) {
                 continue;

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.4.0</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,16 +47,6 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
-
-        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
-        // as per link above, the default SSLContext will be constructed using the default KeyManager and
-        // default TrustManager. Those can be configured using the following system properties:
-        // javax.net.ssl.keyStore
-        // javax.net.ssl.keyStorePassword
-        // javax.net.ssl.keyStoreType
-        // javax.net.ssl.trustStore
-        // javax.net.ssl.trustStorePassword
-        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,6 +13,7 @@
  */
 package io.hetu.core.common.util;
 
+import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -29,6 +30,8 @@ import java.util.Optional;
 
 public class TrustStore
 {
+    private static final Logger LOGGER = Logger.get(TrustStore.class);
+
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -48,6 +51,7 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
+            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
+            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
+            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.4.0</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-cube</artifactId>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -36,8 +36,7 @@ public class CubeFilter
 
     public CubeFilter(String sourceTablePredicate)
     {
-        this.sourceTablePredicate = sourceTablePredicate;
-        this.cubePredicate = null;
+        this(sourceTablePredicate, null);
     }
 
     public String getSourceTablePredicate()

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -140,13 +140,13 @@ public class CubeStatement
             return this;
         }
 
-        public Builder groupBy(String column)
+        public Builder groupByAddString(String column)
         {
             this.groupBy.add(column);
             return this;
         }
 
-        public Builder groupBy(String... columns)
+        public Builder groupByAddStringList(String... columns)
         {
             this.groupBy.addAll(Arrays.asList(columns));
             return this;

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -34,8 +34,8 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
-                .groupBy("address")
-                .groupBy("name", "nationkey")
+                .groupByAddString("address")
+                .groupByAddStringList("name", "nationkey")
                 .build();
 
         assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.4.0</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,6 +55,7 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
+    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -56,11 +56,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this.catalogName = catalogName;
-        this.schemaName = requireNonNull(schemaName, "schemaName is null");
-        this.tableName = requireNonNull(tableName, "tableName is null");
-        this.limit = requireNonNull(limit, "limit is null");
-        this.pushDownSql = "";
+        this(catalogName,
+                requireNonNull(schemaName, "schemaName is null"),
+                requireNonNull(tableName, "tableName is null"),
+                requireNonNull(limit, "limit is null"),
+                "");
     }
 
     /**
@@ -125,6 +125,7 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
+    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -179,7 +179,7 @@ public class DataCenterPlanOptimizer
             List<RowExpression> pushable = new ArrayList<>();
             List<RowExpression> nonPushable = new ArrayList<>();
 
-            for (RowExpression conjunct : logicalRowExpressions.extractConjuncts(node.getPredicate())) {
+            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
                 try {
                     conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
                     pushable.add(conjunct);

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterQueryGenerator.java

@@ -116,6 +116,20 @@ public class DataCenterQueryGenerator
                     .setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
                     .setSelections(selections)
                     .setFrom(Optional.of(table.toString()));
+
+            String catalogName = dcTableHandle.getCatalogName();
+            if (catalogName != null) {
+                contextBuilder.setRemoteCatalogName(catalogName);
+            }
+            String schemaName = dcTableHandle.getSchemaName();
+            if (schemaName != null) {
+                contextBuilder.setRemoteSchemaName(schemaName);
+            }
+            String tableName = dcTableHandle.getTableName();
+            if (tableName != null) {
+                contextBuilder.setRemoteTablename(tableName);
+            }
+
             // If LIMIT has been push down, add it to context
             if (dcTableHandle.getLimit().isPresent()) {
                 contextBuilder.setLimit(dcTableHandle.getLimit());

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1392,24 +1392,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner queryRunner = null;
+        DistributedQueryRunner distributedQueryRunner = null;
         try {
-            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            queryRunner.installPlugin(new DataCenterPlugin());
-            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            queryRunner.installPlugin(new TpchPlugin());
-            queryRunner.createCatalog("tpch", "tpch", properties);
+            distributedQueryRunner.installPlugin(new DataCenterPlugin());
+            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            distributedQueryRunner.installPlugin(new TpchPlugin());
+            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
 
-            return queryRunner;
+            return distributedQueryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, queryRunner);
+            closeAllSuppress(e, distributedQueryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
+openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,15 +24,17 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-Other audit logging properties include: 
+The following is a detailed description of audit logging properties:
 
-`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
 
-`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
+`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
+`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
 
 Example configuration file:
 
@@ -41,7 +43,6 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
-hetu.event.listener.audit.filecount=1
-hetu.event.listener.audit.limit=100000
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```

hetu-docs/en/admin/extension-execution-planner.md

@@ -0,0 +1,25 @@
+#Extension Physical Execution Planner
+This section describes how to add an extension physical execution planner in openLooKeng. With the extension physical execution planner, openLooKeng can utilize other operator acceleration libraries to speed up the execution of SQL statements.
+
+##Configuration
+To enable extension physical execution feature, the following configs must be added in
+`config.properties`：
+
+``` properties
+extension_execution_planner_enabled=true
+extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
+extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
+```
+
+The above attributes are described below:
+
+- `extension_execution_planner_enabled`: Enable extension physical execution feature.
+- `extension_execution_planner_jar_path`: Set the file path of the extension physical execution jar package.
+- `extension_execution_planner_class_path`: Set the package path of extension physical execution generated class in jar。
+
+
+##Usage
+The below command can control the enablement of extension physical execution feature in WebUI or Cli while running openLooKeng:
+```
+set session extension_execution_planner_enabled=true/false
+```

hetu-docs/en/admin/properties.md

@@ -118,6 +118,20 @@ This section describes the most important config properties that may be used to
 >
 > This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
 
+### `query.suspend-query-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables running query temporary suspension when system is in low resource situation.
+
+### `query.max-suspended-queries`
+
+> -   **Type:** `integer`
+> -   **Default value:** `10`
+>
+> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
+
 ## Spilling Properties
 
 ### `experimental.spill-enabled`
@@ -161,6 +175,30 @@ This section describes the most important config properties that may be used to
 >
 > This config property can be overridden by the `spill_window_operator` session property.
 
+
+### `experimental.spill-build-for-outer-join-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables spill feature for right-outer and full-outer join operations.
+>
+>
+>
+> This config property can be overridden by the `spill_build_for_outer_join_enabled` session property.
+
+### `experimental.inner-join-spill-filter-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables bloom filter based build-side spill matching for probe side spill decision.
+>
+>
+>
+> This config property can be overridden by the `inner_join_spill_filter_enabled` session property.
+
+
 ### `experimental.spill-reuse-tablescan`
 
 > -   **Type:** `boolean`
@@ -179,7 +217,7 @@ This section describes the most important config properties that may be used to
 >
 > Directory where spilled content will be written. It can be a comma separated list to spill simultaneously to multiple directories, which helps to utilize multiple drives installed in the system.
 >
->
+> When `experimental.spiller-spill-to-hdfs` is to `true`, `experimental.spiller-spill-path` must contain only  a single directory.
 >
 > It is not recommended to spill to system drives. Most importantly, do not spill to the drive on which the JVM logs are written, as disk overutilization might cause JVM to pause for lengthy periods, causing queries to fail.
 
@@ -240,6 +278,71 @@ This section describes the most important config properties that may be used to
 >
 > Enables using a randomly generated secret key (per spill file) to encrypt and decrypt data spilled to disk
 
+### `experimental.spill-direct-serde-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables to serialize/read the page directly to/from the stream.
+
+### `experimental.spill-prefetch-read-pages`
+
+> -   **Type:** `integer`
+> -   **Default value:** `1`
+>
+> Sets number of pages prefetched while reading from spilled files.
+
+
+### `experimental.spill-use-kryo-serialization`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables Kryo based serialization for spill to disk, instead of default java serializer.
+
+
+### `experimental.revocable-memory-selection-threshold`
+
+> -   **Type:** `data size`
+> -   **Default value:** `512 MB`
+>
+> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke.
+
+### `experimental.prioritize-larger-spilts-memory-revoke`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `true`
+>
+> Enables to prioritize splits with larger revocable memory.
+
+### `experimental.spill-non-blocking-orderby`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables order by operator to use asynchronous mechanism to spill, i.e it can accumulate input even when a spill is in progress and initiate a secondary spill when the secondary data accumulate exceeds a threshold or when the primary spill is completed, the default value of the threshold is the minimum between 20MB and 5% of available free memory. This property must be used in conjunction with the `experimental.spill-enabled` property.
+>
+>
+>
+> This config property can be overridden by the `spill_non_blocking_orderby` session property.
+
+### `experimental.spiller-spill-to-hdfs`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables spilling into HDFS. When this property is set to `true` the property `experimental.spiller-spill-profile` must be set and also `experimental.spiller-spill-path` must contain only a single path.
+
+### `experimental.spiller-spill-profile`
+
+> -   **Type:** `string`
+> -   **No default value.** Must be set when spilling to hdfs is enabled
+>
+>
+> This property defines the [filesystem](../develop/filesystem.md) profile used to spill. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `experimental.spiller-spill-profile=spill-hdfs`, a profile describing this filesystem `spill-hdfs.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable, refer [filesystem](../develop/filesystem.md) for details).
+>
+> This property is required when `experimental.spiller-spill-to-hdfs` is set to `true`. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the path declared in `experimental.spiller-spill-path` folder in the specified file system.
+
 ## Exchange Properties
 
 Exchanges transfer data between openLooKeng nodes for different stages of a query. Adjusting these properties may help to resolve inter-node communication issues or improve network utilization.
@@ -275,19 +378,8 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
 >
->
->
 > Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
 
-### `exchange.max-error-duration`
-
-> -   **Type:** `duration`
-> -   **Minimum value:** `1m`
-> -   **Default value:** `7m`
->
-> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a failure.
-
-
 ### `sink.max-buffer-size`
 
 > -   **Type:** `data size`
@@ -295,6 +387,113 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
 
+## Failure Recovery handling Properties
+
+### Failure Retry Policies
+
+### `failure.recovery.retry.profile`
+
+> -   **Type:** `String`
+> -   **Default value:** `default`
+>
+> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
+> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
+> The file `test.properties` must contain `failure.recovery.retry.type` specified.
+
+
+### `failure.recovery.retry.type`
+
+> -   **Type:** `String`
+> -   **Default value:** `timeout`
+>
+> The failure detection mechanism in use. Default is timeout based failure detection.
+>
+#### `timeout` based failure detection. 
+> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
+> Additional properties `max.error.duration` can be defined for this type of failure detection.
+>
+#### `max-retry` based failure detection. 
+> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
+> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection. 
+> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
+
+### `max.error.duration`
+> -   **Type:** `duration`
+> -   **Default value:** `300s`
+>
+> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
+
+
+### `max.retry.count`
+
+> -   **Type:** `integer`
+> -   **Default value:** `100`
+>
+> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. 
+> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster. 
+> This property is used only for `max-retry` based failure detection profiles. 
+> The minimum value for this parameter is 100.
+
+### Gossip Protocol Configurations for Failure Detection
+
+### `failure-detection-protocol`
+
+>- **Type:** String
+>- **Default value:** `heartbeat`
+>
+> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
+> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
+> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
+
+### `failure-detector.heartbeat-interval`
+
+>- **Type:** Duration
+>- **Default value:** `500ms` (500 miliseconds)
+>
+> This is the interval of gossip between two nodes in the cluster.
+> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
+> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
+> In workers, this property can be left to use the default value.
+>
+### `failure-detector.worker-gossip-probe-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `5s` (5 seconds)
+>
+> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
+> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
+> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
+> This parameter should have higher value than `failure-detector.heartbeat-interval`.
+>
+### `failure-detector.coordinator-gossip-probe-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `5s` (5 seconds)
+>
+> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
+> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
+> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
+> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
+>
+### `failure-detector.coordinator-gossip-collate-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `2s` (2 seconds)
+>
+> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
+> This property has to be specified only for the coordinator.
+> This parameter should have higher value than `failure-detector.heartbeat-interval`.
+>
+### `failure-detector.gossip-group-size`
+>
+> - **Type:** Integer
+>- **Default value:** `Integer.MAX_VALUE`
+>
+> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
+> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
+> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
+> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
+
 ## Task Properties
 
 ### `task.concurrency`
@@ -679,7 +878,7 @@ helps with cache affinity scheduling.
 > Auto-Vacuum enables the system to automatically manage vacuum jobs by constantly monitoring the tables which needs vacuum in order to maintain optimal performance.
 > Engine gets the tables from data sources that are eligible for vacuum and trigger vacuum operation for those tables.
 
-### `auto-vacuum.enabled:`
+### `auto-vacuum.enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
@@ -749,15 +948,25 @@ helps with cache affinity scheduling.
 > -   **Default value:** `5m`
 >
 > The maximum time coordinator waits for remote-task related error to be resolved before it's considered a failure.
+>
+> Note:
+> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
 
-## Distributed Snapshot
+## Query Recovery
+
+### `recovery_enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
 
 ### `snapshot_enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
 >
-> This session property is used to enable or disable the distributed snapshot functionality.
+> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
 
 ### `hetu.experimental.snapshot.profile`
 
@@ -769,20 +978,67 @@ helps with cache affinity scheduling.
 >
 > This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
 
-### `hetu.snapshot.maxRetries`
+### `hetu.recovery.maxRetries`
 
 > -   **Type:** `int`
 > -   **Default value:** `10`
 >
 > This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
 >
-> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
+> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
 
-### `hetu.snapshot.retryTimeout`
+### `hetu.recovery.retryTimeout`
 
 > -   **Type:** `duration`
 > -   **Default value:** `10m` (10 minutes)
 >
 > This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
 >
-> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.
+> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
+
+### `hetu.snapshot.useKryoSerialization`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables Kryo based serialization for snapshot, instead of default java serializer.
+
+### `experimental.eliminate-duplicate-spill-files`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables elimination of duplicate spill files storage as part of snapshot capture.
+
+
+
+## HTTP Client Configurations
+
+### `http.client.idle-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `30s` (30 seconds)
+>
+> This property defines the time for which a given http client shall stay connected without any operations performed over it.
+> After the specified time elapse with no activity, then the client connection is closed and related resources are released.
+>
+> (Note: this parameter should be configured with higher time when in high load environment)
+
+### `http.client.request-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `10s` (10 seconds)
+>
+> This property defines the time threshold for a given http client for which response should be received.
+> After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
+>
+> (Note: this parameter should be configured with higher time when in high load environment)
+
+## Connector Properties configuration
+
+### `case-insensitive-name-matching`
+>
+> -    **Type:** `boolean`
+> -    **Default value:** `false`
+>
+> Case-insensitive matching between database and collection names. The default is case sensitive.

hetu-docs/en/admin/reliable-execution.md

@@ -11,9 +11,9 @@ To achieve better performance while maintaining execution reliability, the *dist
 
 As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
 
-## Enable Distributed Snapshot
+## Enable Recovery framework
 
-Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
+Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
 
 ## Requirements
 
@@ -37,7 +37,7 @@ When a query that does not meet the above requirements is submitted with distrib
 
 ## Detection
 
-Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration) configuration.
+Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
 
 ## Storage Considerations
 
@@ -55,8 +55,20 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
 
 The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
 
-It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
+It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
+
+## Snapshot statistics
+
+Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
+
+Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query.  These statistics are displayed for all snapshots and for last snapshot separately.
+
+Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots.  Restore statistics are displayed only when there is restore(recovery) happened during the query.
+
+Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details 
+
+![](../images/snapshot_statistics.png)
 
 ## Configurations
 
-Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).
+Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).

hetu-docs/en/admin/spill.md

@@ -34,6 +34,10 @@ saturation of the configured spill paths.
 
 openLooKeng treats spill paths as independent disks (see [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD)), so there is no need to use RAID for spill.
 
+
+## Spill To HDFS
+Spilling directly into HDFS is also possible for that `experimental.spiller-spill-to-hdfs` needs to be set to `true`, `experimental.spiller-spill-profile` needs to be set and `spiller-spill-path` must contain only a single directory when we intend to spill into HDFS. (refer `experimental.spiller-spill-to-hdfs` and  `experimental.spiller-spill-profile` properties for more details )
+
 ## Spill Compression
 
 
@@ -61,14 +65,17 @@ When the build table is partitioned, the spill-to-disk mechanism can decrease th
 
 With this mechanism, the peak memory used by the join operator can be decreased to the size of the largest build table partition. Assuming no data skew, this will be `1 / task.concurrency` times the size of the whole build table.
 
+Note: spill-to-disk is not supported for Cross Join.
+
 ### Aggregations
 
 Aggregation functions perform an operation on a group of values and return one value. If the number of groups you\'re aggregating over is large, a significant amount of memory may be needed. When spill-to-disk
-is enabled, if there is not enough memory, intermediate cumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
+is enabled, if there is not enough memory, intermediate accumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
 
 ### Order By
 
 If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
+Generally when a spill is in progress the operator is blocked from taking inputs, but when `experimental.spill-non-blocking-orderby` is set to `true` order by uses asynchronous mechanism to spill (see`experimental.spill-non-blocking-orderby`).
 
 ### Window functions
 

hetu-docs/en/admin/web-interface.md

@@ -33,4 +33,54 @@ and statistics about the query is available by clicking the *JSON* link. These v
 > -   **Allowed values:** `true`, `false`
 > -   **Default value:** `false`
 >
-> Insecure authentication over HTTP is disabled by default. This could be overridden via "hetu.queryeditor-ui.allow-insecure-over-http" property of "etc/config.properties" (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
+> Insecure authentication over HTTP is disabled by default. This could be overridden via `hetu.queryeditor-ui.allow-insecure-over-http` property of `etc/config.properties` (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
+
+### `hetu.queryeditor-ui.execution-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `100 DAYS`
+>
+> UI Execution timeout is set to 100 days as default. This could be overridden via `hetu.queryeditor-ui.execution-timeout` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.max-result-count`
+
+> - **Type:** `int`
+> - **Default value:** `1000`
+>
+> UI max result count is set to 1000 as default. This could be overridden via `hetu.queryeditor-ui.max-result-count` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.max-result-size-mb`
+
+>- **Type:** `size`
+>- **Default value:** `1GB`
+>
+> UI max result size is set to 1 GB as default. This could be overridden via `hetu.queryeditor-ui.max-result-size-mb` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.session-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `1 DAYS`
+>
+> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
+
+### `hetu.queryhistory.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `1000`
+>
+> The maximum number of query history stored by openLooKeng. This could be overridden via "hetu.queryhistory.max-count" of "etc/config.properties".
+
+### `hetu.collectionsql.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `100`
+>
+> The Maximum number of SQL collected by each user. This could be overridden via "hetu.collectionsql.max-count" of "etc/config.properties".
+
+## Remarks
+
+The max length of the favorite SQL is 600 by default. You can modify it through the following steps:
+
+1. Login MySQL database according to the JDBC configuration of `hetu-metastore.properties`
+
+2. Select table hetu_favorite, execute script `alter table hetu_favorite modify query varchar(2000) not null;` to modify the max length of the favorite SQL.

hetu-docs/en/connector/Hudi.md

@@ -1,5 +1,8 @@
 # Hudi  Connector
 
+### Release Notes
+Currently Hudi only supports version 0.7.0.
+
 ### Hudi Introduction
 
 Apache Hudi is a fast growing data lake storage system that helps organizations build and manage petabyte-scale data lakes. Hudi enables storing vast amounts of data on top of existing DFS compatible storage while also enabling stream processing in addition to typical batch-processing. This is made possible by providing two new primitives. Specifically,

hetu-docs/en/connector/elasticsearch.md

@@ -192,4 +192,4 @@ The array fields of this structure can be defined by using the following command
     
 ## Limitations
 1. openLooKeng does not support to query table in Elasticsearch which has duplicated columns, such as column "name" and "NAME";
-2. opneLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.
+2. openLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.

hetu-docs/en/connector/hive.md

@@ -620,6 +620,36 @@ The following operations are not supported when `avro_schema_url` is set:
     columns are not supported in `CREATE TABLE`.
 -   `ALTER TABLE` commands modifying columns are not supported.
 
+## Drop Column Behavior
+
+Syntax supported for Drop Column is as follows:
+
+```sql
+ALTER TABLE 'name' DROP COLUMN 'column_name'
+```
+
+In case of Hive connector, DROP COLUMN drops column which is at the end of existing columns. Hive doesn't support DROP COLUMN syntax, however closest DDL supported by Hive which enumerates the above is REPLACE COLUMNS. REPLACE COLUMNS removes all existing columns and adds the new set of columns whereas DROP COLUMN does remove all existing columns and add the same set of columns excluding column specified in the query, manifesting as it dropped the column.
+
+For example, consider a table with columns **a**, **b** and **c**.
+
+```sql
+lk:default> SELECT * FROM hive_table;
+ a | b  |  c
+---+----+-----
+ 1 | 10 | 100
+(1 row)
+```
+
+ On dropping column **a**:
+
+```sql
+lk:default> SELECT * FROM hive_table;
+  b | c
+---+----
+ 1 | 10
+(1 row)
+```
+
 ## Procedures
 
 
@@ -716,7 +746,7 @@ Drop a schema:
 DROP SCHEMA hive.web
 ```
 
-## Metastore Cache:
+## Metastore Cache
 
 Hive connector maintains a metastore cache to service the metastore request faster to various operations. Loading, reloading and retention times of the cache entries can be configured in `hive.properties`.
 
@@ -740,7 +770,7 @@ REFRESH META CACHE
 Additionally, metadata cache refresh command can be used to reload the metastore cache by user.
 
 
-## Performance tuning notes:
+## Performance tuning notes
 
 #### INSERT
 

hetu-docs/en/connector/kafka-tutorial.md

@@ -565,4 +565,4 @@ lk:default> SELECT created_at, raw_date FROM (
 (5 rows)
 ```
 
-The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or miilliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.
+The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or milliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.

hetu-docs/en/connector/memory.md

@@ -37,7 +37,7 @@ memory.spill-path=/opt/hetu/data/spill
   hetu.metastore.cache.type=local
   ```
   ##### Multi-Node Setup
-  - This section will give an example configuration for Memory Connector an a cluster with more than one node.
+  - This section will give an example configuration for Memory Connector and a cluster with more than one node.
   - Create a file `etc/catalog/memory.properties` with the following information:
   ``` properties
   connector.name=memory
@@ -80,10 +80,10 @@ memory.spill-path=/opt/hetu/data/spill
 **Note:**
 - `spill-path` should be set to a directory with enough free space to hold
  the table data.
-- See **Configuration Properties** section for additional properties and
+- See [**Configuration Properties**](#configuration-properties) section for additional properties and
  details.
-- In `etc/config.properties` ensure that `task.writer-count` is set to
- `>=` number of nodes in the cluster running openLooKeng. This will help
+- In `etc/config.properties` ensure that `task.writer-count` is set
+ `>=` to number of nodes in the cluster running openLooKeng. This will help
   distribute the data uniformly between all the workers.
 
 Examples
@@ -112,17 +112,45 @@ Create a table using the Memory Connector with sorting, indices and spill compre
     CREATE TABLE memory.default.nation
     WITH (
         sorted_by=array['nationkey'],
-        index_columns=array['name', 'regionkey'],
+        partitioned_by=array['regionkey'],
+        index_columns=array['name'],
         spill_compression=true
     )
     AS SELECT * from tpch.tiny.nation;
 
 After table creation completes, the Memory Connector will start building indices and sorting data in the background. Once the processing is complete any queries using the sort or index columns will be faster and more efficient.
 
-For now, `sorted_by` only accepts a single column.
+For now, `sorted_by` and `partitioned_by` only accepts a single column.
 
+Memory and Disk Usage via JMX
+-----------------------------
+JMX can be used to show memory and disk usage of memory connector tables
+Please refer to [JMX Connector](./jmx.md) for setup
 
-Configuration Properties
+The `io.prestosql.plugin.memory.data:name=MemoryTableManager` table of `jmx.current` contains information on all the tables' memory and disk usage size in bytes
+
+    SELECT * FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
+
+```
+ currentbytes | alltablesdiskbyteusage | alltablesmemorybyteusage |   node   |                       object_name                       
+--------------+------------------------+--------------------------+----------+---------------------------------------------------------
+           23 |                   3456 |                       23 | example1 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
+          253 |                   8713 |                      667 | example2 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
+```
+
+Not all tables will be in memory since some may have been spilled to disk. `currentbytes` column will show the current memory occupied by tables which are in memory now.
+
+The usage for each node is shown as a separate row, aggregation functions can be utilized to show total usage across the cluster. For example, to view total disk or memory usage on all nodes, run:
+
+    SELECT sum(alltablesdiskbyteusage) as totaldiskbyteusage, sum(alltablesmemorybyteusage) as totalmemorybyteusage FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
+
+```
+totaldiskbyteusage | totalmemorybyteusage
+-------------------+---------------------
+             12169 |                  690
+```
+
+# Configuration Properties
 ------------------------
 
 | Property Name                         | Default Value   | Required| Description               |
@@ -133,31 +161,50 @@ Configuration Properties
 | `memory.max-page-size                `  | 512KB         | No      | Memory limit for each page. Default value is recommended.|
 | `memory.logical-part-processing-delay`  | 5s            | No      | The delay between when the table is created/updated and LogicalPart processing starts. Default value is recommended.|
 | `memory.thread-pool-size             `  | Half of threads available to the JVM | No      | Maximum threads to allocate for background processing (e.g. sorting, index creation, cleanup, etc)|
+| `memory.table-statistics-enabled`       | False         | No      | When enabled, user can run analyze to collect statistics and leverage that information for accelerating queries.|
 
-Path whitelist：`["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
+Path whitelist:  `["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
 
 Additional WITH properties
---------------
+--------------------------
 Use these properties when creating a table with the Memory Connector to make queries faster.
 
 | Property Name            | Argument type             | Requirements                     | Description|
 |--------------------------|---------------------------|----------------------------------|------------|
 | sorted_by                | `array['col']`            | Maximum of one column. Column type must be comparable.  | Sort and create indexes on the given column|
+| partitioned_by           | `array['col']`            | Maximum of one column. | Partition the table on the given column|
 | index_columns            | `array['col1', 'col2']`   | None                             | Create indexes on the given column|
 | spill_compression        | `boolean`           | None                             | Compress data when spilling to disk|
 
 
 Index Types
 --------------
-These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`. If a query operator is not supported by a particular index you can still
-use that operator, but the query will not benefit from the index.
+These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`.
+If a query operator is not supported by a particular index, you can still use that operator, but the query will not benefit from the index.
 
 | Index ID                          |Built for Columns In | Supported query operators             |
 |-----------------------------------|----------------------------------------|---------------------------------------|
-| Bloom   | `sorted_by,index_columns`                                 | `=` `IN`                             |                   
+| Bloom   | `index_columns`                                 | `=` `IN`                             |                   
 | MinMax  | `sorted_by,index_columns`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 | Sparse  | `sorted_by`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 
+
+Using statistics
+-----------------
+If the statistic configuration is enabled, you can refer to the example below to use it.
+
+Create a table using the Memory Connector:
+
+    CREATE TABLE memory.default.nation AS
+    SELECT * from tpch.tiny.nation;
+
+Run Analyze to collect the statistic information:
+
+    ANALYZE memory.default.nation;
+
+And then run the queries. Note that currently we do not support automatic statistic update, so you will need to run ANALYZE again if the table is updated.
+
+
 Developer Information
 ----------------------------
 
@@ -166,20 +213,11 @@ This section outlines the overall design of the Memory Connector, as shown in th
 ![Memory Connector Overall Design](../images/memory-connector-design.png)
 
 ### Scheduling Process
-The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng.
-In the Memory Connector, each worker has a number of LogicalParts.
-During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion.
-Table data will be automatically spilled to disk as part of a background process as well. 
-If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule.
-HetuMetastore is used to persist table metadata.
-At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
+The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng. In the Memory Connector, each worker has several LogicalParts. During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion. Table data will be automatically spilled to disk as part of a background process as well.  If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule. HetuMetastore is used to persist table metadata. At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
 
 ### LogicalPart
-As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data.
-The sorting and indexing are handled in a background process allowing faster querying,
-but the table is still queriable during processing.
-LogicalParts have a maximum configurable size (default 256 MB). 
-New LogicalParts are created once the previous one is full.
+As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data. The sorting and indexing are handled in a background process allowing faster querying,
+but the table is still queriable during processing. LogicalParts have a maximum configurable size (default 256 MB).  New LogicalParts are created once the previous one is full.
 
 
 ### Indices
@@ -213,4 +251,5 @@ Limitations and known Issues
 
 - Without State Store and Hetu Metastore with global cache, after `DROP TABLE`, memory is not released immediately on the workers. It is released on the next `CREATE TABLE` operation.
 - Currently only a single column in ascending order is supported by `sorted_by`
-- If a CTAS (CREATE TABLE AS) query fails or is cancelled, an invalid table will remain. This table must be dropped manually.
+- If a CTAS (CREATE TABLE AS) query fails or is cancelled, an invalid table will remain. This table must be dropped manually.
+- And we support BOOLEAN, All INT Types, CHAR, VARCHAR, DOUBLE, REAL, DECIMAL, DATE, TIME, UUID types as partition keys.

hetu-docs/en/connector/omnidata.md

@@ -0,0 +1,103 @@
+
+OmniData Connector
+==============
+
+## Overview
+
+The OmniData connector allows querying data stored in the remote Hive data warehouse.
+It pushes the operators of openLooKeng down to the storage node to achieve near-data calculation, thereby reducing the amount of network transmission data and improving computing performance.
+
+For more information, please see: [OmniData](https://www.hikunpeng.com/en/developer/boostkit/big-data?accelerated=3) and [OmniData connector](https://github.com/kunpengcompute/omnidata-openlookeng-connector).
+
+## Supported File Types
+
+
+The following file types are supported for the OmniData connector:
+
+-   ORC
+-   Parquet
+-   Text
+
+## Configuration
+
+Create `etc/catalog/omnidata.properties` with the following configurations, replacing `example.net:9083` with the correct host and port for your Hive metastore Thrift service:
+
+``` properties
+connector.name=omnidata-openlookeng
+hive.metastore.uri=thrift://example.net:9083
+```
+
+### HDFS Configuration
+
+For basic setups, openLooKeng configures the HDFS client automatically and does not require any configuration files. In some cases, such as when using federated HDFS or NameNode high availability, it is necessary to specify additional HDFS client options in order to access your HDFS cluster. To do so, add the `hive.config.resources` property to reference your HDFS config files:
+
+``` properties
+hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop/conf/hdfs-site.xml
+```
+
+Only specify additional configuration files if necessary for your setup. We also recommend reducing the configuration files to have the minimum set of required properties, as additional properties may cause problems.
+
+The configuration files must exist on all openLooKeng nodes. If you are referencing existing Hadoop config files, make sure to copy them to any openLooKeng nodes that are not running Hadoop.
+
+## OmniData Configuration Properties
+
+| Property Name                   | Description                                                  | Default |
+| ------------------------------- | ------------------------------------------------------------ | ------- |
+| hive.metastore                  | The type of Hive metastore                                   | thrift  |
+| hive.config.resources           | An optional comma-separated list of HDFS configuration files. These files must exist on the machines running openLooKeng. Only specify this if absolutely necessary to access HDFS. Example: `/etc/hdfs-site.xml` |         |
+| hive.omnidata-enabled           | Allows push-down operators to execute on the storage side. If disabled, all operators will not be pushed down. | true    |
+| hive.min-offload-row-number     | If the number of rows in the table is less than the threshold, all operators of the table will not be pushed down. | 500     |
+| hive.filter-offload-enabled     | Allows the filter operator to be pushed down to the storage side. If disabled, the filter operator will not be pushed down. | true    |
+| hive.filter-offload-factor      | Only when the selection rate of the filter operator is less than the threshold, it will be pushed down. | 0.25    |
+| hive.aggregator-offload-enabled | Allows the aggregator operator to be pushed down to the storage side. If disabled, the aggregator operator will not be pushed down. | true    |
+| hive.aggregator-offload-factor  | Only when the aggregation rate of the aggregator operator is less than the threshold, it will be pushed down. | 0.25    |
+
+For more configuration, please refer to the [Hive Configuration Properties](./hive.md#Hive Configuration Properties) chapter.
+
+### Querying OmniData
+
+The SQL query plan after some operators are pushed down:
+
+```sql
+lk:tpch_flat_orc_date_1000> explain select sum(l_extendedprice * l_discount) as revenue
+				 		 -> from
+				 		 -> lineitem
+				 		 -> where
+				 		 -> l_shipdate >= DATE '1993-01-01'
+				 		 -> and l_shipdate < DATE '1994-01-01'
+				 		 -> and l_discount between 0.06 - 0.01 and 0.06 + 0.01
+				 		 -> and l_quantity < 25;
+				 							Query Plan
+------------------------------------------------------------------------------------------------------
+Output[revenue]
+│ Layout: [sum:double]
+│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
+│ revenue := sum
+└─ Aggregate(FINAL)
+│ Layout: [sum:double]
+│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
+│ sum := sum(sum_4)
+└─ LocalExchange[SINGLE] ()
+│ Layout: [sum_4:double]
+│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
+└─ RemoteExchange[GATHER]
+│ Layout: [sum_4:double]
+│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
+└─ Aggregate(PARTIAL)
+│ Layout: [sum_4:double]
+│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 0B}
+│ sum_4 := sum(expr)
+└─ ScanProject[table = hive:tpch_flat_orc_date_1000:lineitem offload={ filter=[AND(AND(BETWEEN(l_discount, 0.05, 0.07), LESS_THAN(l_quantity, 25.0)), AND(GREATER_THAN_OR_EQUAL(l_shipdate, 8401), LESS_THAN(l_shipdate, 8766)))]} ]
+ Layout: [expr:double]
+ Estimates: {rows: 5999989709 (50.29GB), cpu: 100.58G, memory: 0B, network: 0B}/{rows: 5999989709 (50.29GB), cpu: 150.87G, memory: 0B, network: 0B}
+ expr := (l_extendedprice) * (l_discount)
+ l_extendedprice := l_extendedprice:double:5:REGULAR
+ l_discount := l_discount:double:6:REGULAR
+```
+
+
+## OmniData Connector Limitations
+
+
+- The OmniData service needs to be deployed on the storage node.
+- Only the pushdown of Filter, Aggregator, and Limit operators are supported.

hetu-docs/en/connector/opengauss.md

@@ -45,6 +45,95 @@ Finally, you can access the `hetutb` table in the `public` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `opengauss` in the above examples.
 
+## openGauss Update/Delete Support
+
+### Create openGauss Table
+
+Example：
+
+```sql
+CREATE TABLE opengauss_table (
+    id int,
+    name varchar(255));
+```
+
+### INSERT on openGauss tables
+
+Example：
+
+```sql
+INSERT INTO opengauss_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### UPDATE on openGauss tables
+
+Example：
+
+```sql
+UPDATE opengauss_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
+
+SELECT result before UPDATE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+SELECT result after UPDATE
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+id | name
+----+------
+2 | Bob
+1 | Tim
+(2 rows)
+```
+
+### DELETE on openGauss tables
+
+Example：
+
+```sql
+DELETE FROM opengauss_table
+  WHERE id=2;
+```
+
+Above example delete the rows with column `id` having value `2`.
+
+SELECT result before DELETE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+SELECT result after DELETE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 ****Note:****
 
 > - When the compatibility type of the openGuass database is O (DBCOMPATIBILITY = A), the `Date` data type is not supported.
@@ -64,4 +153,4 @@ openGauss Connector Limitations
 
 The following SQL statements are not yet supported:
 
-[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/postgresql.md

@@ -46,9 +46,98 @@ Finally, you can access the `clicks` table in the `web` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `postgresql` in the above examples.
 
+## PostgreSQL Update/Delete Support
+
+### Create PostgreSQL Table
+
+Example：
+
+```sql
+CREATE TABLE postgresql_table (
+    id int,
+    name varchar(255));
+```
+
+### INSERT on PostgreSQL tables
+
+Example：
+
+```sql
+INSERT INTO postgresql_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### UPDATE on PostgreSQL tables
+
+Example：
+
+```sql
+UPDATE postgresql_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
+
+SELECT result before UPDATE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+SELECT result after UPDATE
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+id | name
+----+------
+2 | Bob
+1 | Tim
+(2 rows)
+```
+
+### DELETE on PostgreSQL tables
+
+Example：
+
+```sql
+DELETE FROM postgresql_table
+  WHERE id=2;
+```
+
+Above example delete the rows with column `id` having value `2`.
+
+SELECT result before DELETE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+SELECT result after DELETE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 PostgreSQL Connector Limitations
 --------------------------------
 
 The following SQL statements are not yet supported:
 
-[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/redis.md

@@ -0,0 +1,173 @@
+Redis Connector
+====================
+Overview
+--------
+this connector allows the use of Redis key/value pair is presented as a single row in openLooKeng.
+
+**Note**
+
+*In Redis,key/value pair can only be mapped to string or hash value types.keys can be stored in a zset,then keys can split into multiple slice*
+
+*Support Redis 2.8.0 or higher*
+
+Configuration
+-------------
+To configure the Redis connector, create a catalog properties file `etc/catalog/redis.properties` with the following contents, replacing the properties as appropriate:
+``` properties
+connector.name=redis
+redis.table-names=schema1.table1,schema1.table2
+redis.nodes=host1:port
+```
+### Multiple Redis Servers
+You can have as many catalogs as you need. If you have additional
+Redis servers, simply add another properties file to ``etc/catalog``
+with a different name, making sure it ends in ``.properties``.
+For example, if you name the property file `sales.properties`, openLooKeng will create a catalog named `sales` using the configured connector.
+
+Configuration properties
+------------------------
+The following configuration properties are available:
+
+| Property Name                                                    | Description                                                                                                           |
+|:-----------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
+| `redis.table-names`                                              | List of all tables provided by the catalog                                                                            |
+| `redis.default-schema`                                           | Default schema name for tables  (default `default`)                                                                   |
+| `redis.nodes`                                                    | List of nodes in the Redis server                                                                                     |
+| `redis.connect-timeout`                                          | Timeout for connecting to the  Redis server (ms) (default 2000)                                                       |
+| `redis.scan-count`                                               | The number of keys obtained from each scan for string and hash value types (default 100)                              |
+| `redis.key-prefix-schema-table`                                  | Redis keys have schema-name:table-name prefix   (default  false)                                                      |
+| `redis.key-delimiter`                                            | Delimiter separating schema_name and table_name if redis.key-prefix-schema-table is used (default `:`)                |
+| `redis.table-description-dir`                                    | Directory containing table description files (default `etc/redis/`)                                                   |
+| `redis.hide-internal-columns`                                    | Whether internal columns are shown in table metadata or not. (default true)                                           |
+| `redis.database-index`                                           | Redis database index  (default 0)                                                                                     |
+| `redis.password`                                                 | Redis server password  (default null)                                                                                 |
+| `redis.table-description-interval`                               | the interval of flush description files (ms) (default no flush,table description will be memoized without expiration) |
+
+
+Internal columns
+----------------
+
+| Column name        | Type    | Description                                                                                                                              |
+|:-------------------| :------ |:-----------------------------------------------------------------------------------------------------------------------------------------|
+| `_key`             | VARCHAR  | Redis key.                                                                                                                               |
+| `_value`           | VARCHAR   | Redis value corresponding to the key                                                                                                     |
+| `_key_length`   | BIGINT  | Number of bytes in the key.                                                                                                              |
+| `_key_corrupt`     | BOOLEAN  | True if the decoder could not decode the key for this row. When true, data columns mapped from the key should be treated as invalid.     |
+| `_value_corrupt`   | BOOLEAN | True if the decoder could not decode the value for this row. When true, data columns mapped from the value should be treated as invalid. |
+
+
+Table Definition Files
+----------------------
+For openLooKeng, every key/value pair  must be mapped into columns to allow queries against the data. It is like kafka conntector,so you can refer to kafka-tutorial
+
+A table definition file consists of a JSON definition for a table. The name of the file can be arbitrary but must end in `.json`.
+
+for example,there is a nation.json
+``` json
+{
+    "tableName": "nation",
+    "schemaName": "tpch",
+    "key": {
+        "dataFormat": "raw",
+        "fields": [
+            {
+                "name": "redis_key",
+                "type": "VARCHAR(64)",
+                "hidden": "true"
+            }
+        ]
+    },
+    "value": {
+        "dataFormat": "json",
+        "fields": [
+            {
+                "name": "nationkey",
+                "mapping": "nationkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "name",
+                "mapping": "name",
+                "type": "VARCHAR(25)"
+            },
+                        {
+                "name": "regionkey",
+                "mapping": "regionkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "comment",
+                "mapping": "comment",
+                "type": "VARCHAR(152)"
+            }
+       ]
+    }
+}
+```
+In redis,such data exists
+```shell
+127.0.0.1:6379> keys tpch:nation:*
+ 1) "tpch:nation:2"
+ 2) "tpch:nation:4"
+ 3) "tpch:nation:16"
+ 4) "tpch:nation:18"
+ 5) "tpch:nation:10"
+ 6) "tpch:nation:17"
+ 7) "tpch:nation:1"
+```
+```shell
+127.0.0.1:6379> get tpch:nation:1
+"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
+```
+Now we can use redis connector get data from redis,(redis_key don't show,because we set "hidden": "true" ) 
+```shell
+lk> select * from redis.tpch.nation;
+ nationkey |      name      | regionkey |                                                      comment                                                       
+-----------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
+         3 | CANADA         |         1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold              
+         9 | INDONESIA      |         2 |  slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 
+        19 | ROMANIA        |         3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account    
+         2 | BRAZIL         |         1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special   
+```
+**Note**
+
+*if redis.key-prefix-schema-table is false (default is false),all keys in redis will be mapped to table's key,no matching occurs*
+
+Please refer to the `kafka-tutorial` for the description of the ``dataFormat`` as well as various available decoders.
+
+In addition to the above Kafka types, the Redis connector supports ``hash`` type for the ``value`` field which represent data stored in the Redis hash.
+Redis connector use `hgetall key` to get data.
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "value": {
+        "dataFormat": "hash",
+        "fields": [
+          ...
+        ]
+      }
+    }
+```
+
+the Redis connector supports ``zset`` type for the ``key`` field which represent key stored in the Redis zset. 
+if and only if ``zset`` is used as key datafomart,the split is truly supported , because we can use `zrange zsetkey split.start split.end` to get keys of a split.
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "key": {
+        "dataFormat": "zset",
+        "name": "zsetkey", //zadd zsetkey score member
+        "fields": [
+            ...
+        ]
+      }
+    }
+```
+
+Redis Connector Limitations
+---------------------------
+only support read operation,don't support write operation.
+
+

hetu-docs/en/develop/_index.md

@@ -1,5 +1,5 @@
 
 # Developer Guide
 
-openLooKeng is based on Trino(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
+openLooKeng is based on Trino 316(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
 

hetu-docs/en/develop/connectors.md

@@ -21,10 +21,10 @@ This interface is too big to list in this documentation, but if you are interest
 connector. If your underlying data source supports schemas, tables and columns, this interface should be straightforward to implement. If you are attempting to adapt something that is not a relational database (as
 the Example HTTP connector does), you may need to get creative about how you map your data source to openLooKeng\'s schema, table, and column concepts.
 
-### ConnectorSplitManger
+### ConnectorSplitManager
 
 The split manager partitions the data for a table into the individual chunks that openLooKeng will distribute to workers for processing. For example, the Hive connector lists the files for each Hive partition and creates
-one or more split per file. For data so urces that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.
+one or more split per file. For data sources that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.
 
 ### ConnectorRecordSetProvider
 

hetu-docs/en/develop/externalfunction-registration-pushdown.md

@@ -3,7 +3,7 @@ External Function Registration and Push Down
 
 Introduction
 ------------
-The connector can register `external function` into openLooKeng. In Jdbc connector, openLookeng can push them down to data source which support to execute those functions.
+The connector can register `external function` into openLooKeng. In Jdbc connector, openLooKeng can push them down to data source which support to execute those functions.
 
 Function Registration in Connector
 ----------------------------------

hetu-docs/en/develop/functions.md

@@ -216,7 +216,7 @@ An in-depth look at the various annotations relevant to writing an aggregation f
     
 -   `@CombineFunction`:
 
-    The `@CombineFunction` annotation declares the function used to ombine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
+    The `@CombineFunction` annotation declares the function used to combine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
     the results into the first one (in the above example, just by adding them together).
     
 -   `@OutputFunction`:

hetu-docs/en/develop/getting-started.md

@@ -5,7 +5,7 @@
 
 * Mac OS X or Linux
 * Java 8 Update 161 or higher (8u161+), 64-bit. Both Oracle JDK and OpenJDK are supported.
-* AArch64 ([Bisheng JDK 11 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
+* AArch64 ([Bisheng JDK 1.8.262 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
 * Maven 3.3.9+ (for building)
 * Python 2.4+ (for running with the launcher script)
 

hetu-docs/en/develop/types.md

@@ -13,7 +13,7 @@ Below is a high level overview of the `Type` interface, for more details see the
     
 -   Native encoding:
 
-    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the unix  epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
+    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the Unix epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
     
 -   Type signature:
 

hetu-docs/en/faq/faq.md

@@ -40,6 +40,10 @@
    >
    > openLooKeng Bilibili channel:  https://space.bilibili.com/627629884   
 
+8. Which version of Trino is openLooKeng developed on?
+
+   > Based on Trino 316 version development.
+
 ## Functions
 
 1. What connectors does the openLooKeng support?

hetu-docs/en/functions/conversion.md

@@ -106,7 +106,7 @@ Obviously, not all data types are compatible with each other, below table lists
 
 **Note:**
 
-- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. please refer to below item.
+- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. Please refer to below item.
 - N: standard for not support implicit convert
 
 (1): BOOLEAN-\>NUMBER the converted result can be only 0 or 1
@@ -123,23 +123,23 @@ Obviously, not all data types are compatible with each other, below table lists
 
 (7): VARCHAR-\>BOOLEAN only \'0\',\'1\',\'TRUE\',\'FALSE\' can be converted. Others will be failed
 
-(8): VARCHAR-\>DECIMAL conversion will fail when its not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
+(8): VARCHAR-\>DECIMAL conversion will fail when it's not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
 
 (9): VARCHAR-\>CHAR if length of VARCHAR is larger than CHAR, it will be cut off.
 
-(10): VARCHAR-\>DATE The VARCHAR can only be formatted like:\'YYYY-MM-DD\', e.g. 2000-01-01
+(10): VARCHAR-\>DATE The VARCHAR can only be formatted like: \'YYYY-MM-DD\', e.g. 2000-01-01
 
-(11): VARCHAR-\>TIME The VARCHAR can only be formatted like:\'HH:MM:SS.XXX\'
+(11): VARCHAR-\>TIME The VARCHAR can only be formatted like: \'HH:MM:SS.XXX\'
 
-(12): VARCHAR-\>TIME ZONE The VARCHAR can only be formatted like:\'HH:MM:SS.XXX XXX\', e.g. 01:02:03.456 America/Los\_Angeles
+(12): VARCHAR-\>TIME ZONE The VARCHAR can only be formatted like: \'HH:MM:SS.XXX XXX\', e.g. 01:02:03.456 America/Los\_Angeles
 
-(13): VARCHAR-\>TIMESTAMP The VARCHAR can only be formatted like:YYYY-MM-DD HH:MM:SS.XXX
+(13): VARCHAR-\>TIMESTAMP The VARCHAR can only be formatted like: YYYY-MM-DD HH:MM:SS.XXX
 
-(14): DATE-\>TIMESTAMP will auto padding the time with 0. e.g.\'2010-01-01\' -> 2010-01-01 00:00:00.000
+(14): DATE-\>TIMESTAMP will auto padding the time with 0. e.g. \'2010-01-01\' -> 2010-01-01 00:00:00.000
 
 (15): TIME-\>TIME WITH TIME ZONE will auto padding the default time zone
 
-(16): TIME-\>TIMESTAMP will auto add the default date:1970-01-01
+(16): TIME-\>TIMESTAMP will auto add the default date: 1970-01-01
 
 Miscellaneous
 -------------

hetu-docs/en/functions/geospatial.md

@@ -245,7 +245,7 @@ Returns `true` if this Geometry is an empty geometrycollection, polygon, point e
 
 **ST\_IsSimple(Geometry)** -\> boolean
 
-Returns `true` if this Geometry has no anomalous geometric points, such as self intersection or self tangency.
+Returns `true` if this Geometry has no anomalous geometric points, such as self-intersection or self-tangency.
 
 **ST\_IsRing(Geometry)** -\> boolean
 

hetu-docs/en/functions/lambda.md

@@ -13,7 +13,7 @@ Lambda expressions are written with `->`:
     x -> CAST(x AS JSON)
     x -> x + TRY(1 / 0)
 
-Most SQL expressions can be used in a lambda body, with a fewexceptions:
+Most SQL expressions can be used in a lambda body, with a few exceptions:
 
 -   Subqueries are not supported. `x -> 2 + (SELECT 3)`
 -   Aggregations are not supported. `x -> max(y)`

hetu-docs/en/functions/window.md

@@ -45,7 +45,7 @@ Returns the rank of a value in a group of values. This is similar to `rank`, exc
 **ntile(n)** -\> bigint
 
 Divides the rows for each window partition into `n` buckets ranging from `1` to at most `n`. Bucket values will differ by at most `1`. If the number of rows in the partition does not divide evenly into the number
-of buckets, then the remainder values are distributed one per bucket,starting with the first bucket.
+of buckets, then the remainder values are distributed one per bucket, starting with the first bucket.
 
 For example, with `6` rows and `4` buckets, the bucket values would be as follows: `1` `1` `2` `2` `3` `4`
 

hetu-docs/en/index.md

@@ -46,6 +46,7 @@ headless: true
     - [Audit Log]({{< relref "./docs/admin/audit-log.md" >}})
     - [Reliable Execution]({{< relref "./docs/admin/reliable-execution.md" >}})
     - [JDBC Data Source Multi-Split Management]({{< relref "./docs/admin/multi-split-for-jdbc-data-source.md" >}})
+    - [Extension Physical Execution Planner]({{< relref "./docs/admin/extension-execution-planner.md" >}})
 
 - [Query Optimizer]("#")
     - [Table Statistics]({{< relref "./docs/optimizer/statistics.md" >}})
@@ -62,10 +63,11 @@ headless: true
     - [BTree Index]({{< relref "./docs/indexer/btree.md" >}})
     - [HIndex Statements]({{< relref "./docs/indexer/hindex-statements.md" >}})
     - [New Index]({{< relref "./docs/indexer/new-index.md" >}})
-    - 
+    
 - [Star Tree Cubes](#)
-    - [Overview] ({{< relref "./docs/preagg/overview.md" >}}>)
-    - [Statements] ({{< "./docs/preagg/statements.md" >}})
+    - [Overview]({{< relref "./docs/preagg/overview.md" >}})
+    - [Join Support]({{< relref "./docs/preagg/join-queries.md" >}})
+    - [Statements]({{< relref "./docs/preagg/statements.md" >}})
   
 - [Connectors]({{< relref "./docs/connector/_index.md" >}})
     - [Carbondata]({{< relref "./docs/connector/carbondata.md" >}})
@@ -82,6 +84,7 @@ headless: true
     - [JMX]({{< relref "./docs/connector/jmx.md" >}})
     - [Kafka]({{< relref "./docs/connector/kafka.md" >}})
     - [Kafka Connector Tutorial]({{< relref "./docs/connector/kafka-tutorial.md" >}})
+    - [Redis] ({{< relref "./docs/connector/redis.md" >}})
     - [Local File]({{< relref "./docs/connector/localfile.md" >}})
     - [Memory]({{< relref "./docs/connector/memory.md" >}})
     - [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
@@ -96,7 +99,8 @@ headless: true
     - [TPCH]({{< relref "./docs/connector/tpch.md" >}})
     - [VDM]({{< relref "./docs/connector/vdm.md" >}})
     - [Kylin]({{< relref "./docs/connector/kylin.md" >}})
-
+    - [OmniData]({{< relref "./docs/connector/omnidata.md" >}})
+    
 - [Functions and Operators]("#")
     - [Logical Operators]({{< relref "./docs/functions/logical.md" >}})
     - [Comparison Functions and Operators]({{< relref "./docs/functions/comparison.md" >}})
@@ -158,8 +162,6 @@ headless: true
     - [GRANT ROLES]({{< relref "./docs/sql/grant-roles.md" >}})
     - [INSERT]({{< relref "./docs/sql/insert.md" >}})
     - [INSERT OVERWRITE]({{< relref "./docs/sql/insert-overwrite.md" >}})
-    - [INSERT CUBE]({{< relref "./docs/sql/insert-cube.md" >}})
-    - [INSERT OVERWRITE CUBE]({{< relref "./docs/sql/insert-overwrite-cube.md" >}})
     - [JMX]({{< relref "./docs/sql/jmx.md" >}})
     - [PREPARE]({{< relref "./docs/sql/prepare.md" >}})
     - [RESET SESSION]({{< relref "./docs/sql/reset-session.md" >}})
@@ -172,9 +174,9 @@ headless: true
     - [SHOW CACHE]({{< relref "./docs/sql/show-cache.md" >}})
     - [SHOW CATALOGS]({{< relref "./docs/sql/show-catalogs.md" >}})
     - [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
+    - [SHOW CREATE CUBE]({{< relref "./docs/sql/show-create-cube.md" >}})
     - [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
     - [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
-    - [SHOW CUBES]({{< relref "./docs/sql/show-cubes.md" >}})
     - [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
     - [SHOW EXTERNAL FUNCTION]({{< relref "./docs/sql/show-external-function.md" >}})
     - [SHOW GRANTS]({{< relref "./docs/sql/show-grants.md" >}})
@@ -217,6 +219,11 @@ headless: true
     - [Task Resource]({{< relref "./docs/rest/task.md" >}})
 
 - [Release Notes]("#")
+    - [1.6.1 (27 Apr 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.1.md" >}})
+    - [1.6.0 (30 Mar 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.0.md" >}})
+    - [1.5.0 (30 Dec 2021)]({{< relref "./docs/releasenotes/releasenotes-1.5.0.md" >}})
+    - [1.4.1 (12 Nov 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.1.md" >}})
+    - [1.4.0 (15 Oct 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.0.md" >}})
     - [1.3.0 (30 Jun 2021)]({{< relref "./docs/releasenotes/releasenotes-1.3.0.md" >}})
     - [1.2.0 (31 Mar 2021)]({{< relref "./docs/releasenotes/releasenotes-1.2.0.md" >}})
     - [1.1.0 (30 Dec 2020)]({{< relref "./docs/releasenotes/releasenotes-1.1.0.md" >}})

hetu-docs/en/indexer/hindex-statements.md

@@ -31,7 +31,7 @@ CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WITH ("bloom.
 CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WHERE p in (part1, part2, part3);
 ```
 
-**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), only index creation on the **first** level is supported (colA).
+**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), for BTree index, only index creation on the **first** level is supported (colA). Bloom, Bitmap and Minmax index creation on either (colA or colB) is supported.
 
 ## SHOW
 

hetu-docs/en/indexer/new-index.md

@@ -10,17 +10,13 @@ The `getID()` method in the `Index` interface returns the ID of this index type
 
 ### Level
 
-A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must 
-have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e. 
-it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();` 
-which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
+A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e.  it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();`  which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
 
 ## Interface overlook
 
 ### Indexing methods
 
-Apart from the methods metioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete 
-document on `Index` interface, please refer to the Java Doc of the source code.
+Apart from the methods mentioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete document on `Index` interface, please refer to the Java Doc of the source code.
 
 There are two main functionalities in the `Index` interface:
 
@@ -46,7 +42,7 @@ boolean addValues(Map<String, List<Object>> values) throws IOException;
 Index deserialize(InputStream in) throws IOException;
 
 void serialize(OutputStream out) throws IOException;
-``` 
+```
 
 The usage of them are pretty straightforward. A good example to help understand their usage is the source code of `MinMaxIndex`, where adding values is just to 
 update the `max` and `min` variables according to the input number, and `serialize()/deserialize()`

hetu-docs/en/installation/deployment-auto.md

@@ -3,13 +3,13 @@
 
 
 
-In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete deployment faster and easier. The script is friendly to most of linux OS. However, to Ubuntu, you need to manually install the following dependencies:
+In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete the deployment faster and easier. The script is friendly to most of Linux OS. However, to Ubuntu, you need to manually install the following dependencies:
 
 > sshpass1.06 or above
 
 ## Deploying openLooKeng on a Single Node
 
-Execute below command can help you download the necessary packages and deploy openLooKeng server in one-click:
+Executing the below command can help you download the necessary packages and deploy openLooKeng server in one-click:
 
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh)
@@ -21,7 +21,7 @@ or:
 wget -O - https://download.openlookeng.io/install.sh|bash
 ```
 
-Normally, you don\'t need to do any thing, except for the installation to complete. It will automatically start the service.
+Normally, you don\'t need to do anything, except waiting for the installation to complete. It will automatically start the service.
 
 Execute below command to stop openLooKeng service.:
 
@@ -51,13 +51,13 @@ or:
 bash <(wget -qO- https://download.openlookeng.io/install.sh) --multi-node
 ```
 
-First of all, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
+First, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
 
 Besides, jdk version is required to be greater than 1.8.0\_151. If not, jdk1.8.0\_201 will be installed in the cluster. It is recommended to manually install these dependencies before installing openLooKeng service.
 
 Secondly, the script will download openLooKeng-server tarball and copy that tarball to all the nodes in the cluster. Then install the openLooKeng-server by using this tarball.
 
-Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and also for build-in catalogs like `tpch`, `tpcds`, `memory connector`.
+Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and for built-in catalogs like `tpch`, `tpcds`, `memory connector`.
 
 By design, the script will check if there are existing configuration under directory:
 `/home/openlkadmin/.openlkadmin/cluster_node_info`
@@ -79,9 +79,9 @@ The general configurations for openLooKeng\'s coordinator, workers are taken fro
 `/home/openlkadmin/.openlkadmin/cluster_config_info` and configurations for connectors are taken from the directory `/home/openlkadmin/.openlkadmin/catalog` respectively. If these directories or any required configuration files are absent during the deploy script running, default configuration files will be generated
 automatically and deployed to all nodes.
 
-Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customized the deployment.
+Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customize the deployment.
 
-If above process all succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
+If all the above processes succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
 
 ```shell
 /opt/openlookeng/bin/stop.sh
@@ -108,7 +108,7 @@ bash <(wget -qO- https://download.openlookeng.io/install.sh) --file <cluster_nod
 ```
 
 
-For more help message,execute below command to deploy single node cluster:
+For more help, execute below command to deploy single node cluster:
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh) -h
 ```
@@ -150,9 +150,9 @@ execute below command to deploy the configurations to openLooKeng cluster:
 bash /opt/openlookeng/bin/configuration_deploy.sh
 ```
 
-Note, if you want to add more configrations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
+Note, if you want to add more configurations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
 
-The property format has to be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it it a dynamic value. For example:
+The property format must be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it is a dynamic value. For example:
 
 ``` properties
 http-server.http.port=<http-server.http.port>
@@ -174,7 +174,7 @@ It is very easy and straight forward to uninstall openLooKeng Service, simply ru
 bash /opt/openlookeng/bin/uninstall.sh
 ```
 
-This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you wan to delete user and configuration files, you need to run the below command:
+This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you want to delete user and configuration files, you need to run the below command:
 
 ```shell
 bash /opt/openlookeng/bin/uninstall.sh --all
@@ -190,7 +190,7 @@ If you can't access the download URL from the machine where you want to install
 
 1. Also save third party dependencies under `/opt/openlookeng/resource`. That is, download all files from either `https://download.openlookeng.io/auto-install/third-resource/x86/` or `https://download.openlookeng.io/auto-install/third-resource/aarch64/`, depending on the machine's architecture. This should include 1 `OpenJDK` file and 2 `sshpass` files.
 
-1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture, and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
+1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
 
 After all resources are available, execute below command to deploy single node cluster:
 
@@ -217,7 +217,7 @@ bash /opt/openlookeng/bin/install_offline.sh --help
 
 ## Adding Node to Cluster
 
-If you want to add node to make the cluster bigger,execute the below command:
+If you want to add node to make the cluster bigger, execute the below command:
 
 ```shell
 bash /opt/openlookeng/bin/add_cluster_node.sh -n <ip_address_1,……ip_address_N>
@@ -238,11 +238,11 @@ or:
 bash /opt/openlookeng/bin/add_cluster_node.sh --file <add_nodes_file_path>
 ```
 
-If there are multiple nodes, separated by commas(,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
+If there are multiple nodes, separated by commas (,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
 
 ## Removing Node to Cluster
 
-If you want to remove node to make the cluster smaller,execute the below command:
+If you want to remove node to make the cluster smaller, execute the below command:
 
 ```shell
 bash /opt/openlookeng/bin/remove_cluster_node.sh -n <ip_address_1,……ip_address_N>
@@ -263,7 +263,7 @@ or:
 bash /opt/openlookeng/bin/remove_cluster_node.sh --file <remove_nodes_file_path>
 ```
 
-If there are multiple nodes, separate them with commas(,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
+If there are multiple nodes, separate them with commas (,). add_ nodes_ File example: ip_address_1,ip_address_2……,ip_address_N.
 
 ## See Also
 

hetu-docs/en/installation/deployment-ha.md

@@ -25,7 +25,7 @@ The above properties are described below:
 - `hetu.multiple-coordinator.enabled`: Enable multiple coordinators.
 - `hetu.embedded-state-store.enabled`: Enable coordinators to start embedded state store. 
 
-Note: It is suggested to enable embedded state store on all coordinators(or at least 3) to guarantee the high availability of service when node/network is down.
+Note: It is suggested to enable embedded state store on all coordinators (or at least 3) to guarantee the high availability of service when node/network is down.
 
 ###Configuring State Store
 Please refer to the section [State Store](../admin/state-store.md) to configure state store.

hetu-docs/en/installation/odbc.md

@@ -61,7 +61,7 @@ Before an application uses the openLooKeng ODBC driver, the data source DSN must
 
 ### Opening the ODBC Data Source Administrator (64-bit)
 
-1. Click **Start**, and choose **Control Panel**.
+1. Click **Start** and choose **Control Panel**.
 
 2. In **Control Panel**, click **System and Security**, and then click **Administrative Tools**.
 
@@ -171,6 +171,6 @@ You can obtain the details about data types by calling **SQLGetTypInfo** in **Ca
 
 The openLooKeng ODBC driver supports **both ANSI and Unicode** applications. The default connection character set is the system default character set for ANSI applications and utf8 for Unicode applications. If the character set used by the application is different from the above-mentioned character set,  it may cause garbled characters. For this, the user should specify the connection character set to adapt to the character set required by the application. The corresponding configuration of the connection character set is described as follows.
 
-When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by deafult, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
+When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by default, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
 
 While configuring data source all connection character sets supported by the openLooKeng ODBC driver can be set in the **Character Set** drop-down box on the page 3 of the User interface. User can select the connection character from the drop-down box after the **Test DSN** is success.

hetu-docs/en/preagg/join-queries.md

@@ -0,0 +1,65 @@
+## Join Query Support
+StarTree Cube can help optimize aggregation over join queries as well. The optimizer looks for aggregation subtree pattern in the logical plan that typically looks like following.
+```
+  AggregationNode
+  |- ProjectNode[Optional]
+  .  |- ProjectNode[Optional]
+  .  .  |- FilterNode[Optional]
+  .  .  .  |- JoinNode
+  .  .  .  .  [More Joins]
+  .  .  .  .  .
+  .  .  .  .  |- ProjectNode[Optional] - Left
+  .  .  .  .  .  |- TableScanNode [Fact Table]
+  .  .  .  .  |- ProjectNode[Optional] - Right
+  .  .  .  .  .  |- TableScanNode [Dim Table]
+```
+
+If the query matches the pattern, the optimizer rewrites the logical plan by replacing the Fact TableScanNode with Cube TableScanNode. This is similar to the single 
+table rewrite.
+
+### Star Schema Support
+Join Query optimizer supports star schema only. A star schema is a data warehousing architecture model where one fact table references multiple dimension tables, which, when viewed as a diagram, 
+looks like a star with the fact table in the center and the dimension tables radiating from it. All kinds of joins are supported.
+
+![star-schema](../images/star-schema.png "star schema")
+
+### Cube Management
+`Create Cube` can be still be used to define Cubes to optimize Join queries as well. The difficult part is identifying GROUP construct while building the Cubes. With single table 
+queries, the GROUP BY clause will contain columns only from same the table. But with join queries, especially star schema queries, the GROUP BY contain columns from Dimension tables and not the Fact table. 
+Let's analyze more with following query
+```sql
+SELECT SUM(lo_revenue) AS lo_revenue, d_year, p_brand
+FROM lineorder
+LEFT JOIN dates ON lo_orderdate = d_datekey
+LEFT JOIN part on lo_partkey = p_partkey
+LEFT JOIN supplier on lo_suppkey = s_suppkey
+WHERE p_category = 'MFGR#12' AND s_region = 'AMERICA'
+GROUP BY d_year, p_brand
+ORDER BY d_year, p_brand;
+```
+
+Here `lineorder` is the Fact table and `dates`, `part`, `supplier` are the Dimension tables. Cubes will be defined on the `lineorder` table. The group by columns `d_year`, `p_brand` are part 
+of the dimension tables `dates` and `part` appropriately. They cannot be used directly in `CREATE CUBE` statement. The proper solution is to use the foreign key columns of `lineorder` table in 
+the GROUP construct while building Cubes.
+```sql
+CREATE CUBE lineorder_cube ON lineorder WITH(
+AGGREGATIONS = (sum(lo_revenue)),
+GROUP = (lo_orderdate, lo_partkey, lo_suppkey));
+```
+The optimizer parses the join conditions and uses those columns to identify the matching Cubes. The performance gain is realized if Cube size is smaller than fact table.
+
+### Limitations
+* Only star schema is supported.
+* Count distinct not supported because Cube does not store actual dimension values.
+* Queries won't be optimized if Cubes are defined on both Fact and Dimension as the optimizer does not have capability to differentiate between two.
+* If Cubes are defined on more than one table of the Join query - then Optimizer does not work. Assumption is that Cubes are defined only the Fact table.
+* Supports only simple aggregation like SUM, COUNT, AVG, MIN, MAX - defined on Single column. Cube does not support SUM(revenue - supplycost) aggregation. The following query cannot be optimized using Cube.
+```
+   SELECT sum(lo_extendedprice * lo_discount) AS revenue
+   FROM lineorder
+   WHERE toYear(lo_orderdate) = 1993 AND lo_discount BETWEEN 1 AND 3 AND lo_quantity < 25;
+```
+
+### Future
+* Support for snowflake schema
+* Building a single cube over multiple tables

hetu-docs/en/preagg/overview.md

@@ -15,8 +15,8 @@ Few of the Cube properties are
  - Query latency is reduced by rewriting the logical plan to use Cube instead of the original table.
 
 ## Cube Optimizer Rule
-As part of logical plan optimization, Cube optimizer rule analyzes and optimizes the aggregation sub-tree of the logical plan with Cubes. 
-The rule looks for the aggregation sub-tree that typically looks like the following
+As part of logical plan optimization, Cube optimizer rule analyzes and optimizes the aggregation subtree of the logical plan with Cubes. 
+The rule looks for the aggregation subtree that typically looks like the following
 
 ```
 AggregationNode
@@ -27,17 +27,17 @@ AggregationNode
         |- TableScanNode
 ```
 
-The rule parses through the sub-tree and identifies the table name, aggregate functions, where clause, group by clause that is matched with Cube metadata 
+The rule parses through the subtree and identifies the table name, aggregate functions, where clause, group by clause that is matched with Cube metadata 
 to identify any Cube that can help optimize the query. In case of multiple match, recently created Cube is selected for optimization. If any match found, entire 
-aggregation sub-tree is rewritten using the Cube. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can be supported by the 
-Cubes.  
+aggregation subtree is rewritten using the Cube. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can be supported by the 
+Cubes.
 
 The following picture depicts the change in the logical plan after the optimization. 
 
 ![img](../images/cube-logical-plan-optimizer.png)
 
 ## Recommended Usage
-1. Cubes are mose useful for iceberg queries that takes huge input and produces small input
+1. Cubes are most useful for iceberg queries that takes huge input and produces small output.
 2. Query performance is best when size of the Cube is less that on the actual table on which Cube was built.
 3. Cubes need to be rebuilt if the source table is updated. 
 
@@ -47,7 +47,11 @@ operation on the update is considered as a change in the existing data even if o
 can't be differentiated, Cubes can't be used as it might result in incorrect result. We are working on a solution to overcome this limitation.
 
 ## Supported Connectors
-The following are supported Connectors for storing a cube
+Star Tree Cube can be stored in following Connectors
+1. Hive
+2. Memory
+   
+Tables from following Connectors can be used as source to build a StarTree Cube.
 1. Hive
 2. Memory
 3. Clickhouse
@@ -58,8 +62,6 @@ The following are supported Connectors for storing a cube
 
    2.1. Overcome the limitation of Creating Cube for larger dataset.
 
-   2.2. Update cube if source table has been updated.
-
 ## Enabling and Disabling StarTree Cube
 To enable:
 ```sql 
@@ -73,13 +75,13 @@ SET SESSION enable_star_tree_index=false;
 ## Configuration Properties
 | Property Name                                     | Default Value       | Required| Description|
 |---------------------------------------------------|---------------------|---------|--------------|
-| optimizer.enable-star-tree-index                  | false               | No      | Enables StarTree Cube|
-| cube.metadata-cache-size                          | 50                  | No      | The maximum number of metadata for StarTree Cubes that could be loaded into cache before eviction happens|
+| optimizer.enable-star-tree-index                  | false               | No      | Enables StarTree Cube |
+| cube.metadata-cache-size                          | 50                  | No      | The maximum number of metadata for StarTree Cubes that could be loaded into cache before eviction happens |
 | cube.metadata-cache-ttl                           | 1h                  | No      | The maximum time to live of StarTree Cubes that are be loaded into cache before eviction happens |
 
 ## Dependencies
 
-StarTree Cube relies on Hetu metastore to store the Cube related metadata.
+StarTree Cube relies on Hetu Metastore to store the Cube related metadata.
 Please check [Hetu Metastore](../admin/meta-store.md) for more information.
 
 ## Examples
@@ -118,11 +120,22 @@ SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >=
 Since the data inserted into the Cube was for `nationkey >= 5`, only queries matching this condition will utilize the Cube.
 Queries not matching the condition would continue to work but won't use the Cube.
 
-## Building Cube for Large dataset
+If the source table of a Cube gets updated, the corresponding Cube gets expired automatically. In order to overcome
+this issue, we have added support in openLooKeng CLI by introducing **RELOAD CUBE** command. The user will have the
+ability to manually reload a cube if the status of the Cube becomes INACTIVE or EXPIRED. The syntax to reload the
+Cube nation_cube is as follows,
+
+```sql 
+RELOAD CUBE nation_cube
+```
+Please note that this feature is only supported via the CLI. During this reload process if an unexpected error occurs, the user will get to see the original SQL statement
+to recreate the cube manually.
+
+## Building Cube for Large Dataset
 One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
 Processing large number of rows requires more memory than cluster is configured with. This results in query failing with message **Query exceeded per-node user memory
-limit**. To overcome this issue, **INSERT INTO CUBE** sql support was added. The user has ability to build a Cube for larger data by executing multiple
-insert into cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into Cube.
+limit**. To overcome this issue, **INSERT INTO CUBE** SQL support was added. The user has ability to build a Cube for larger data by executing multiple
+insert into Cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into Cube.
 
 This section explains the steps to build a Cube for larger dataset.
 
@@ -146,7 +159,7 @@ INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2422
 ```
 
 ### Solution 1)
-To overcome this issue, multiple insert statements can be used into process rows and insert into cube and the number of rows can be limited by using where clause;
+To overcome this issue, multiple insert statements can be used to process rows and insert into Cube and the number of rows can be limited by using where clause;
 
 ```sql
 INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2452010;
@@ -157,8 +170,8 @@ INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452211 AND 2452
 
 ### Solution 2)
 CLI has been modified to support creating Cubes for larger dataset and without need for multiple insert statements. CLI internally handles this process.
-Once the user runs create cube statement with where clause, the CLI takes care of creating the cube as well as inserting the data into it. This process improves the user experience and
-improves the memory footprint based on the cluster memory limits. CLI internally parses the converts the statement into one create cube statement followed by 
+Once the user runs create Cube statement with where clause, the CLI takes care of creating the Cube as well as inserting the data into it. This process improves the user experience and
+improves the memory footprint based on the cluster memory limits. CLI internally parses the converts the statement into one create Cube statement followed by 
 one or more insert statements. This change is only works if user executes the command from CLI and not via any other means i.e. JDBC, etc...
 
 ```sql
@@ -176,46 +189,73 @@ SHOW CUBES;
 ```
 
 **Note:**
-1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together. 
+1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together.
    All continuous predicates will be merged into a single range predicate and remaining predicates are untouched.
 
-   Only the following types are supported and can be merged together. 
-   `Integer, TinyInt, SmallInt, BigInt, Date`
-    
-   For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is 
-   possibility that particular cube may not be used during query optimization even if the cube has all the required data. For example,
+   Only the following types are supported and can be merged together.
+   `Integer, TinyInt, SmallInt, BigInt, Date, String`
+
+   For String data type, predicate merge logic functionally works only if the Strings are ending with a digit and all are of same length. 
+   For example,
 
 ```sql
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
 ```
-   Here these two predicates cannot be merged into store_id BETWEEN 'A01' AND 'A20'; So the cube won't be used 
-   for queries that are spanning over two the predicates;
+After the insertion, the two predicates will be merged into `'A01' AND 'A20'`
 
 ```sql
-   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube won't be used for optimizing this query. This is a limitation as of now.
+   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube would be used for this query.
 ```
-   Because of the predicate rewrite some of the following queries can't be supported
+
+   Consider the following example where `store_id` values are not of same length.
+
+```sql
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A1';
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A2' 
+```   
+   store_id predicate will be rewritten as `store_id >= 'A1' and store < 'A3'` as per the varchar predicate merge logic;
+
+```sql   
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A10' 
+```
+   The above query would fail because `A10` is subset of the range `store_id >= 'A1' and store < 'A3'`. So Users should be wary of this issue. 
+
+For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is
+possibility that particular Cube may not be used during query optimization even if the Cube has all the required data.
+
+2. Predicate rewrite has some limitations as well. Consider the following query
 
 ```sql   
    INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
-```   
-   The predicate is rewriten as ss_sold_date_sk >= 2451912 to be prepare for merging continous predicates. 
-   Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to 
-   optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
+```
+The predicate is rewritten as ss_sold_date_sk >= 2451912 to support merging continuous predicates.
+Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to
+optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
 
 ```sql   
    SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
 ```   
-3. Only Single column predicates can be merged. 
+3. Only single column predicates can be merged.
+
 
 ## Open issues and Limitations
 1. StarTree Cube is only effective when the group by cardinality is considerably fewer than the number of rows in source table.
 2. A significant amount of user effort required in maintaining Cubes for large datasets.
-3. Only incremental insert into cube is supported. Cannot delete specific rows from Cube.
+3. Only incremental insert into Cube is supported. Cannot delete specific rows from Cube.
 4. Cubes created on a transaction table may expire automatically even if the source table has not been updated. This is due to the compaction policy which
    merges delta files into single large ORC file which in turn changes the last modified of time of the table. Cube status is determined by comparing last modified
-   timestamp of table when cube was created with the last modified time of the table when queries are executed.
-5. Openlookeng CLI has been modified to ease the process of creating Cubes for larger datasets. But still there are limitations with this implementation
-   as the process involves merging multiple cube predicates into one. Only cube predicates defined on Integer, Long and Date types can be merged properly. Support for Char, 
-   String types still need to be implemented.
+   timestamp of table when Cube was created with the last modified time of the table when queries are executed.
+5. OpenLooKeng CLI has been modified to ease the process of creating Cubes for larger datasets. But still there are limitations with this implementation
+   as the process involves merging multiple Cube predicates into one. Only Cube predicates defined on Integer, Long and Date types can be merged properly. Support for Char, 
+   String types still need to be implemented.
+6. Varchar predicates can be merged only if the values are of same length.  
+
+## Performance Optimizations on Star Tree
+1. Star Tree Query re-write optimization for same group by columns: If the group by columns of the cube and query matches, the query is 
+re-written internally to select the pre-aggregated data. If the group by columns does not matches, the additional aggregations are 
+internally applied on the re-written query.
+2. Star Tree table scan optimization for Average aggregation function: If the group by columns of the cube and query matches, the select query 
+is re-written internally to select the startree cube's pre-aggregated Average column data. If the group by columns does not match, 
+the select query is re-written internally to select the startree cube's pre-aggregated Sum and Count column data, from which the 
+average is later calculated.

hetu-docs/en/preagg/statements.md

@@ -44,7 +44,7 @@ Create a new partitioned Cube `orders_cube`:
       partitioned_by = ARRAY['orderdate']
     )
 
-Create a new Cube `orders_cube` with some source data filter
+Create a new Cube `orders_cube` with some source data filter:
 
     CREATE CUBE orders_cube ON orders WITH (
       AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
@@ -52,7 +52,7 @@ Create a new Cube `orders_cube` with some source data filter
       FILTER = (orderdate BETWEEN 2512450 AND 2512460)
     )
 
-Create a new Cube `orders_cube` with some additional predicate on Cube columns
+Create a new Cube `orders_cube` with some additional predicate on Cube columns:
 
     CREATE CUBE orders_cube ON orders WITH (
       AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
@@ -60,7 +60,7 @@ Create a new Cube `orders_cube` with some additional predicate on Cube columns
       FILTER = (orderdate BETWEEN 2512450 AND 2512460)
     ) WHERE orderstatus = 'PENDING';
 
-This is same as following
+This is same as following:
     
     CREATE CUBE orders_cube ON orders WITH (
       AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
@@ -86,12 +86,12 @@ INSERT INTO CUBE cube_name [WHERE condition]
 ```
 
 ### Description
-`CREATE CUBE` statement creates Cube without any data. To insert data into Cube, use `INSERT INTO CUBE` sql.
+`CREATE CUBE` statement creates Cube without any data. To insert data into Cube, use `INSERT INTO CUBE` SQL.
 The `WHERE` clause is optional. If predicate is provided, only data matching the given predicate are processed from the source table and inserted into the Cube. 
 Otherwise, entire data from the source table is processed and inserted into Cube.
 
 ### Examples
-Insert data into the `orders_cube` Cube
+Insert data into the `orders_cube` Cube:
 
     INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
     INSERT INTO CUBE order_all_cube;
@@ -117,8 +117,10 @@ INSERT OVERWRITE CUBE cube_name [WHERE condition]
 ```
 
 ### Description
-Similar to INSERT INTO CUBE statement but with this statement the existing data is overwritten. Predicates
-are optional.
+Similar to `INSERT INTO CUBE` statement but with this statement the existing data is overwritten. Predicates
+are optional.`INSERT OVERWRITE CUBE` is not supported on partitioned cubes. Cubes are essentially stored as tables and so `INSERT OVERWRITE` only 
+replaces the matching partitions and does not overwrite the entire table. So this operation is blocked on partitioned cube. 
+Drop and recreate cube if needed.
 
 ### Examples
 Insert data based on condition into the `orders_cube` Cube:
@@ -148,7 +150,25 @@ Show Cubes for `orders` table:
 ```sql
     SHOW CUBES FOR orders;
 ```
+## RELOAD CUBE
 
+### Synopsis
+
+``` sql
+RELOAD CUBE cube_name
+```
+
+### Description
+Reloads the Cube if the source table has been updated.
+
+### Examples
+
+If the source table `orders` of the cube `orders_cube` gets updated then the status of the cube `orders_cube` 
+gets EXPIRED. Use the command `RELOAD CUBE cube_name` to overcome this issue as follows:
+
+```sql
+    RELOAD CUBE orders_cube
+```
 ## DROP CUBE
 
 ### Synopsis

hetu-docs/en/releasenotes/releasenotes-1.4.1.md

@@ -0,0 +1,14 @@
+# Release 1.4.1 (12 Nov 2021)
+
+## Key Features
+
+This release mainly adds the introduction of OmniData Connector and jdk8 support under arm architecture.
+
+| Area                    | Feature                                                      | PR #s                                                        |
+| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| Data Source Connector                | openLooKeng supports offload operators to the near data side for process through omnidata connector, reducing the transmission of invalid data on the network and effectively improving the performance of big data computing.                                               | 1219                                                          |
+| Arm architecture | Eliminate the mandatory requirements for Java version under arm architecture caused by JDK paused problem, and support jdk1.8.262 and above under arm architecture. | 1214                                                           |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.5.0.md

@@ -0,0 +1,23 @@
+# Release 1.5.0
+
+## Key Features
+
+| Area             | Feature                                                      |
+| ---------------- | ------------------------------------------------------------ |
+| Star Tree        | 1. Support for optimizing join queries such as star schema queries.<br/>2. Optimized the query plan by eliminating unnecessary aggregations on top of the cube since the cube already contains the rolled up results. The performance optimizations benefits queries whose group by clause exactly matches the cube’s group.<br/>3. Bug fixes to further enhance the usability, and robustness of cubes. |
+| Memory Connector | 1. Improved performance of memory connector by adding support for memory table partitioning to allow data skipping of entire partitions<br/>2. Collect statistics on memory table to support openLooKeng cost based optimizers. |
+| Task Recovery    | Fixed several important bugs to address data inconsistency issues, and query hanging issues that occasionally occur during high concurrency, and during worker failures. |
+| OLK-on-Yarn      | Support deploying an HA-enabled openLooKeng cluster instance on-yarn, that contains a reverse proxy (ngnix by default), and 2 or more coordinator nodes. The cluster can be horizontally scaled manually by adding and removing yarn containers to the coordinator and worker components.|
+| Spill to Disk    | Optimized the spill to disk mechansim to directly write Pages to disk instead of buffering it. Changed the strategy to spill those operators which can free up maximum memory. This resulted in improvement of spill to disk performance by 30% |
+
+## Known Issues
+
+| Category              | Description                                                  | Gitee issue                                                       |
+| --------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| Task Recovery         |When the query process reaches stage1,a worker is killed result some values become smaller(occasionally).   |[I4M2LW](https://e.gitee.com/open_lookeng/issues/list?issue=I4M2LW) |
+| Memory Connector      |When a table with the index_columns parameter is queried, an error message is displayed：java.lang.NullPointerException. | [I4NVW3](https://e.gitee.com/open_lookeng/issues/list?issue=I4NVW3)|
+|                       |When drop and then create a same partitioned table with data type double, query result rows is greater than expected. | [I4LUDF](https://e.gitee.com/open_lookeng/issues/list?issue=I4LUDF) |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.6.0.md

@@ -0,0 +1,25 @@
+# Release 1.6.0
+
+## Key Features
+
+| Area                  | Feature                                                      |
+| --------------------- | ------------------------------------------------------------ |
+| Star Tree             | Support update cube command to allow admin to easily update an existing cube when the underlying data changes |
+| Bloom Index           | Hindex-Optimize Bloom Index Size-Reduce bloom index size by 10X+ times |
+| Task Recovery         | 1. Improve failure detection time: It need take 300s to determine a task is failed and resume after that. Improving this would improve the resume & also the overall query time<br/>2. snapshotting speed & size: When sql execute takes a snapshot, now use direct Java serialization which is slow and also takes more size. Using kryo serialization would reduce size and also increase speed there by increasing the overall throughput |
+| Spill to Disk         | 1. Spill to disk speed & size improvement: When spill happens during HashAggregation & GroupBy, the data serialized to disk is slow and also size is more. It can improve the overall performance by reducing size and also improving  the writing speed. Using kryo serialization improves both speed and reduces size<br/>2. Support spilling to hdfs: Currently data can spill to multiple disks, now support spill to hdfs to improve throughput<br/>3. Async spill/unspill: When revocable memory crosses threshold and spill is triggered, it blocks accepting the data from the downstream operators. Accepting this and adding to the existing spill would help to complete the pipeline faster<br/>4. Enable spill for right outer & full join for spilling: It don’t spill the build side data when the join type is right outer or full join as it needs the entire data in memory for lookup. This leads to out of  memory when the data size is more. Instead by enable spill and create a Bloom Filter to identify the data spilled and use it during join with probe side |
+| Connector Enhancement | Support data update and delete operator for PostgreSQL and openGauss |
+
+## Known Issues
+
+| Category      | Description                                                  | Gitee issue                                               |
+| ------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
+| Task Recovery | When a snapshot is enabled and a CTAS with transaction is executed, an error is reported in the SQL statement. | [I502KF](https://e.gitee.com/open_lookeng/issues/list?issue=I502KF) |
+|               | An error occurs occasionally when snapshot is enabled and exchange.is-timeout-failure-detection-enabled is disabled. | [I4Y3TQ](https://e.gitee.com/open_lookeng/issues/list?issue=I4Y3TQ) |
+| Star Tree     | In the memory connector, after the star tree is enabled, data inconsistency occurs during query. | [I4QQUB](https://e.gitee.com/open_lookeng/issues/list?issue=I4QQUB) |
+|               | When the reload cube command is executed for 10 different cubes at the same time, some cubes fail to be reloaded. | [I4VSVJ](https://e.gitee.com/open_lookeng/issues/list?issue=I4VSVJ) |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en)
+

hetu-docs/en/releasenotes/releasenotes-1.6.1.md

@@ -0,0 +1,15 @@
+# Release 1.6.1 (27 Apr 2022)
+
+## Key Features
+
+This release is mainly about modification and enhancement of some SPIs, which are used in more scenarios.
+
+| Area                    | Feature                                                      | PR #s                                                        |
+| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| Data Source statistics                | The method of obtaining statistics is added so that statistics can be directly obtained from the Connector. Some operators can be pushed down to the Connector for calculation. You may need to obtain statistics from the Connector to display the amount of processed data.                                               | 1450                                                          |
+| Operator processing extension | Users can customize the physical execution plan of worker nodes. Users can use their own operator pipelines to replace the native implementation to accelerate operator processing. | 1436                                                           |
+| HIVE UDF extension | Adds the adaptation of HIVE UDF function namespace to support the execution of UDFs (including GenericUDF) written based on the HIVE UDF framework. | 1456                                                           |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en)

hetu-docs/en/security/built-in-system-access-control.md

@@ -1,4 +1,3 @@
-
 Built-in System Access Control
 ==============================
 
@@ -57,7 +56,10 @@ composed of the following fields:
 
 -   `user` (optional): regex to match against user name. Defaults to `.*`.
 -   `catalog` (optional): regex to match against catalog name. Defaults to `.*`.
--   `allow` (required): boolean indicating whether a user has access to the catalog
+-   ``allow`` (required): string indicating whether a user has access to the catalog.
+    This value can be ``all``, ``read-only`` or ``none``, and defaults to ``none``.
+    Setting this value to ``read-only`` has the same behavior as the ``read-only``
+    system access control plugin.
 
 
 **Note**
@@ -65,7 +67,9 @@ composed of the following fields:
 *By default, all users have access to the `system` catalog. You can override this behavior by adding a rule.*
 
 
-For example, if you want to allow only the user `admin` to access the `mysql` and the `system` catalog, allow all users to access the `hive` catalog, and deny all other access, you can use the following rules:
+For example, if you want to allow only the user ``admin`` to access the``mysql`` and the ``system`` catalog, 
+allow all users to access the ``hive`` catalog, allow the user ``alice`` read-only access to the ``postgresql`` 
+catalog, and deny all other access, you can use the following rules:
 
 ``` json
 {
@@ -73,15 +77,20 @@ For example, if you want to allow only the user `admin` to access the `mysql` an
     {
       "user": "admin",
       "catalog": "(mysql|system)",
-      "allow": true
+      "allow": all
     },
     {
       "catalog": "hive",
-      "allow": true
+      "allow": all
+    },
+    {
+      "user": "alice",
+      "catalog": "postgresql",
+      "allow": "read-only"
     },
     {
       "catalog": "system",
-      "allow": false
+      "allow": none
     }
   ]
 }
@@ -195,6 +204,7 @@ If you want to allow users to use the  extactly same name as their Kerberos prin
 ```
 
 ### Node State Rules
+
 These rules govern the node state info particular users can access. The user is granted access to update a node state based on the first matching rule read from top to bottom. If no rule matches, access is denied. Each rule is
 composed of the following fields:
 

hetu-docs/en/security/ranger-access-control.md

@@ -7,7 +7,7 @@ Overview
 
 Apache Ranger delivers a comprehensive approach to security for a Hadoop cluster. It provides a centralized platform to define, administer and manage security policies consistently across Hadoop components. Check [Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index) for detail introduction and user guide.
 
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
+[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is developed based on Ranger 2.1.0, which is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
 
 Build Process
 -------------------------

hetu-docs/en/sql/cache-table.md

@@ -38,7 +38,7 @@ Cache data with complex predicate string:
 Limitations
 -----------
 
-- Only Hive connector(ORC Format) support this functionality at this time. See connector documentation for more details.
+- Only Hive connector (ORC Format) support this functionality at this time. See connector documentation for more details.
 - Does not support `LIKE` in `WHERE` clause.
 - Does not support 'OR' operator in complex predicate.
 

hetu-docs/en/sql/create-view.md

@@ -14,7 +14,7 @@ AS query
 Description
 -----------
 
-Create a new view of a [SELECT](./select.md) query. The view is a logical table that can be referenced by future queries. Views do not contain any data. Instead, the query stored by the view is executed everytime the view is referenced by another query.
+Create a new view of a [SELECT](./select.md) query. The view is a logical table that can be referenced by future queries. Views do not contain any data. Instead, the query stored by the view is executed every time the view is referenced by another query.
 
 The optional `OR REPLACE` clause causes the view to be replaced if it already exists rather than raising an error.
 

hetu-docs/en/sql/insert-overwrite.md

@@ -28,7 +28,7 @@ Assume `orders` is not a partitioned table, and have 100 rows, then execute belo
 
     INSERT OVERWRITE orders VALUES (1, 'SUCCESS', '10.25', DATA '2020-01-01');
 
-Then the `orders` table will only have 1 rows, that is the data specified in the `VALUE` clause.
+Then the `orders` table will only have 1 row that is the data specified in the `VALUE` clause.
 
 Assume `users` has 3 columns: (`id`, `name`, `state`) and partitioned by `state`, and the existing data has follow rows:
 

hetu-docs/en/sql/select.md

@@ -508,7 +508,7 @@ _col0
 **INTERSECT**
 
 `INTERSECT` returns only the rows that are in the result sets of both the first and the second queries. The following is an example of one of the simplest possible `INTERSECT` clauses. It selects the values `13`
-and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results.:
+and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results:
 
     SELECT * FROM (VALUES 13, 42)
     INTERSECT
@@ -524,7 +524,7 @@ _col0
 **EXCEPT**
 
 `EXCEPT` returns the rows that are in the result set of the first query, but not the second. The following is an example of one of the simplest possible `EXCEPT` clauses. It selects the values `13` and `42` and
-combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result.:
+combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result:
 
     SELECT * FROM (VALUES 13, 42)
     EXCEPT
@@ -746,7 +746,7 @@ Joins allow you to combine data from multiple relations.
 
 ### CROSS JOIN
 
-A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
+A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explicit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
 
 Both of the following queries are equivalent:
 

hetu-docs/en/sql/show-create-cube.md

@@ -0,0 +1,32 @@
+
+SHOW CREATE CUBE
+=================
+
+Synopsis
+--------
+
+``` sql
+SHOW CREATE CUBE cube_name
+```
+
+Description
+-----------
+
+Show the SQL statement that creates the specified cube.
+
+Examples
+--------
+
+Create a cube `orders_cube` on `orders` table as follows
+
+    CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+    GROUP = (custKEY, ORDERkey), format= 'orc')
+
+Use `SHOW CREATE CUBE` command to show the SQL statement that was used to create the cube `orders_cube`:
+
+    SHOW CREATE CUBE orders_cube;
+
+``` sql
+CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+GROUP = (custKEY, ORDERkey), format= 'orc')
+```

hetu-docs/zh/admin/audit-log.md

@@ -1,6 +1,6 @@
 # 审计日志
 
-openLooKeng审计日志记录功能是一个自定义事件监听器，在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
+openLooKeng审计日志记录功能是一个自定义事件监听器，监听openLooKeng集群启停与集群中节点的动态添加与删除事件；监听WebUi用户登录与退出事件；监听查询事件,在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
 
 1. 事件发生时间
 2. 用户ID
@@ -23,15 +23,17 @@ openLooKeng审计日志记录功能是一个自定义事件监听器，在查询
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
 其他审计日志记录属性包括：
 
-`hetu.event.listener.audit.file`：可选属性，用于定义审计文件的绝对文件路径。确保运行openLooKeng服务器的进程对该目录有写权限。
+`hetu.event.listener.type`：用于定义审计日志的记录类型，允许的值为AUDIT和LOGGER。
 
-`hetu.event.listener.audit.filecount`：可选属性，用于定义要使用的文件数。
+`hetu.auditlog.logoutput`：用于定义审计文件的绝对目录路径。确保运行openLooKeng服务器的进程对该目录有写权限。
 
-`hetu.event.listener.audit.limit`：可选属性，用于定义写入任一文件的最大字节数。
+`hetu.auditlog.logconversionpattern`：用于定义审计日志的轮转模式。允许的值为yyyy-MM-dd.HH和yyyy-MM-dd。
 
 配置文件示例：
 
@@ -43,4 +45,6 @@ hetu.event.listener.listen.query.completion=true
 hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
 hetu.event.listener.audit.filecount=1
 hetu.event.listener.audit.limit=100000
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```

hetu-docs/zh/admin/extension-execution-planner.md

@@ -0,0 +1,24 @@
+#扩展物理执行计划
+本节介绍openLooKeng如何添加扩展物理执行计划。通过物理执行计划的扩展，openLooKeng可以使用其他算子加速库来加速SQL语句的执行。
+
+##配置
+在配置文件`config.properties`增加如下配置：
+
+``` properties
+extension_execution_planner_enabled=true
+extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
+extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
+```
+
+上述属性说明如下：
+
+- `extension_execution_planner_enabled`：是否开启扩展物理执行计划特性。
+- `extension_execution_planner_jar_path`：指定扩展jar包的文件路径。
+- `extension_execution_planner_class_path`：指定扩展jar包中执行计划生成类的包路径。
+
+
+##使用
+当运行openLooKeng时，可在WebUI或Cli中通过如下命令控制扩展物理执行计划的开启:
+```
+set session extension_execution_planner_enabled=true/false
+```

hetu-docs/zh/admin/properties.md

@@ -115,6 +115,20 @@
 > 
 > 此属性是在JVM堆中为openLooKeng不跟踪的分配留作裕量/缓冲区的内存量。
 
+### `query.suspend-query-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 系统资源不足时，临时挂起运行中的查询。
+
+### `query.max-suspended-queries`
+
+> -   **类型：** `integer`
+> -   **默认值：** `10`
+>
+> 终止查询之前，查询挂起尝试的最大次数。仅当`query.suspend-query-enabled`设置为`true`时，此属性才生效。
+
 ## 溢出属性
 
 ### `experimental.spill-enabled`
@@ -148,6 +162,24 @@
 > 
 > 此配置属性可由`spill_window_operator`会话属性重写。
 
+### `experimental.spill-build-for-outer-join-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 为右外连接和全外连接操作启用溢出功能。
+>
+> 此config属性可被`spill_build_for_outer_join_enabled`会话属性覆盖。
+
+### `experimental.inner-join-spill-filter-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用基于布隆过滤器的构建侧溢出匹配，以进行探查侧溢出决策。
+>
+> 此config属性可被`inner_join_spill_filter_enabled`会话属性覆盖。
+
 ### `experimental.spill-reuse-tablescan`
 
 > - **类型**：`boolean`
@@ -157,13 +189,13 @@
 >
 > 此配置属性可由`spill_reuse_tablescan`会话属性重写。
 
-### experimental.spiller-spill-path`
+### `experimental.spiller-spill-path`
 
 > - **类型：** `string`
 > - **无默认值。** 启用溢出时必须设置。
 > 
 > 溢出内容写入的目录。该属性可以是一个逗号分隔的列表，以同时溢出到多个目录，这有助于利用系统中安装的多个驱动器。
-> 
+> 当`experimental.spiller-spill-to-hdfs`为`true`时，`experimental.spiller-spill-path`必须只包含一个目录。
 > 不建议溢出到系统驱动器上。最重要的是，不要溢出到写入JVM日志的驱动器，因为磁盘过度使用可能导致JVM长时间暂停，从而导致查询失败。
 
 ### `experimental.spiller-max-used-space-threshold`
@@ -208,7 +240,7 @@
 >
 > 用于在Reuse Exchange中缓存页面的内存限制。
 
-### experimental.spill-compression-enabled`
+### `experimental.spill-compression-enabled`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -222,6 +254,69 @@
 > 
 > 允许使用随机生成的密钥（每个溢出文件）来加密和解密溢出到磁盘的数据。
 
+### `experimental.spill-direct-serde-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 允许将页面直接序列化/读取到流中或从流中序列化/读取页面。
+
+### `experimental.spill-prefetch-read-pages`
+
+> -   **类型：** `integer`
+> -   **默认值：** `1`
+>
+> 设置从溢出文件读取时预取的页数。
+
+
+### `experimental.spill-use-kryo-serialization`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用基于Kryo的序列化以溢出到磁盘，而不使用默认的Java序列化器。
+
+
+### `experimental.revocable-memory-selection-threshold`
+
+> -   **类型：** `data size`
+> -   **默认值：** `512 MB`
+>
+> 设置运算符可撤销内存的内存选择阈值，直接为准备撤销的剩余字节分配可撤销内存。
+
+### `experimental.prioritize-larger-spilts-memory-revoke`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `true`
+>
+> 启用对具有较大可撤销内存的Split进行优先级排序。
+
+### `experimental.spill-non-blocking-orderby`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 开启按照运算符排序，使用异步机制溢出。即使在溢出正在进行时，也可以累积输入，并在次要数据累积超过阈值或主溢出完成时启动次溢出。阈值的默认值是20MB到可用内存的5%之间的最小值。此属性必须与`experimental.spill-enabled`属性结合使用。
+>
+> 此config属性可被`spill_non_blocking_orderby`会话属性覆盖。
+
+### `experimental.spiller-spill-to-hdfs`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用溢出到HDFS。当此属性设置为`true`时，必须设置`experimental.spiller-spill-profile`属性，并且`experimental.spiller-spill-path`必须仅包含单个路径。
+
+### `experimental.spiller-spill-profile`
+
+> -   **类型：** `string`
+> -   **无默认值。** 启用溢出到HDFS时必须设置此属性。
+>
+>
+> 此属性定义用于溢出的[filesystem](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果此属性设置为`experimental.spiller-spill-profile=spill-hdfs`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`spill-hdfs.properties`，其中包含必要的信息，包括身份验证类型、config和keytab（如果适用，详情请参见[filesystem](../develop/filesystem.md）)。
+>
+> 当`experimental.spiller-spill-to-hdfs`设置为`true`时，必须配置此属性。所有Coordinator和Worker的配置文件中必须包含此属性。指定的文件系统必须可由所有Worker访问，并且Worker必须能够读取和写入指定文件系统中`experimental.spiller-spill-path`文件夹中指明的路径。
+
 ## 交换属性
 
 在openLooKeng节点之间为查询的不同阶段交换数据。调整这些属性可有助于解决节点间通信问题或提高网络利用率。
@@ -259,21 +354,124 @@
 > 
 > 如果网络延迟较高，增大该值可以提高网络吞吐量。减小该值可以提高大型集群的查询性能，因为它减少了由于交换客户端缓冲区保存了较多任务（而不是保存较少任务中的较多数据）的响应而导致的倾斜。
 
-### `exchange.max-error-duration`
-
-> - **类型：** `duration`
-> - **最小值：** `1m`
-> - **默认值：** `7m`
-> 
-> 交换错误最大缓冲时间，超过该时限则查询失败。
-
 ### `sink.max-buffer-size`
 
 > - **类型：** `data size`
 > - **默认值：** `32MB`
 > 
-> 上游任务等待拉取任务数据的输出缓冲区大小。如果任务输出是经过哈希分区的，那么缓冲区将在所有分区的使用者之间共享。如果网络延迟较高或集群中有多个节点，增加此值可以提高在阶段之间传输的数据的网络吞吐量。
-
+>等待上游任务拉取的任务数据的输出缓冲区大小。如果任务输出是哈希分区的，则缓冲区将在所有分区的消费者之间共享。如果网络延迟高或集群中有许多节点，则增加此值可以提高阶段之间传输数据的网络吞吐量。
+ 
+ ## 故障恢复处理属性
+ 
+ ### 失败重试策略
+ 
+ ### `failure.recovery.retry.profile`
+ 
+ > -   **类型：** `string`
+ > -   **默认值：** `default`
+ >
+ > 此属性定义用于确定HTTP客户端上是否发生故障的故障检测配置文件。此属性的值`<profile-name>`必须对应`etc/failure-retry-policy/`路径中的`<profile-name>.properties`文件。如果没有此类配置文件可用，并且未设置此属性，则使用“default”配置文件。
+ > 例如，`failure.recovery.retry.profile="test"`要求`test.properties`文件存在于`etc/failure-retry-policy`路径中。
+ > `test.properties`文件必须包含指定的`failure.recovery.retry.type`。
+ 
+ 
+ ### `failure.recovery.retry.type`
+ 
+ > -   **类型：** `string`
+ > -   **默认值：** `timeout`
+ >
+ > 此属性用来设置正在使用的故障检测机制。默认值是基于`timeout`的故障检测。
+ 
+ #### 基于`timeout`的故障检测
+ 
+ > 如果使用此机制，HTTP客户端故障将在指定时间段内重试，重试失败则被视为永久故障。
+ >
+ > 可以为此类故障检测定义`max.error.duration`属性。
+ 
+ #### 基于`max-retry`的故障检测
+ 
+ > 如果使用此机制，HTTP客户端故障将在被视为永久故障之前重试指定次数。
+ > 可以为此类故障检测定义`max.retry.count`和`max.error.duration`属性。
+ > 在这种类型的故障检测中，在查询故障检测模块之前，会执行`max.retry.count`次重试。当故障检测器模块检测到远程节点发生故障时，HTTP客户端将此故障视为永久故障。否则，例如，当远程工作节点处于活动状态但没有响应时，在`max.error.duration`指定的时间段内重试，重试失败则被视为永久故障。
+ 
+ ### `max.error.duration`
+ 
+ > -   **类型：** `duration`
+ > -   **默认值：** `300s`
+ >
+ > 被视为永久故障前，协调器等待解决任务间相关错误的最长时间。
+ 
+ ### `max.retry.count`
+ 
+ > -   **类型：** `integer`
+ > -   **默认值：** `100`
+ >
+ > 协调器在向故障检测器模块查询远程节点状态之前，对失败任务执行的最大重试次数。
+ > 此属性指定查询失败检测模块之前的最小重试次数。因此，实际故障数量可能会因为集群大小和集群负载而略有不同。
+ > 此属性仅用于基于`max-retry`的故障检测配置文件。
+ > 最小值为100。
+ 
+ ### 故障检测Gossip协议配置
+ 
+ ### `failure-detection-protocol`
+ 
+ >- **类型：** `string`
+ >- **默认值：** `heartbeat`
+ >
+ >此属性定义正在使用的故障检测器的类型。默认配置为`heartbeat`故障检测器。
+ >在`config.properties`文件中，将此属性配置为`gossip`，可以启用Gossip协议。
+ >集群中的所有节点（即协调器和工作节点）都应在其各自的`etc/config.properties`文件中指定此属性。
+ 
+ ### `failure-detector.heartbeat-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `500ms` （500毫秒）
+ >
+ >集群中两个节点之间的消息散播间隔。
+ >在Gossip协议中，两个工作节点间的消息散播频率高于协调器和一个工作节点间。
+ >在协调器的`config.properties`文件中，可以为此属性配置一个较大的值，例如`5s`（5秒）。
+ >在工作节点中，可以使用默认值。
+ 
+ ### `failure-detector.worker-gossip-probe-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `5s`（5秒）
+ >
+ >Gossip协议使用监控任务（与`heartbeat`故障检测器相同）来监控其他节点。
+ >此属性指定监控任务刷新间隔，以触发工作节点消息散播。
+ >仅可以为工作节点指定默认值以外的任何其他值。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
+ 
+ ### `failure-detector.coordinator-gossip-probe-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `5s`（5秒）
+ >
+ >Gossip协议使用监控任务（与heartbeat故障检测器相同）来监控其他节点。
+ >此属性指定监控任务刷新间隔，以触发协调器参与工作节点消息散播。
+ >仅可以为协调器指定默认值以外的任何其他值。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`和`failure-detector.worker-gossip-probe-interval`的值。
+ 
+ ### `failure-detector.coordinator-gossip-collate-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `2s`（2秒）
+ >
+ >此属性指定协调器整理从所有工作节点获得的所有散播消息的间隔。
+ >此属性只支持为协调器配置。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
+ 
+ ### `failure-detector.gossip-group-size`
+ 
+ >- **类型：** `integer`
+ >- **默认值：** `Integer.MAX_VALUE`
+ >
+ >此属性定义单个工作节点在集群中散播消息的工作节点数量。
+ >任何大于集群大小（即工作节点数量）的值都意味着all-to-all消息散播。
+ >要保持较低的网络开销，针对大型集群，请将此属性设置为一个较小的值（例如，100工作节点的集群设置为10）。
+ >每次刷新协调器上的工作节点监视任务时，协调器都会定义工作节点URI列表，其大小由`failure-detector.gossip-group-size`指定，以触发worker-to-worker消息散播。
+ >
+>
 ## 任务属性
 
 ### `task.concurrency`
@@ -517,7 +715,7 @@
 
 ## 启发式索引属性
 
-启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持使用ORC存储格式的表支持蜂巢连接器。
+启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持Hive连接器的ORC存储格式的表。
 
 ### `hetu.heuristicindex.filter.enabled`
 
@@ -618,7 +816,7 @@
 
 ### `hetu.split-cache-map.enabled`
 
-> - **类型：**`boolean`
+> - **类型：** `boolean`
 > - **默认值：** `false`
 > 
 > 此属性启用分片缓存功能。 如果启用了状态存储，则分片缓存映射配置也会自动复制到状态存储中。 在具有多个协调器的HA设置的情况下，状态存储用于在协调器之间共享分片的缓存映射。
@@ -634,7 +832,7 @@
 
 > 自动清空使系统能够通过持续监测需要清空的表来自动管理清空作业，以保持最佳性能。引擎从符合清空条件的数据源获取表，并触发对这些表的清空操作。
 
-### `auto-vacuum.enabled:`
+### `auto-vacuum.enabled`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -661,7 +859,7 @@
 >
 > **注意：** 此属性只能在协调节点中配置。
 
-## **CTE属性**
+## CTE属性
 
 ### `cte.cte-max-queue-size`
 
@@ -700,18 +898,25 @@
 > 
 > 远程任务错误最大缓冲时间，超过该时限则查询失败。
 
-## 分布式快照
+## 查询恢复
+
+### `recovery_enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 此会话属性用于启用或禁用恢复框架，该框架在发生故障时启用或禁用查询重启/恢复。
 
 ### `snapshot_enabled`
 
-> - 类型：`boolean`
-> - **默认值**：`false`
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
 >
-> 此会话属性用于启用或禁用分布式快照功能。
+> 启用恢复框架时，启用此会话属性可以在查询执行期间捕获快照。如果未启用恢复框架，则此属性不生效。
 
 ### `hetu.experimental.snapshot.profile`
 
-> - 类型：`string`
+> - **类型：**`string`
 >
 > 此属性定义用于存储快照的[文件系统](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果将该属性设置为`hetu.experimental.snapshot.profile=snapshot-hdfs1`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`snapshot-hdfs1.properties`，其中包含的必要信息包括身份验证类型、配置和密钥表（如适用）。具体细节请参考[文件系统](../develop/filesystem.md)相关章节。
 >
@@ -719,20 +924,58 @@
 >
 > 作为实验性属性，或可以将快照存储在非文件系统位置，如连接器。
 
-### `hetu.snapshot.maxRetries`
+### `hetu.recovery.maxRetries`
 
-> - 类型：`int`
-> - **默认值**：`10`
+> - **类型：** `integer`
+> - **默认值：** `10`
 >
-> 此属性定义查询的错误恢复尝试的最大次数。达到限制时，查询失败。
+> 此属性定义查询错误恢复尝试的最大次数。当达到限制时，查询失败。
 >
-> 也可以使用`snapshot_max_retries`会话属性在每个查询基础上指定。
+> 也可以使用`recovery_max_retries`会话属性为每个查询指定此属性。
 
-### `hetu.snapshot.retryTimeout`
+### `hetu.recovery.retryTimeout`
 
-> - 类型：`duration`
-> - **默认值：**`10m`（10分钟）
+> - **类型：** `duration`
+> - **默认值：** `10m`（10分钟）
 >
-> 此属性定义系统等待所有任务成功恢复的最大时长。如果在此超时时限内任何任务未就绪，则认为恢复失败，查询将尝试从较早快照恢复（如果可用）。
+> 此属性定义系统等待所有任务成功恢复的最长时间。如果在此时间内有任何任务未就绪，则恢复尝试将被视为失败，查询将尝试从较早的快照恢复（如果可用）。
 >
-> 也可以使用`snapshot_retry_timeout`会话属性在每个查询基础上指定。
+> 也可以使用`recovery_retry_timeout`会话属性为每个查询指定此属性。
+
+### `hetu.snapshot.useKryoSerialization`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 为快照启用基于Kryo的序列化，而不是默认的Java序列化。
+
+## HTTP客户端属性配置
+
+### `http.client.idle-timeout`
+
+> -   **类型:** `duration`
+> -   **默认值:** `30s` （30秒）
+>
+> 此参数定义了当http客户端没有任何操作时，其保持连接的时间。
+> 当超过指定时间还没有任何操作的话，将关闭客户端并释放相关资源。
+>
+> （注意：建议在高负载环境下，该参数配置大一点。）
+
+### `http.client.request-timeout`
+
+> -   **类型:** `duration`
+> -   **默认值:** `10s` （10秒）
+>
+> 此参数定义了http客户端接收响应的时间阈值。
+> 当超过所配置时间，客户端没有接收到任何响应，则视为客户端的请求提交失败。
+>
+> （注意： 建议在高负载环境下，该参数配置大一点。）
+
+## 连接器属性配置
+
+### `case-insensitive-name-matching`
+
+> -   **类型:** `boolean`
+> -   **默认值:** `false`
+>
+> 不区分大小写匹配数据库和集合名称，默认区分大小写。

hetu-docs/zh/admin/reliable-execution.md

@@ -10,9 +10,9 @@
 
 自版本1.2.0起，openLooKeng支持恢复任务和工作节点故障。
 
-## 启用分布式快照
-
-分布式快照适用于长时间运行的查询任务。该功能默认为禁用状态，可以通过会话属性[`snapshot_enabled`](properties.md#snapshot_enabled)启用或禁用。建议仅在对可靠性要求高的复杂查询场景下启用该功能。
+## 启用恢复框架
+     
+恢复框架对于长时间运行的查询最有用。默认禁用，可以使用会话属性[`recovery_enabled`](properties.md#recovery_enabled)启用和禁用恢复框架。建议仅对可靠性要求高的复杂查询启用该功能。
 
 ## 要求
 
@@ -35,7 +35,7 @@
 
 ## 检测
 
-协调节点与远程任务之间的通信长时间失败时，将触发错误恢复，由[`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration)配置控制。
+当协调器与远程任务之间的通信长时间失败时，将触发错误恢复，由[`故障恢复处理属性`](properties.md#故障恢复处理属性)配置控制。
 
 ## 存储注意事项
 
@@ -53,8 +53,20 @@
 
 从错误和快照中恢复需要成本。捕获快照需要时间，时间长短取决于复杂性。因此，需要在性能和可靠性之间进行权衡。
 
-建议仅在必要时启用分布式快照，如运行时间较长的查询任务。对于这些类型的工作负载，捕获快照的开销可以忽略不计。
+建议在必要时打开快照捕获，例如对于长时间运行的查询。对于这些类型的工作负载，拍摄快照的开销可以忽略不计。
+
+## 快照统计信息
+
+在调试模式下启动CLI时，快照捕获信息和恢复信息将与查询结果一起显示在CLI中。
+
+快照捕获统计信息包括捕获的快照数量、捕获的快照大小、捕获快照所需的CPU时间和在查询期间捕获快照所需的挂钟时间。所有快照和最后一个快照的统计信息会分别显示。
+
+快照恢复信息包括查询期间从快照恢复的次数、加载用于恢复的快照大小、从快照恢复所需的CPU时间和从快照恢复所需的挂钟时间。仅当查询期间发生恢复时，才会显示恢复信息。
+
+此外，在查询正在进行时，将显示捕获的快照数量和恢复的快照的ID。更多详细信息，见下图。
+
+![](../images/snapshot_statistics_cn.png)
 
 ## 配置
 
-与分布式快照功能相关的配置可参见[属性参考](properties.md#分布式快照)。
+与恢复框架功能相关的配置，请参见[属性参考](properties.md#查询恢复)。

hetu-docs/zh/admin/spill.md

@@ -31,6 +31,10 @@
 
 openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD )），因此无需使用RAID进行溢出。
 
+## 溢出到HDFS
+
+操作可以直接溢出到HDFS。将`experimental.spiller-spill-to-hdfs`设置为`true`，配置`experimental.spiller-spill-profile`，并且`spiller-spill-path`必须仅包含一个目录。（更多详情请参见`experimental.spiller-spill-to-hdfs`和`experimental.spiller-spill-profile`属性）
+
 ## 溢出压缩
 
 当启用溢出压缩（`tuning-spilling`中的`spill-compression-enabled`属性）时，溢出页将被压缩后再写入磁盘。启用此特性可以减少磁盘I/O，但会牺牲额外的CPU负载来压缩和解压缩溢出页。
@@ -53,6 +57,8 @@ openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipe
 
 通过这种机制，联接操作符使用的峰值内存可以降低到最大构建表分区的大小。假设没有数据倾斜，这个值将是整个构建表大小的`1 / task.concurrency`倍。
 
+注意：spill-to-disk不支持交叉连接。
+
 ### 聚合
 
 聚合函数对一组值执行操作并返回一个值。如果要聚合的组数量很大，可能需要大量内存。当启用溢出到磁盘时，如果没有足够的内存，则中间累积的聚合结果将写入磁盘。结果被重新加载回来，并以较低的内存占用量合并。
@@ -60,6 +66,7 @@ openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipe
 ### 排序
 
 如果尝试对大量数据进行排序，可能需要大量内存。当启用为排序溢出到磁盘时，如果内存不足，则中间排序结果将写入磁盘。结果被重新加载回来，并以较低的内存占用量合并。
+通常，当溢出正在进行时，运算符将被阻止接受输入，但当`experimental.spill-non-blocking-orderby`设置为`true`时，使用异步机制溢出（请参见`experimental.spill-non-blocking-orderby`）。
 
 ### 开窗函数
 

hetu-docs/zh/admin/web-interface.md

@@ -31,3 +31,54 @@ openLooKeng提供了一个用于监视和管理查询的Web界面。Web界面可
 > -   **默认值:** `false`
 >
 > 默认情况下，基于HTTP的非安全环境禁用WEB UI。可以通过配置`etc/config.properties`文件的`hetu.queryeditor-ui.allow-insecure-over-http`属性启用(例子: hetu.queryeditor-ui.allow-insecure-over-http=true)。
+
+### `hetu.queryeditor-ui.execution-timeout`
+
+> -   **类型：** `duration`
+> -   **默认值：** `100 DAYS`
+>
+> UI执行超时默认设置为100天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.execution-timeout`属性修改。
+
+### `hetu.queryeditor-ui.max-result-count`
+
+> - **类型：** `int`
+> - **默认值：** `1000`
+>
+> UI最大结果计数默认设置为1000。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.max-result-count`属性修改。
+
+### `hetu.queryeditor-ui.max-result-size-mb`
+
+>- **类型：** `size`
+>- **默认值：** `1GB`
+>
+>UI最大结果大小默认设置为1 GB。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.max-result-size-mb`属性修改。
+
+### `hetu.queryeditor-ui.session-timeout`
+
+> -   **类型：** `duration`
+> -   **默认值：** `1 DAYS`
+>
+> UI会话超时默认设置为1天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.session-timeout`属性修改。
+
+### `hetu.queryhistory.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `1000`
+>
+> openLooKeng储存的历史查询记录最大数量。可以通过配置`etc/config.properties`文件的`hetu.queryhistory.max-count`属性修改。
+
+### `hetu.collectionsql.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `100`
+>
+> 每位用户收藏sql语句条数上限.可以通过配置`etc/config.properties`文件的`hetu.collectionsql.max-count`属性修改。
+
+
+## 备注
+
+收藏sql语句的最大长度默认为600，可通过如下步骤对其进行修改：
+
+1. 根据hetu-metastore.properties文件中jdbc配置信息登录mysql数据库。
+
+2. 选中hetu_favorite表，使用命令`alter table hetu_favorite modify query varchar(2000) not null;`修改收藏语句最大长度为2000。

hetu-docs/zh/connector/Hudi.md

@@ -1,5 +1,8 @@
 # Hudi连接器
 
+### 版本说明
+目前Hudi只支持0.7.0版本。
+
 ### Hudi介绍
 
 Apache Hudi是一个快速迭代的数据湖存储系统，可以帮助企业构建和管理PB级数据湖。它提供在DFS上存储超大规模数据集，同时使得流式处理如果批处理一样，该实现主要是通过如下两个原语实现。