!1531 Fixed 0 row count issue while recovering

Merge pull request !1531 from Kishore Kumar M V/0rows_after_recovery

hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.5.0-SNAPSHOT</version>
+    <version>1.7.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,8 +108,7 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this.metastore = metastore;
-            this.schemaName = null;
+            this(metastore, null);
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -233,7 +232,6 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
-        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,16 +73,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putShorts(rowId, count, value);
+            columnVector.putShorts(inputRowId, count, value);
         }
     }
 
@@ -97,16 +98,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putInts(rowId, count, value);
+            columnVector.putInts(inputRowId, count, value);
         }
     }
 
@@ -121,16 +123,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putLongs(rowId, count, value);
+            columnVector.putLongs(inputRowId, count, value);
         }
     }
 
@@ -145,11 +148,12 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -164,16 +168,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putDoubles(rowId, count, value);
+            columnVector.putDoubles(inputRowId, count, value);
         }
     }
 
@@ -196,11 +201,12 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -223,16 +229,17 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
+        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[rowId]) {
+                if (!filteredRows[inputRowId]) {
                     columnVector.putNull(counter++);
                 }
-                rowId++;
+                inputRowId++;
             }
         }
         else {
-            columnVector.putNulls(rowId, count);
+            columnVector.putNulls(inputRowId, count);
         }
     }
 
@@ -319,66 +326,72 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[rowId]) {
+            if (!filteredRows[inputRowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -127,15 +127,16 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        this.outPutPath = requireNonNull(outPutPath, "path is null");
+        Path localOutPutPath = paramOutPutPath;
+        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            outPutPath = new Path(properties.getProperty("location"));
+            localOutPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -211,7 +212,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -226,25 +227,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path outPutPath = this.outPutPath;
-        Properties properties = this.properties;
-        JobConf configuration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
+        Path finalOutPutPath = this.outPutPath;
+        Properties finalProperties = this.properties;
+        JobConf finalConfiguration = this.configuration;
+        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
-                        properties, Reporter.NULL);
+                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
+                        finalProperties, Reporter.NULL);
     }
 
     @Override
@@ -285,7 +286,7 @@ public class CarbondataFileWriter
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter recordWriter = null;
+        FileSinkOperator.RecordWriter finalRecordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -334,7 +335,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -351,7 +352,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            recordWriter = this.recordWriter;
+            finalRecordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -365,8 +366,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (recordWriter != null) {
-                recordWriter.write(serDe.serialize(row, tableInspector));
+            if (finalRecordWriter != null) {
+                finalRecordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,6 +48,7 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
+    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java

@@ -306,13 +306,13 @@ public class CarbondataMetadata
 
     private void setupCommitWriter(Properties hiveSchema, Path outputPath, Configuration initialConfiguration, boolean isOverwrite) throws PrestoException
     {
-        CarbonLoadModel carbonLoadModel;
+        CarbonLoadModel finalCarbonLoadModel;
         TaskAttemptID taskAttemptID = TaskAttemptID.forName(initialConfiguration.get("mapred.task.id"));
         try {
             ThreadLocalSessionInfo.setConfigurationToCurrentThread(initialConfiguration);
-            carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
-            carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
-            CarbonTableOutputFormat.setLoadModel(initialConfiguration, carbonLoadModel);
+            finalCarbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
+            finalCarbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
+            CarbonTableOutputFormat.setLoadModel(initialConfiguration, finalCarbonLoadModel);
         }
         catch (IOException ex) {
             LOG.error("Error while creating carbon load model", ex);
@@ -360,13 +360,13 @@ public class CarbondataMetadata
         this.user = session.getUser();
         return hdfsEnvironment.doAs(user, () -> {
             SchemaTableName tableName = parent.getSchemaTableName();
-            Optional<Table> table =
+            Optional<Table> finalTable =
                     metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-            if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+            if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
                 throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
             }
 
-            this.table = table;
+            this.table = finalTable;
             Path outputPath =
                     new Path(parent.getLocationHandle().getJsonSerializableTargetPath());
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -382,7 +382,7 @@ public class CarbondataMetadata
             }
 
             /* Create committer object */
-            setupCommitWriter(table, outputPath, initialConfiguration, isOverwrite);
+            setupCommitWriter(finalTable, outputPath, initialConfiguration, isOverwrite);
 
             return new CarbondataInsertTableHandle(parent.getSchemaName(),
                     parent.getTableName(),
@@ -416,13 +416,13 @@ public class CarbondataMetadata
         currentState = State.UPDATE;
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> table =
+        Optional<Table> finalTable =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = table;
+        this.table = finalTable;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -430,8 +430,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", table.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -470,13 +470,13 @@ public class CarbondataMetadata
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         List<HiveColumnHandle> inputColumns = parent.getInputColumns().stream().filter(HiveColumnHandle::isRequired).collect(toList());
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> table =
+        Optional<Table> finalTable =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
+        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = table;
+        this.table = finalTable;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -484,8 +484,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", table.get().getStorage().getLocation());
+            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -643,7 +643,7 @@ public class CarbondataMetadata
 
         return hdfsEnvironment.doAs(session.getUser(), () -> {
             Properties hiveSchema = MetastoreUtil.getHiveSchema(this.table.get());
-            CarbonTable carbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
+            CarbonTable finalCarbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
                     carbondataVacuumTableHandle.getTableName(),
                     hiveSchema,
                     initialConfiguration);
@@ -705,7 +705,7 @@ public class CarbondataMetadata
                     SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath()));
                     String source;
                     for (String currPartitionName : partitionNames) {
-                        source = carbonTable.getTablePath() + "/" + currPartitionName;
+                        source = finalCarbonTable.getTablePath() + "/" + currPartitionName;
                         moveFromTempFolder(source + "/" + carbonLoadModel.getSegmentId() + "_" + timeStamp + ".tmp", source);
                     }
                     segmentFilesToBeUpdatedLatest.add(new Segment(carbonLoadModel.getSegmentId(), segmentFileName));
@@ -719,7 +719,7 @@ public class CarbondataMetadata
                 for (CarbondataSegmentInfoUtil segmentInfo : newMergedSegmentInfoUtilList) {
                     String mergedLoadNumber = segmentInfo.getDestinationSegment();
                     try {
-                        String segmentFileName = SegmentFileStore.writeSegmentFile(carbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
+                        String segmentFileName = SegmentFileStore.writeSegmentFile(finalCarbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
                     }
                     catch (IOException e) {
                         throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while merging segment files", e);
@@ -900,9 +900,9 @@ public class CarbondataMetadata
     private LocationHandle getCarbonDataTableCreationPath(ConnectorSession session, ConnectorTableMetadata tableMetadata, HiveWriteUtils.OpertionType opertionType) throws PrestoException
     {
         Path targetPath = null;
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String tableName = finalSchemaTableName.getTableName();
         Optional<String> location = getCarbondataLocation(tableMetadata.getProperties());
         LocationHandle locationHandle;
         FileSystem fileSystem;
@@ -914,32 +914,32 @@ public class CarbondataMetadata
                     throw new PrestoException(NOT_SUPPORTED, format("Setting %s property is not allowed", LOCATION_PROPERTY));
                 }
                 /* if path not having prefix with filesystem type, than we will take fileSystem type from core-site.xml using below methods */
-                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, schemaName), new Path(location.get()));
+                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, finalSchemaName), new Path(location.get()));
                 targetLocation = fileSystem.getFileStatus(new Path(location.get())).getPath().toString();
-                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, schemaName, tableName), targetLocation, false);
+                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, finalSchemaName, tableName), targetLocation, false);
             }
             else {
-                updateEmptyCarbondataTableStorePath(session, schemaName);
+                updateEmptyCarbondataTableStorePath(session, finalSchemaName);
                 targetLocation = carbondataTableStore;
-                targetLocation = targetLocation + File.separator + schemaName + File.separator + tableName;
+                targetLocation = targetLocation + File.separator + finalSchemaName + File.separator + tableName;
                 targetPath = new Path(targetLocation);
             }
         }
         catch (IllegalArgumentException | IOException e) {
             throw new PrestoException(NOT_SUPPORTED, format("Error %s store path %s ", e.getMessage(), targetLocation));
         }
-        locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
+        locationHandle = locationService.forNewTable(metastore, session, finalSchemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
         return locationHandle;
     }
 
     @Override
     public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
     {
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName localSchemaTableName = tableMetadata.getTable();
+        String localSchemaName = localSchemaTableName.getSchemaName();
+        String tableName = localSchemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = schemaName;
+        this.schemaName = localSchemaName;
         currentState = State.CREATE_TABLE;
         List<String> partitionedBy = new ArrayList<String>();
         List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@@ -947,29 +947,29 @@ public class CarbondataMetadata
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
 
-        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
+        metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
 
         BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
         Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
 
-        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                        new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
+                        new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
                         new Path(locationHandle.getJsonSerializableTargetPath())));
 
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             try {
                 Map<String, String> serdeParameters = initSerDeProperties(tableName);
-                Table table = buildTableObject(
+                Table localTable = buildTableObject(
                         session.getQueryId(),
-                        schemaName,
+                        localSchemaName,
                         tableName,
                         session.getUser(),
                         columnHandles,
@@ -981,11 +981,11 @@ public class CarbondataMetadata
                         true, // carbon table is set as external table
                         prestoVersion,
                         serdeParameters);
-                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
-                HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
+                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
+                HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
                 metastore.createTable(
                         session,
-                        table,
+                        localTable,
                         principalPrivileges,
                         Optional.empty(),
                         ignoreExisting,
@@ -1092,8 +1092,8 @@ public class CarbondataMetadata
     public CarbondataTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
     {
         requireNonNull(tableName, "tableName is null");
-        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!table.isPresent()) {
+        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!finalTable.isPresent()) {
             return null;
         }
 
@@ -1102,14 +1102,14 @@ public class CarbondataMetadata
             throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, "Unexpected table present in Hive metastore: " + tableName);
         }
 
-        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(table.get()), table.get().getParameters());
+        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(finalTable.get()), finalTable.get().getParameters());
 
         return new CarbondataTableHandle(
                 tableName.getSchemaName(),
                 tableName.getTableName(),
-                table.get().getParameters(),
-                getPartitionKeyColumnHandles(table.get()),
-                HiveBucketing.getHiveBucketHandle(table.get()));
+                finalTable.get().getParameters(),
+                getPartitionKeyColumnHandles(finalTable.get()),
+                HiveBucketing.getHiveBucketHandle(finalTable.get()));
     }
 
     private Optional<ConnectorOutputMetadata> finishUpdateAndDelete(ConnectorSession session,
@@ -1133,12 +1133,12 @@ public class CarbondataMetadata
 
         hdfsEnvironment.doAs(user, () -> {
             if (blockUpdateDetailsList.size() > 0) {
-                CarbonTable carbonTable = getCarbonTable(tableHandle.getSchemaName(),
+                CarbonTable finalCarbonTable = getCarbonTable(tableHandle.getSchemaName(),
                         tableHandle.getTableName(),
                         MetastoreUtil.getHiveSchema(table.get()),
                         initialConfiguration);
 
-                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(carbonTable);
+                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(finalCarbonTable);
                 SegmentUpdateDetails[] segementDetailsList = statusManager.getUpdateStatusDetails();
                 for (SegmentUpdateDetails segementDetails : segementDetailsList) {
                     segementDetails.getDeletedRowsInBlock();
@@ -1179,26 +1179,26 @@ public class CarbondataMetadata
                                             List<HiveColumnHandle> columnHandles,
                                             Map<String, String> tableProperties)
     {
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String finalTableName = finalSchemaTableName.getTableName();
 
         partitionedBy.addAll(CarbondataTableProperties.getPartitionedBy(tableMetadata.getProperties()));
         sortBy.addAll(CarbondataTableProperties.getSortedBy(tableMetadata.getProperties()));
         Optional<HiveBucketProperty> bucketProperty = Optional.empty();
         columnHandles.addAll(getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator));
-        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, schemaName, tableName)));
+        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName)));
     }
 
     @Override
     public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
     {
         // get the root directory for the database
-        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String schemaName = schemaTableName.getSchemaName();
-        String tableName = schemaTableName.getTableName();
+        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String finalTableName = finalSchemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = schemaName;
+        this.schemaName = finalSchemaName;
         currentState = State.CREATE_TABLE_AS;
 
         List<String> partitionedBy = new ArrayList<String>();
@@ -1206,7 +1206,7 @@ public class CarbondataMetadata
         List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
-        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
+        metastore.getDatabase(finalSchemaName).orElseThrow(() -> new SchemaNotFoundException(finalSchemaName));
 
         // to avoid type mismatch between HiveStorageFormat & Carbondata StorageFormat this hack no option
         HiveStorageFormat tableStorageFormat = HiveStorageFormat.valueOf("CARBON");
@@ -1222,29 +1222,29 @@ public class CarbondataMetadata
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE_AS);
         Path targetPath = locationService.getTableWriteInfo(locationHandle, false).getTargetPath();
-        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
+        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+                new CarbonTableIdentifier(finalSchemaName, finalTableName, UUID.randomUUID().toString()));
 
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                                new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
+                                new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName),
                                 new Path(locationHandle.getJsonSerializableTargetPath())));
             // Create Carbondata metadata folder and Schema file
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             Path outputPath = new Path(locationHandle.getJsonSerializableTargetPath());
-            Properties schema = readSchemaForCarbon(schemaName, tableName, targetPath, columnHandles, partitionColumns);
+            Properties schema = readSchemaForCarbon(finalSchemaName, finalTableName, targetPath, columnHandles, partitionColumns);
             // Create committer object
             setupCommitWriter(schema, outputPath, initialConfiguration, false);
         });
         try {
             CarbondataOutputTableHandle result = new CarbondataOutputTableHandle(
-                    schemaName,
-                    tableName,
+                    finalSchemaName,
+                    finalTableName,
                     columnHandles,
-                    metastore.generatePageSinkMetadata(new HiveIdentity(session), schemaTableName),
+                    metastore.generatePageSinkMetadata(new HiveIdentity(session), finalSchemaTableName),
                     locationHandle,
                     tableStorageFormat,
                     partitionStorageFormat,
@@ -1255,7 +1255,7 @@ public class CarbondataMetadata
                     EncodedLoadModel, jobContext.getConfiguration().get(LOAD_MODEL)));
 
             LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
-            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
+            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), finalSchemaTableName);
             return result;
         }
         catch (RuntimeException ex) {
@@ -1386,7 +1386,7 @@ public class CarbondataMetadata
         List<Segment> segmentFilesToBeUpdated = blockUpdateDetailsList.stream()
                 .map(SegmentUpdateDetails::getSegmentName)
                 .map(Segment::new).collect(Collectors.toList());
-        List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
+        List<Segment> finalSegmentFilesToBeUpdatedLatest = new ArrayList<>();
         List<Segment> segmentFilesToBeDeleted = blockUpdateDetailsList.stream()
                 .filter(segmentUpdateDetails -> segmentUpdateDetails.getSegmentStatus() != null &&
                         segmentUpdateDetails.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE))
@@ -1396,12 +1396,12 @@ public class CarbondataMetadata
         for (Segment segment : segmentFilesToBeUpdated) {
             String file =
                     SegmentFileStore.writeSegmentFile(carbonTable, segment.getSegmentNo(), timeStamp.toString());
-            segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
+            finalSegmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
         }
         if (!(updateSegmentStatusSuccess &&
                 CarbonUpdateUtil.updateTableMetadataStatus(new HashSet<>(segmentFilesToBeUpdated),
                         carbonTable, timeStamp.toString(), true, segmentFilesToBeDeleted,
-                        segmentFilesToBeUpdatedLatest, ""))) {
+                        finalSegmentFilesToBeUpdatedLatest, ""))) {
             CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timeStamp.toString());
         }
     }
@@ -1463,11 +1463,10 @@ public class CarbondataMetadata
                 Properties hiveschema = MetastoreUtil.getHiveSchema(table);
                 Configuration configuration = jobContext.getConfiguration();
                 configuration.set(SET_OVERWRITE, "false");
-                CarbonLoadModel carbonLoadModel =
-                        HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
-                LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
-                carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
-                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
+                CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
+                LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
+                loadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
             }
             catch (IOException e) {
                 LOG.error("Error occurred while committing the insert job.", e);
@@ -1554,14 +1553,14 @@ public class CarbondataMetadata
         try {
             hdfsEnvironment.doAs(session.getUser(), () -> {
                 metastore.dropTable(session, handle.getSchemaName(), handle.getTableName());
-                Configuration initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
+                Configuration finalInitialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
                         .getConfiguration(new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
                                 handle.getTableName()), new Path(this.tableStorageLocation.get())));
 
                 Properties schema = MetastoreUtil.getHiveSchema(target.get());
                 schema.setProperty("tablePath", this.tableStorageLocation.get());
                 this.carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(),
-                        schema, initialConfiguration);
+                        schema, finalInitialConfiguration);
                 takeLocks(State.DROP_TABLE);
                 AbsoluteTableIdentifier identifier = this.carbonTable.getAbsoluteTableIdentifier();
                 if (SegmentStatusManager.isLoadInProgressInTable(carbonTable)) {
@@ -1570,7 +1569,7 @@ public class CarbondataMetadata
                 try {
                     //Simultaneous case after acquiring locks we should check table exist.
                     //if table is not there clean the lock folders
-                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, initialConfiguration);
+                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, finalInitialConfiguration);
                 }//CarbonFileException
                 catch (RuntimeException e) {
                     try {
@@ -1867,8 +1866,8 @@ public class CarbondataMetadata
     {
         String tableName = absoluteTableIdentifier.getTableName();
         String databaseName = absoluteTableIdentifier.getDatabaseName();
-        TableInfo tableInfo = carbonTable.getTableInfo();
-        List<SchemaEvolutionEntry> evolutionEntryList = tableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
+        TableInfo finalTableInfo = carbonTable.getTableInfo();
+        List<SchemaEvolutionEntry> evolutionEntryList = finalTableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
         Long updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).getTimeStamp();
         LOG.info("Reverting changes for " + databaseName + "." + tableName);
         List<ColumnSchema> addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getAdded();
@@ -1880,7 +1879,7 @@ public class CarbondataMetadata
                     break;
                 }
                 case DROP_COLUMN: {
-                    tableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
+                    finalTableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
                         if (cols.isInvisible() && removedCols.getColumnUniqueId().equals(cols.getColumnUniqueId())) {
                             cols.setInvisible(false);
                         }
@@ -2027,38 +2026,38 @@ public class CarbondataMetadata
     @Override
     protected ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName)
     {
-        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        if (!finalTable.isPresent() || finalTable.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
             throw new TableNotFoundException(tableName);
         }
 
-        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
+        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(finalTable.get(), typeManager);
         ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
             columns.add(metadataGetter.apply(columnHandle));
         }
 
         // External location property
         ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
-        properties.put(LOCATION_PROPERTY, table.get().getStorage().getLocation());
+        properties.put(LOCATION_PROPERTY, finalTable.get().getStorage().getLocation());
 
         // Storage format property
         properties.put(HiveTableProperties.STORAGE_FORMAT_PROPERTY, CarbondataStorageFormat.CARBON);
 
         // Partitioning property
-        List<String> partitionedBy = table.get().getPartitionColumns().stream()
+        List<String> partitionedBy = finalTable.get().getPartitionColumns().stream()
                 .map(Column::getName)
                 .collect(toList());
         if (!partitionedBy.isEmpty()) {
             properties.put(HiveTableProperties.PARTITIONED_BY_PROPERTY, partitionedBy);
         }
 
-        Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));
+        Optional<String> comment = Optional.ofNullable(finalTable.get().getParameters().get(TABLE_COMMENT));
 
         // add partitioned columns into immutableColumns
         ImmutableList.Builder<ColumnMetadata> immutableColumns = ImmutableList.builder();
 
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
             if (columnHandle.getColumnType().equals(HiveColumnHandle.ColumnType.PARTITION_KEY)) {
                 immutableColumns.add(metadataGetter.apply(columnHandle));
             }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -191,7 +191,7 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore metastore =
+        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
@@ -200,7 +200,7 @@ public class CarbondataMetadataFactory
                         this.hiveTransactionHeartbeatInterval,
                         this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
 
-        return new CarbondataMetadata(metastore,
+        return new CarbondataMetadata(semiTransactionalHiveMetastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
                 this.writesToNonManagedTablesEnabled,
@@ -212,8 +212,8 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(metastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(metastore),
+                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
+                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        hdfsEnvironment.doAs(session.getUser(), () -> {
+        finalHdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,6 +163,7 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
+    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int batchSize = 0;
+        int columnBatchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    batchSize = columnarBatch.numRows();
-                    if (batchSize == 0) {
+                    columnBatchSize = columnarBatch.numRows();
+                    if (columnBatchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(batchSize, blocks);
+            Page page = new Page(columnBatchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,16 +90,17 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
+        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            name = name.substring(0, name.length() - 4).trim();
+            finalName = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            name = name.substring(0, name.length() - 5).trim();
+            finalName = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(name, order);
+        return new SortingColumn(finalName, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -133,6 +133,7 @@ public class CarbondataWriterFactory
     {
     }
 
+    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,14 +143,15 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putDecimal(rowId, value, precision);
+                columnVector.putDecimal(inputRowId, value, precision);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
@@ -185,8 +186,9 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(rowId++, value);
+            columnVector.putByteArray(inputRowId++, value);
         }
     }
 
@@ -303,84 +305,90 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putFloat(rowId, src[i]);
+                columnVector.putFloat(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putShort(rowId, src[i]);
+                columnVector.putShort(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putInt(rowId, src[i]);
+                columnVector.putInt(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putLong(rowId, src[i]);
+                columnVector.putLong(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putDouble(rowId, src[i]);
+                columnVector.putDouble(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(rowId);
+            if (nullBitSet.get(inputRowId)) {
+                columnVector.putNull(inputRowId);
             }
             else {
-                columnVector.putByte(rowId, src[i]);
+                columnVector.putByte(inputRowId, src[i]);
             }
-            rowId++;
+            inputRowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,8 +58,9 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
+        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIndex++] == 1);
+            type.writeBoolean(builder, src[srcIdx++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,8 +76,9 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
+        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(rowId++, value, precision);
+            putDecimal(id++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,8 +58,9 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
+        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(rowId++, value);
+            putInt(id++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java

@@ -15,12 +15,10 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import com.esotericsoftware.minlog.Log;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import io.prestosql.hive.$internal.au.com.bytecode.opencsv.CSVReader;
 import io.prestosql.spi.PrestoException;
-import io.prestosql.spi.StandardErrorCode;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -53,7 +51,6 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.text.DateFormat;
@@ -67,10 +64,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
-import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
@@ -1429,7 +1424,7 @@ public class TestCarbonAllDataType
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtabledrop", false), false);
 
         } catch (IOException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1474,7 +1469,7 @@ public class TestCarbonAllDataType
                     i++;
                 }
             } catch (IOException e) {
-                e.printStackTrace();
+                logger.error(e.getMessage());
             }
 
             // Step 3: convert format level TableInfo to code level TableInfo
@@ -1555,7 +1550,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1569,7 +1564,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1578,7 +1573,7 @@ public class TestCarbonAllDataType
                 return   "date '" + data + "'";
             }
             case "varchar":
-            {//'china'
+            {
                 return "'" + data + "'";
             }
             case "timestamp":
@@ -1592,7 +1587,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1603,7 +1598,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1614,7 +1609,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1625,7 +1620,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        e.printStackTrace();
+                        logger.error(e.getMessage());
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1637,9 +1632,10 @@ public class TestCarbonAllDataType
                 return dateString;
             }
             case "smallint":  {
-                // smallint '12'
                 return "smallint '" + data + "'";
             }
+            default:
+                break;
         }
         return data;
     }
@@ -1697,7 +1693,7 @@ public class TestCarbonAllDataType
             hetuServer.execute(inserData);
         }
         catch(Exception e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1736,7 +1732,7 @@ public class TestCarbonAllDataType
 
         }
         catch (IOException | InterruptedException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
     }
 
@@ -1785,14 +1781,15 @@ public class TestCarbonAllDataType
     */
     private boolean checkStatusFileForDeleteMarked(String tableName, int updateNumber, int segmentNumber) throws SQLException
     {
+        BufferedReader reader = null;
         try {
             File dir = new File(storePath + "/carbon.store/testdb/" + tableName + "/Metadata");
             File[] tableUpdateStatusFiles = dir.listFiles((d, name) -> name.startsWith("tableupdatestatus"));
             Arrays.sort(tableUpdateStatusFiles);
             Gson gson = new Gson();
-            BufferedReader reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
+            reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
             SegmentUpdateDetails[] segmentUpdateDetails = gson.fromJson(reader, SegmentUpdateDetails[].class);
-            File tableStatusFile = new File(dir.getAbsolutePath() + "/tablestatus");
+            File tableStatusFile = new File(dir.getCanonicalPath() + "/tablestatus");
             reader = new BufferedReader(new FileReader(tableStatusFile));
             LoadMetadataDetails loadMetadataDetails = gson.fromJson(reader, LoadMetadataDetails[].class)[segmentNumber];
             if ((segmentUpdateDetails[0].getSegmentStatus() != null && segmentUpdateDetails[0].getSegmentStatus().toString().equals("Marked for Delete")) &&
@@ -1803,6 +1800,16 @@ public class TestCarbonAllDataType
             hetuServer.execute("drop table if exists testdb." + tableName);
             Assert.fail("Failed to read status files");
         }
+        finally {
+            if (reader != null) {
+                try {
+                    reader.close();
+                }
+                catch (IOException e) {
+                    logger.error(e.getMessage());
+                }
+            }
+        }
         return false;
     }
 
@@ -1822,7 +1829,7 @@ public class TestCarbonAllDataType
                     "/carbon.store/testdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
@@ -1846,7 +1853,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
@@ -1879,7 +1886,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException | InterruptedException e) {
             hetuServer.execute("DROP TABLE if exists testdb.myectable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.myectable");
@@ -1904,7 +1911,7 @@ public class TestCarbonAllDataType
                 FileFactory.mkdirs( storePath + "/carbon.store/mytestDb");
             }
         } catch (IOException e) {
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         String location = "'" + "file:///" + storePath + "/carbon.store/mytestDb" + "')" ;

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -100,7 +100,6 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
-        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -151,7 +150,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-
+            logger.debug(e.getMessage());
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -217,7 +216,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-
+            logger.debug(e.getMessage());
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -130,7 +130,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -159,7 +159,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -189,7 +189,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -221,7 +221,7 @@ public class TestCarbondataAutoCleanup
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-
+                logger.debug(exception.getMessage());
             }
         }
         finally {
@@ -254,7 +254,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -285,7 +285,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -314,7 +314,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -344,7 +344,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -374,7 +374,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-
+            logger.debug(exception.getMessage());
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -403,7 +403,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                e.printStackTrace();
+                logger.error(e.getMessage());
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            e.printStackTrace();
+            logger.error(e.getMessage());
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -15,6 +15,7 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
+import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -36,6 +37,7 @@ import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
+    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -114,7 +116,7 @@ public class TestsWithHiveConnector
         assertEquals(FileFactory.isFileExist(storePath +
                 "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
         hetuServer.execute("DROP TABLE hive.default.parttable");
     }
@@ -141,7 +143,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2013", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
 
         hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
@@ -150,7 +152,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2014", false), false);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
 
         hetuServer.execute("DROP TABLE hive.default.parttable2");
@@ -178,7 +180,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "/hive.store/default/parttable3/year=2013", false), true);
         } catch (IOException exception) {
-            exception.printStackTrace();
+            log.error(exception.getMessage());
         }
         hetuServer.execute("DROP TABLE hive.default.parttable3");
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -192,7 +192,7 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
@@ -203,7 +203,7 @@ public class HetuTestServer
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
         }
         catch (RuntimeException e) {

hetu-clickhouse/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-clickhouse</artifactId>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -345,8 +345,9 @@ public class ClickHouseClient
     }
 
     @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
+    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
     {
+        String newColumnName = inputNewColumnName;
         try (Connection connection = connectionFactory.openConnection(identity)) {
             if (connection.getMetaData().storesUpperCaseIdentifiers()) {
                 newColumnName = newColumnName.toUpperCase(ENGLISH);

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -41,6 +41,7 @@ public class ClickHouseApplyRemoteFunctionPushDown
     /**
      * rewrite the remote function to a executable function in the data source.
      */
+    @Override
     public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
     {
         if (!isConnectorSupportedRemoteFunction(callExpression)) {

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -37,8 +37,9 @@ public class ClickHouseSqlStatementWriter
     }
 
     @Override
-    public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
+    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
     {
+        String functionName = inputFunctionName;
         if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
             functionName = "varPop";
         }

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -205,7 +205,7 @@ public final class ClickHouseServerTest
     {
         String actualTable = tablePattern;
 
-        for (String table : tables) { //tableName + _ + UUID
+        for (String table : tables) {
             int lastIndex = table.lastIndexOf("_");
             if (lastIndex == -1) {
                 continue;

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,16 +47,6 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
-
-        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
-        // as per link above, the default SSLContext will be constructed using the default KeyManager and
-        // default TrustManager. Those can be configured using the following system properties:
-        // javax.net.ssl.keyStore
-        // javax.net.ssl.keyStorePassword
-        // javax.net.ssl.keyStoreType
-        // javax.net.ssl.trustStore
-        // javax.net.ssl.trustStorePassword
-        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,6 +13,7 @@
  */
 package io.hetu.core.common.util;
 
+import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -29,6 +30,8 @@ import java.util.Optional;
 
 public class TrustStore
 {
+    private static final Logger LOGGER = Logger.get(TrustStore.class);
+
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -48,6 +51,7 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
+            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
+            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
+            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-cube</artifactId>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -36,8 +36,7 @@ public class CubeFilter
 
     public CubeFilter(String sourceTablePredicate)
     {
-        this.sourceTablePredicate = sourceTablePredicate;
-        this.cubePredicate = null;
+        this(sourceTablePredicate, null);
     }
 
     public String getSourceTablePredicate()

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -140,13 +140,13 @@ public class CubeStatement
             return this;
         }
 
-        public Builder groupBy(String column)
+        public Builder groupByAddString(String column)
         {
             this.groupBy.add(column);
             return this;
         }
 
-        public Builder groupBy(String... columns)
+        public Builder groupByAddStringList(String... columns)
         {
             this.groupBy.addAll(Arrays.asList(columns));
             return this;

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -34,8 +34,8 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
-                .groupBy("address")
-                .groupBy("name", "nationkey")
+                .groupByAddString("address")
+                .groupByAddStringList("name", "nationkey")
                 .build();
 
         assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,6 +55,7 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
+    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -56,11 +56,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this.catalogName = catalogName;
-        this.schemaName = requireNonNull(schemaName, "schemaName is null");
-        this.tableName = requireNonNull(tableName, "tableName is null");
-        this.limit = requireNonNull(limit, "limit is null");
-        this.pushDownSql = "";
+        this(catalogName,
+                requireNonNull(schemaName, "schemaName is null"),
+                requireNonNull(tableName, "tableName is null"),
+                requireNonNull(limit, "limit is null"),
+                "");
     }
 
     /**
@@ -125,6 +125,7 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
+    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -179,7 +179,7 @@ public class DataCenterPlanOptimizer
             List<RowExpression> pushable = new ArrayList<>();
             List<RowExpression> nonPushable = new ArrayList<>();
 
-            for (RowExpression conjunct : logicalRowExpressions.extractConjuncts(node.getPredicate())) {
+            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
                 try {
                     conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
                     pushable.add(conjunct);

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1392,24 +1392,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner queryRunner = null;
+        DistributedQueryRunner distributedQueryRunner = null;
         try {
-            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            queryRunner.installPlugin(new DataCenterPlugin());
-            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            queryRunner.installPlugin(new TpchPlugin());
-            queryRunner.createCatalog("tpch", "tpch", properties);
+            distributedQueryRunner.installPlugin(new DataCenterPlugin());
+            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            distributedQueryRunner.installPlugin(new TpchPlugin());
+            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
 
-            return queryRunner;
+            return distributedQueryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, queryRunner);
+            closeAllSuppress(e, distributedQueryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(config);
+        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
+openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,15 +24,17 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-Other audit logging properties include: 
+The following is a detailed description of audit logging properties:
 
-`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
 
-`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
+`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
+`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
 
 Example configuration file:
 
@@ -41,7 +43,6 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
-hetu.event.listener.audit.filecount=1
-hetu.event.listener.audit.limit=100000
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```

hetu-docs/en/admin/extension-execution-planner.md

@@ -0,0 +1,25 @@
+#Extension Physical Execution Planner
+This section describes how to add an extension physical execution planner in openLooKeng. With the extension physical execution planner, openLooKeng can utilize other operator acceleration libraries to speed up the execution of SQL statements.
+
+##Configuration
+To enable extension physical execution feature, the following configs must be added in
+`config.properties`：
+
+``` properties
+extension_execution_planner_enabled=true
+extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
+extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
+```
+
+The above attributes are described below:
+
+- `extension_execution_planner_enabled`: Enable extension physical execution feature.
+- `extension_execution_planner_jar_path`: Set the file path of the extension physical execution jar package.
+- `extension_execution_planner_class_path`: Set the package path of extension physical execution generated class in jar。
+
+
+##Usage
+The below command can control the enablement of extension physical execution feature in WebUI or Cli while running openLooKeng:
+```
+set session extension_execution_planner_enabled=true/false
+```

hetu-docs/en/admin/properties.md

@@ -118,6 +118,20 @@ This section describes the most important config properties that may be used to
 >
 > This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
 
+### `query.suspend-query-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables running query temporary suspension when system is in low resource situation.
+
+### `query.max-suspended-queries`
+
+> -   **Type:** `integer`
+> -   **Default value:** `10`
+>
+> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
+
 ## Spilling Properties
 
 ### `experimental.spill-enabled`
@@ -161,6 +175,30 @@ This section describes the most important config properties that may be used to
 >
 > This config property can be overridden by the `spill_window_operator` session property.
 
+
+### `experimental.spill-build-for-outer-join-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables spill feature for right-outer and full-outer join operations.
+>
+>
+>
+> This config property can be overridden by the `spill_build_for_outer_join_enabled` session property.
+
+### `experimental.inner-join-spill-filter-enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables bloom filter based build-side spill matching for probe side spill decision.
+>
+>
+>
+> This config property can be overridden by the `inner_join_spill_filter_enabled` session property.
+
+
 ### `experimental.spill-reuse-tablescan`
 
 > -   **Type:** `boolean`
@@ -179,7 +217,7 @@ This section describes the most important config properties that may be used to
 >
 > Directory where spilled content will be written. It can be a comma separated list to spill simultaneously to multiple directories, which helps to utilize multiple drives installed in the system.
 >
->
+> When `experimental.spiller-spill-to-hdfs` is to `true`, `experimental.spiller-spill-path` must contain only  a single directory.
 >
 > It is not recommended to spill to system drives. Most importantly, do not spill to the drive on which the JVM logs are written, as disk overutilization might cause JVM to pause for lengthy periods, causing queries to fail.
 
@@ -254,12 +292,21 @@ This section describes the most important config properties that may be used to
 >
 > Sets number of pages prefetched while reading from spilled files.
 
+
+### `experimental.spill-use-kryo-serialization`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables Kryo based serialization for spill to disk, instead of default java serializer.
+
+
 ### `experimental.revocable-memory-selection-threshold`
 
 > -   **Type:** `data size`
 > -   **Default value:** `512 MB`
 >
-> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke. 
+> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke.
 
 ### `experimental.prioritize-larger-spilts-memory-revoke`
 
@@ -268,6 +315,34 @@ This section describes the most important config properties that may be used to
 >
 > Enables to prioritize splits with larger revocable memory.
 
+### `experimental.spill-non-blocking-orderby`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables order by operator to use asynchronous mechanism to spill, i.e it can accumulate input even when a spill is in progress and initiate a secondary spill when the secondary data accumulate exceeds a threshold or when the primary spill is completed, the default value of the threshold is the minimum between 20MB and 5% of available free memory. This property must be used in conjunction with the `experimental.spill-enabled` property.
+>
+>
+>
+> This config property can be overridden by the `spill_non_blocking_orderby` session property.
+
+### `experimental.spiller-spill-to-hdfs`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables spilling into HDFS. When this property is set to `true` the property `experimental.spiller-spill-profile` must be set and also `experimental.spiller-spill-path` must contain only a single path.
+
+### `experimental.spiller-spill-profile`
+
+> -   **Type:** `string`
+> -   **No default value.** Must be set when spilling to hdfs is enabled
+>
+>
+> This property defines the [filesystem](../develop/filesystem.md) profile used to spill. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `experimental.spiller-spill-profile=spill-hdfs`, a profile describing this filesystem `spill-hdfs.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable, refer [filesystem](../develop/filesystem.md) for details).
+>
+> This property is required when `experimental.spiller-spill-to-hdfs` is set to `true`. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the path declared in `experimental.spiller-spill-path` folder in the specified file system.
+
 ## Exchange Properties
 
 Exchanges transfer data between openLooKeng nodes for different stages of a query. Adjusting these properties may help to resolve inter-node communication issues or improve network utilization.
@@ -303,19 +378,8 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
 >
->
->
 > Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
 
-### `exchange.max-error-duration`
-
-> -   **Type:** `duration`
-> -   **Minimum value:** `1m`
-> -   **Default value:** `7m`
->
-> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a failure.
-
-
 ### `sink.max-buffer-size`
 
 > -   **Type:** `data size`
@@ -323,6 +387,113 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
 
+## Failure Recovery handling Properties
+
+### Failure Retry Policies
+
+### `failure.recovery.retry.profile`
+
+> -   **Type:** `String`
+> -   **Default value:** `default`
+>
+> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
+> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
+> The file `test.properties` must contain `failure.recovery.retry.type` specified.
+
+
+### `failure.recovery.retry.type`
+
+> -   **Type:** `String`
+> -   **Default value:** `timeout`
+>
+> The failure detection mechanism in use. Default is timeout based failure detection.
+>
+#### `timeout` based failure detection. 
+> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
+> Additional properties `max.error.duration` can be defined for this type of failure detection.
+>
+#### `max-retry` based failure detection. 
+> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
+> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection. 
+> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
+
+### `max.error.duration`
+> -   **Type:** `duration`
+> -   **Default value:** `300s`
+>
+> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
+
+
+### `max.retry.count`
+
+> -   **Type:** `integer`
+> -   **Default value:** `100`
+>
+> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. 
+> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster. 
+> This property is used only for `max-retry` based failure detection profiles. 
+> The minimum value for this parameter is 100.
+
+### Gossip Protocol Configurations for Failure Detection
+
+### `failure-detection-protocol`
+
+>- **Type:** String
+>- **Default value:** `heartbeat`
+>
+> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
+> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
+> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
+
+### `failure-detector.heartbeat-interval`
+
+>- **Type:** Duration
+>- **Default value:** `500ms` (500 miliseconds)
+>
+> This is the interval of gossip between two nodes in the cluster.
+> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
+> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
+> In workers, this property can be left to use the default value.
+>
+### `failure-detector.worker-gossip-probe-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `5s` (5 seconds)
+>
+> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
+> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
+> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
+> This parameter should have higher value than `failure-detector.heartbeat-interval`.
+>
+### `failure-detector.coordinator-gossip-probe-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `5s` (5 seconds)
+>
+> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
+> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
+> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
+> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
+>
+### `failure-detector.coordinator-gossip-collate-interval`
+>
+> - **Type:** Duration
+>- **Default value:** `2s` (2 seconds)
+>
+> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
+> This property has to be specified only for the coordinator.
+> This parameter should have higher value than `failure-detector.heartbeat-interval`.
+>
+### `failure-detector.gossip-group-size`
+>
+> - **Type:** Integer
+>- **Default value:** `Integer.MAX_VALUE`
+>
+> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
+> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
+> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
+> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
+
 ## Task Properties
 
 ### `task.concurrency`
@@ -707,7 +878,7 @@ helps with cache affinity scheduling.
 > Auto-Vacuum enables the system to automatically manage vacuum jobs by constantly monitoring the tables which needs vacuum in order to maintain optimal performance.
 > Engine gets the tables from data sources that are eligible for vacuum and trigger vacuum operation for those tables.
 
-### `auto-vacuum.enabled:`
+### `auto-vacuum.enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
@@ -777,15 +948,25 @@ helps with cache affinity scheduling.
 > -   **Default value:** `5m`
 >
 > The maximum time coordinator waits for remote-task related error to be resolved before it's considered a failure.
+>
+> Note:
+> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
 
-## Distributed Snapshot
+## Query Recovery
+
+### `recovery_enabled`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
 
 ### `snapshot_enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
 >
-> This session property is used to enable or disable the distributed snapshot functionality.
+> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
 
 ### `hetu.experimental.snapshot.profile`
 
@@ -797,23 +978,39 @@ helps with cache affinity scheduling.
 >
 > This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
 
-### `hetu.snapshot.maxRetries`
+### `hetu.recovery.maxRetries`
 
 > -   **Type:** `int`
 > -   **Default value:** `10`
 >
 > This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
 >
-> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
+> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
 
-### `hetu.snapshot.retryTimeout`
+### `hetu.recovery.retryTimeout`
 
 > -   **Type:** `duration`
 > -   **Default value:** `10m` (10 minutes)
 >
 > This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
 >
-> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.
+> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
+
+### `hetu.snapshot.useKryoSerialization`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables Kryo based serialization for snapshot, instead of default java serializer.
+
+### `experimental.eliminate-duplicate-spill-files`
+
+> -   **Type:** `boolean`
+> -   **Default value:** `false`
+>
+> Enables elimination of duplicate spill files storage as part of snapshot capture.
+
+
 
 ## HTTP Client Configurations
 
@@ -836,3 +1033,12 @@ helps with cache affinity scheduling.
 > After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
 >
 > (Note: this parameter should be configured with higher time when in high load environment)
+
+## Connector Properties configuration
+
+### `case-insensitive-name-matching`
+>
+> -    **Type:** `boolean`
+> -    **Default value:** `false`
+>
+> Case-insensitive matching between database and collection names. The default is case sensitive.

hetu-docs/en/admin/reliable-execution.md

@@ -11,9 +11,9 @@ To achieve better performance while maintaining execution reliability, the *dist
 
 As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
 
-## Enable Distributed Snapshot
+## Enable Recovery framework
 
-Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
+Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
 
 ## Requirements
 
@@ -37,7 +37,7 @@ When a query that does not meet the above requirements is submitted with distrib
 
 ## Detection
 
-Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration) configuration.
+Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
 
 ## Storage Considerations
 
@@ -55,8 +55,20 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
 
 The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
 
-It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
+It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
+
+## Snapshot statistics
+
+Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
+
+Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query.  These statistics are displayed for all snapshots and for last snapshot separately.
+
+Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots.  Restore statistics are displayed only when there is restore(recovery) happened during the query.
+
+Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details 
+
+![](../images/snapshot_statistics.png)
 
 ## Configurations
 
-Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).
+Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).

hetu-docs/en/admin/spill.md

@@ -34,6 +34,10 @@ saturation of the configured spill paths.
 
 openLooKeng treats spill paths as independent disks (see [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD)), so there is no need to use RAID for spill.
 
+
+## Spill To HDFS
+Spilling directly into HDFS is also possible for that `experimental.spiller-spill-to-hdfs` needs to be set to `true`, `experimental.spiller-spill-profile` needs to be set and `spiller-spill-path` must contain only a single directory when we intend to spill into HDFS. (refer `experimental.spiller-spill-to-hdfs` and  `experimental.spiller-spill-profile` properties for more details )
+
 ## Spill Compression
 
 
@@ -61,6 +65,8 @@ When the build table is partitioned, the spill-to-disk mechanism can decrease th
 
 With this mechanism, the peak memory used by the join operator can be decreased to the size of the largest build table partition. Assuming no data skew, this will be `1 / task.concurrency` times the size of the whole build table.
 
+Note: spill-to-disk is not supported for Cross Join.
+
 ### Aggregations
 
 Aggregation functions perform an operation on a group of values and return one value. If the number of groups you\'re aggregating over is large, a significant amount of memory may be needed. When spill-to-disk
@@ -69,6 +75,7 @@ is enabled, if there is not enough memory, intermediate accumulated aggregation
 ### Order By
 
 If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
+Generally when a spill is in progress the operator is blocked from taking inputs, but when `experimental.spill-non-blocking-orderby` is set to `true` order by uses asynchronous mechanism to spill (see`experimental.spill-non-blocking-orderby`).
 
 ### Window functions
 

hetu-docs/en/admin/web-interface.md

@@ -33,4 +33,54 @@ and statistics about the query is available by clicking the *JSON* link. These v
 > -   **Allowed values:** `true`, `false`
 > -   **Default value:** `false`
 >
-> Insecure authentication over HTTP is disabled by default. This could be overridden via "hetu.queryeditor-ui.allow-insecure-over-http" property of "etc/config.properties" (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
+> Insecure authentication over HTTP is disabled by default. This could be overridden via `hetu.queryeditor-ui.allow-insecure-over-http` property of `etc/config.properties` (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
+
+### `hetu.queryeditor-ui.execution-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `100 DAYS`
+>
+> UI Execution timeout is set to 100 days as default. This could be overridden via `hetu.queryeditor-ui.execution-timeout` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.max-result-count`
+
+> - **Type:** `int`
+> - **Default value:** `1000`
+>
+> UI max result count is set to 1000 as default. This could be overridden via `hetu.queryeditor-ui.max-result-count` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.max-result-size-mb`
+
+>- **Type:** `size`
+>- **Default value:** `1GB`
+>
+> UI max result size is set to 1 GB as default. This could be overridden via `hetu.queryeditor-ui.max-result-size-mb` of `etc/config.properties`
+
+### `hetu.queryeditor-ui.session-timeout`
+
+> -   **Type:** `duration`
+> -   **Default value:** `1 DAYS`
+>
+> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
+
+### `hetu.queryhistory.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `1000`
+>
+> The maximum number of query history stored by openLooKeng. This could be overridden via "hetu.queryhistory.max-count" of "etc/config.properties".
+
+### `hetu.collectionsql.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `100`
+>
+> The Maximum number of SQL collected by each user. This could be overridden via "hetu.collectionsql.max-count" of "etc/config.properties".
+
+## Remarks
+
+The max length of the favorite SQL is 600 by default. You can modify it through the following steps:
+
+1. Login MySQL database according to the JDBC configuration of `hetu-metastore.properties`
+
+2. Select table hetu_favorite, execute script `alter table hetu_favorite modify query varchar(2000) not null;` to modify the max length of the favorite SQL.

hetu-docs/en/connector/Hudi.md

@@ -1,5 +1,8 @@
 # Hudi  Connector
 
+### Release Notes
+Currently Hudi only supports version 0.7.0.
+
 ### Hudi Introduction
 
 Apache Hudi is a fast growing data lake storage system that helps organizations build and manage petabyte-scale data lakes. Hudi enables storing vast amounts of data on top of existing DFS compatible storage while also enabling stream processing in addition to typical batch-processing. This is made possible by providing two new primitives. Specifically,

hetu-docs/en/connector/memory.md

@@ -179,11 +179,12 @@ Use these properties when creating a table with the Memory Connector to make que
 
 Index Types
 --------------
-These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`. If a query operator is not supported by a particular index, you can still use that operator, but the query will not benefit from the index.
+These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`.
+If a query operator is not supported by a particular index, you can still use that operator, but the query will not benefit from the index.
 
 | Index ID                          |Built for Columns In | Supported query operators             |
 |-----------------------------------|----------------------------------------|---------------------------------------|
-| Bloom   | `sorted_by,index_columns`                                 | `=` `IN`                             |                   
+| Bloom   | `index_columns`                                 | `=` `IN`                             |                   
 | MinMax  | `sorted_by,index_columns`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 | Sparse  | `sorted_by`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 

hetu-docs/en/connector/opengauss.md

@@ -45,6 +45,95 @@ Finally, you can access the `hetutb` table in the `public` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `opengauss` in the above examples.
 
+## openGauss Update/Delete Support
+
+### Create openGauss Table
+
+Example：
+
+```sql
+CREATE TABLE opengauss_table (
+    id int,
+    name varchar(255));
+```
+
+### INSERT on openGauss tables
+
+Example：
+
+```sql
+INSERT INTO opengauss_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### UPDATE on openGauss tables
+
+Example：
+
+```sql
+UPDATE opengauss_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
+
+SELECT result before UPDATE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+SELECT result after UPDATE
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+id | name
+----+------
+2 | Bob
+1 | Tim
+(2 rows)
+```
+
+### DELETE on openGauss tables
+
+Example：
+
+```sql
+DELETE FROM opengauss_table
+  WHERE id=2;
+```
+
+Above example delete the rows with column `id` having value `2`.
+
+SELECT result before DELETE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+SELECT result after DELETE:
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 ****Note:****
 
 > - When the compatibility type of the openGuass database is O (DBCOMPATIBILITY = A), the `Date` data type is not supported.
@@ -64,4 +153,4 @@ openGauss Connector Limitations
 
 The following SQL statements are not yet supported:
 
-[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/postgresql.md

@@ -46,9 +46,98 @@ Finally, you can access the `clicks` table in the `web` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `postgresql` in the above examples.
 
+## PostgreSQL Update/Delete Support
+
+### Create PostgreSQL Table
+
+Example：
+
+```sql
+CREATE TABLE postgresql_table (
+    id int,
+    name varchar(255));
+```
+
+### INSERT on PostgreSQL tables
+
+Example：
+
+```sql
+INSERT INTO postgresql_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### UPDATE on PostgreSQL tables
+
+Example：
+
+```sql
+UPDATE postgresql_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
+
+SELECT result before UPDATE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+SELECT result after UPDATE
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+id | name
+----+------
+2 | Bob
+1 | Tim
+(2 rows)
+```
+
+### DELETE on PostgreSQL tables
+
+Example：
+
+```sql
+DELETE FROM postgresql_table
+  WHERE id=2;
+```
+
+Above example delete the rows with column `id` having value `2`.
+
+SELECT result before DELETE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+SELECT result after DELETE:
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 PostgreSQL Connector Limitations
 --------------------------------
 
 The following SQL statements are not yet supported:
 
-[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/redis.md

@@ -0,0 +1,173 @@
+Redis Connector
+====================
+Overview
+--------
+this connector allows the use of Redis key/value pair is presented as a single row in openLooKeng.
+
+**Note**
+
+*In Redis,key/value pair can only be mapped to string or hash value types.keys can be stored in a zset,then keys can split into multiple slice*
+
+*Support Redis 2.8.0 or higher*
+
+Configuration
+-------------
+To configure the Redis connector, create a catalog properties file `etc/catalog/redis.properties` with the following contents, replacing the properties as appropriate:
+``` properties
+connector.name=redis
+redis.table-names=schema1.table1,schema1.table2
+redis.nodes=host1:port
+```
+### Multiple Redis Servers
+You can have as many catalogs as you need. If you have additional
+Redis servers, simply add another properties file to ``etc/catalog``
+with a different name, making sure it ends in ``.properties``.
+For example, if you name the property file `sales.properties`, openLooKeng will create a catalog named `sales` using the configured connector.
+
+Configuration properties
+------------------------
+The following configuration properties are available:
+
+| Property Name                                                    | Description                                                                                                           |
+|:-----------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
+| `redis.table-names`                                              | List of all tables provided by the catalog                                                                            |
+| `redis.default-schema`                                           | Default schema name for tables  (default `default`)                                                                   |
+| `redis.nodes`                                                    | List of nodes in the Redis server                                                                                     |
+| `redis.connect-timeout`                                          | Timeout for connecting to the  Redis server (ms) (default 2000)                                                       |
+| `redis.scan-count`                                               | The number of keys obtained from each scan for string and hash value types (default 100)                              |
+| `redis.key-prefix-schema-table`                                  | Redis keys have schema-name:table-name prefix   (default  false)                                                      |
+| `redis.key-delimiter`                                            | Delimiter separating schema_name and table_name if redis.key-prefix-schema-table is used (default `:`)                |
+| `redis.table-description-dir`                                    | Directory containing table description files (default `etc/redis/`)                                                   |
+| `redis.hide-internal-columns`                                    | Whether internal columns are shown in table metadata or not. (default true)                                           |
+| `redis.database-index`                                           | Redis database index  (default 0)                                                                                     |
+| `redis.password`                                                 | Redis server password  (default null)                                                                                 |
+| `redis.table-description-interval`                               | the interval of flush description files (ms) (default no flush,table description will be memoized without expiration) |
+
+
+Internal columns
+----------------
+
+| Column name        | Type    | Description                                                                                                                              |
+|:-------------------| :------ |:-----------------------------------------------------------------------------------------------------------------------------------------|
+| `_key`             | VARCHAR  | Redis key.                                                                                                                               |
+| `_value`           | VARCHAR   | Redis value corresponding to the key                                                                                                     |
+| `_key_length`   | BIGINT  | Number of bytes in the key.                                                                                                              |
+| `_key_corrupt`     | BOOLEAN  | True if the decoder could not decode the key for this row. When true, data columns mapped from the key should be treated as invalid.     |
+| `_value_corrupt`   | BOOLEAN | True if the decoder could not decode the value for this row. When true, data columns mapped from the value should be treated as invalid. |
+
+
+Table Definition Files
+----------------------
+For openLooKeng, every key/value pair  must be mapped into columns to allow queries against the data. It is like kafka conntector,so you can refer to kafka-tutorial
+
+A table definition file consists of a JSON definition for a table. The name of the file can be arbitrary but must end in `.json`.
+
+for example,there is a nation.json
+``` json
+{
+    "tableName": "nation",
+    "schemaName": "tpch",
+    "key": {
+        "dataFormat": "raw",
+        "fields": [
+            {
+                "name": "redis_key",
+                "type": "VARCHAR(64)",
+                "hidden": "true"
+            }
+        ]
+    },
+    "value": {
+        "dataFormat": "json",
+        "fields": [
+            {
+                "name": "nationkey",
+                "mapping": "nationkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "name",
+                "mapping": "name",
+                "type": "VARCHAR(25)"
+            },
+                        {
+                "name": "regionkey",
+                "mapping": "regionkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "comment",
+                "mapping": "comment",
+                "type": "VARCHAR(152)"
+            }
+       ]
+    }
+}
+```
+In redis,such data exists
+```shell
+127.0.0.1:6379> keys tpch:nation:*
+ 1) "tpch:nation:2"
+ 2) "tpch:nation:4"
+ 3) "tpch:nation:16"
+ 4) "tpch:nation:18"
+ 5) "tpch:nation:10"
+ 6) "tpch:nation:17"
+ 7) "tpch:nation:1"
+```
+```shell
+127.0.0.1:6379> get tpch:nation:1
+"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
+```
+Now we can use redis connector get data from redis,(redis_key don't show,because we set "hidden": "true" ) 
+```shell
+lk> select * from redis.tpch.nation;
+ nationkey |      name      | regionkey |                                                      comment                                                       
+-----------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
+         3 | CANADA         |         1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold              
+         9 | INDONESIA      |         2 |  slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 
+        19 | ROMANIA        |         3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account    
+         2 | BRAZIL         |         1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special   
+```
+**Note**
+
+*if redis.key-prefix-schema-table is false (default is false),all keys in redis will be mapped to table's key,no matching occurs*
+
+Please refer to the `kafka-tutorial` for the description of the ``dataFormat`` as well as various available decoders.
+
+In addition to the above Kafka types, the Redis connector supports ``hash`` type for the ``value`` field which represent data stored in the Redis hash.
+Redis connector use `hgetall key` to get data.
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "value": {
+        "dataFormat": "hash",
+        "fields": [
+          ...
+        ]
+      }
+    }
+```
+
+the Redis connector supports ``zset`` type for the ``key`` field which represent key stored in the Redis zset. 
+if and only if ``zset`` is used as key datafomart,the split is truly supported , because we can use `zrange zsetkey split.start split.end` to get keys of a split.
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "key": {
+        "dataFormat": "zset",
+        "name": "zsetkey", //zadd zsetkey score member
+        "fields": [
+            ...
+        ]
+      }
+    }
+```
+
+Redis Connector Limitations
+---------------------------
+only support read operation,don't support write operation.
+
+

hetu-docs/en/develop/_index.md

@@ -1,5 +1,5 @@
 
 # Developer Guide
 
-openLooKeng is based on Trino(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
+openLooKeng is based on Trino 316(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
 

hetu-docs/en/develop/connectors.md

@@ -21,7 +21,7 @@ This interface is too big to list in this documentation, but if you are interest
 connector. If your underlying data source supports schemas, tables and columns, this interface should be straightforward to implement. If you are attempting to adapt something that is not a relational database (as
 the Example HTTP connector does), you may need to get creative about how you map your data source to openLooKeng\'s schema, table, and column concepts.
 
-### ConnectorSplitManger
+### ConnectorSplitManager
 
 The split manager partitions the data for a table into the individual chunks that openLooKeng will distribute to workers for processing. For example, the Hive connector lists the files for each Hive partition and creates
 one or more split per file. For data sources that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.

hetu-docs/en/faq/faq.md

@@ -40,6 +40,10 @@
    >
    > openLooKeng Bilibili channel:  https://space.bilibili.com/627629884   
 
+8. Which version of Trino is openLooKeng developed on?
+
+   > Based on Trino 316 version development.
+
 ## Functions
 
 1. What connectors does the openLooKeng support?

hetu-docs/en/index.md

@@ -46,6 +46,7 @@ headless: true
     - [Audit Log]({{< relref "./docs/admin/audit-log.md" >}})
     - [Reliable Execution]({{< relref "./docs/admin/reliable-execution.md" >}})
     - [JDBC Data Source Multi-Split Management]({{< relref "./docs/admin/multi-split-for-jdbc-data-source.md" >}})
+    - [Extension Physical Execution Planner]({{< relref "./docs/admin/extension-execution-planner.md" >}})
 
 - [Query Optimizer]("#")
     - [Table Statistics]({{< relref "./docs/optimizer/statistics.md" >}})
@@ -83,6 +84,7 @@ headless: true
     - [JMX]({{< relref "./docs/connector/jmx.md" >}})
     - [Kafka]({{< relref "./docs/connector/kafka.md" >}})
     - [Kafka Connector Tutorial]({{< relref "./docs/connector/kafka-tutorial.md" >}})
+    - [Redis] ({{< relref "./docs/connector/redis.md" >}})
     - [Local File]({{< relref "./docs/connector/localfile.md" >}})
     - [Memory]({{< relref "./docs/connector/memory.md" >}})
     - [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
@@ -172,6 +174,7 @@ headless: true
     - [SHOW CACHE]({{< relref "./docs/sql/show-cache.md" >}})
     - [SHOW CATALOGS]({{< relref "./docs/sql/show-catalogs.md" >}})
     - [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
+    - [SHOW CREATE CUBE]({{< relref "./docs/sql/show-create-cube.md" >}})
     - [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
     - [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
     - [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
@@ -216,6 +219,8 @@ headless: true
     - [Task Resource]({{< relref "./docs/rest/task.md" >}})
 
 - [Release Notes]("#")
+    - [1.6.1 (27 Apr 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.1.md" >}})
+    - [1.6.0 (30 Mar 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.0.md" >}})
     - [1.5.0 (30 Dec 2021)]({{< relref "./docs/releasenotes/releasenotes-1.5.0.md" >}})
     - [1.4.1 (12 Nov 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.1.md" >}})
     - [1.4.0 (15 Oct 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.0.md" >}})

hetu-docs/en/preagg/overview.md

@@ -62,8 +62,6 @@ Tables from following Connectors can be used as source to build a StarTree Cube.
 
    2.1. Overcome the limitation of Creating Cube for larger dataset.
 
-   2.2. Update Cube if source table has been updated.
-
 ## Enabling and Disabling StarTree Cube
 To enable:
 ```sql 
@@ -122,6 +120,17 @@ SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >=
 Since the data inserted into the Cube was for `nationkey >= 5`, only queries matching this condition will utilize the Cube.
 Queries not matching the condition would continue to work but won't use the Cube.
 
+If the source table of a Cube gets updated, the corresponding Cube gets expired automatically. In order to overcome
+this issue, we have added support in openLooKeng CLI by introducing **RELOAD CUBE** command. The user will have the
+ability to manually reload a cube if the status of the Cube becomes INACTIVE or EXPIRED. The syntax to reload the
+Cube nation_cube is as follows,
+
+```sql 
+RELOAD CUBE nation_cube
+```
+Please note that this feature is only supported via the CLI. During this reload process if an unexpected error occurs, the user will get to see the original SQL statement
+to recreate the cube manually.
+
 ## Building Cube for Large Dataset
 One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
 Processing large number of rows requires more memory than cluster is configured with. This results in query failing with message **Query exceeded per-node user memory
@@ -180,38 +189,55 @@ SHOW CUBES;
 ```
 
 **Note:**
-1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together. 
+1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together.
    All continuous predicates will be merged into a single range predicate and remaining predicates are untouched.
 
-   Only the following types are supported and can be merged together. 
-   `Integer, TinyInt, SmallInt, BigInt, Date`
-    
-   For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is 
-   possibility that particular Cube may not be used during query optimization even if the Cube has all the required data. For example,
+   Only the following types are supported and can be merged together.
+   `Integer, TinyInt, SmallInt, BigInt, Date, String`
+
+   For String data type, predicate merge logic functionally works only if the Strings are ending with a digit and all are of same length. 
+   For example,
 
 ```sql
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
 ```
-   Here these two predicates cannot be merged into store_id BETWEEN 'A01' AND 'A20'; So the Cube won't be used 
-   for queries that are spanning over two the predicates;
+After the insertion, the two predicates will be merged into `'A01' AND 'A20'`
 
 ```sql
-   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube won't be used for optimizing this query. This is a limitation as of now.
+   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube would be used for this query.
 ```
-   Because of the predicate rewrite some of the following queries can't be supported
+
+   Consider the following example where `store_id` values are not of same length.
+
+```sql
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A1';
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A2' 
+```   
+   store_id predicate will be rewritten as `store_id >= 'A1' and store < 'A3'` as per the varchar predicate merge logic;
+
+```sql   
+   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A10' 
+```
+   The above query would fail because `A10` is subset of the range `store_id >= 'A1' and store < 'A3'`. So Users should be wary of this issue. 
+
+For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is
+possibility that particular Cube may not be used during query optimization even if the Cube has all the required data.
+
+2. Predicate rewrite has some limitations as well. Consider the following query
 
 ```sql   
    INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
 ```
-   The predicate is rewritten as ss_sold_date_sk >= 2451912 to be prepare for merging continous predicates. 
-   Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to 
-   optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
+The predicate is rewritten as ss_sold_date_sk >= 2451912 to support merging continuous predicates.
+Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to
+optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
 
 ```sql   
    SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
 ```   
-3. Only single column predicates can be merged. 
+3. Only single column predicates can be merged.
+
 
 ## Open issues and Limitations
 1. StarTree Cube is only effective when the group by cardinality is considerably fewer than the number of rows in source table.
@@ -223,7 +249,8 @@ SHOW CUBES;
 5. OpenLooKeng CLI has been modified to ease the process of creating Cubes for larger datasets. But still there are limitations with this implementation
    as the process involves merging multiple Cube predicates into one. Only Cube predicates defined on Integer, Long and Date types can be merged properly. Support for Char, 
    String types still need to be implemented.
-   
+6. Varchar predicates can be merged only if the values are of same length.  
+
 ## Performance Optimizations on Star Tree
 1. Star Tree Query re-write optimization for same group by columns: If the group by columns of the cube and query matches, the query is 
 re-written internally to select the pre-aggregated data. If the group by columns does not matches, the additional aggregations are 

hetu-docs/en/preagg/statements.md

@@ -150,7 +150,25 @@ Show Cubes for `orders` table:
 ```sql
     SHOW CUBES FOR orders;
 ```
+## RELOAD CUBE
 
+### Synopsis
+
+``` sql
+RELOAD CUBE cube_name
+```
+
+### Description
+Reloads the Cube if the source table has been updated.
+
+### Examples
+
+If the source table `orders` of the cube `orders_cube` gets updated then the status of the cube `orders_cube` 
+gets EXPIRED. Use the command `RELOAD CUBE cube_name` to overcome this issue as follows:
+
+```sql
+    RELOAD CUBE orders_cube
+```
 ## DROP CUBE
 
 ### Synopsis

hetu-docs/en/releasenotes/releasenotes-1.6.0.md

@@ -0,0 +1,25 @@
+# Release 1.6.0
+
+## Key Features
+
+| Area                  | Feature                                                      |
+| --------------------- | ------------------------------------------------------------ |
+| Star Tree             | Support update cube command to allow admin to easily update an existing cube when the underlying data changes |
+| Bloom Index           | Hindex-Optimize Bloom Index Size-Reduce bloom index size by 10X+ times |
+| Task Recovery         | 1. Improve failure detection time: It need take 300s to determine a task is failed and resume after that. Improving this would improve the resume & also the overall query time<br/>2. snapshotting speed & size: When sql execute takes a snapshot, now use direct Java serialization which is slow and also takes more size. Using kryo serialization would reduce size and also increase speed there by increasing the overall throughput |
+| Spill to Disk         | 1. Spill to disk speed & size improvement: When spill happens during HashAggregation & GroupBy, the data serialized to disk is slow and also size is more. It can improve the overall performance by reducing size and also improving  the writing speed. Using kryo serialization improves both speed and reduces size<br/>2. Support spilling to hdfs: Currently data can spill to multiple disks, now support spill to hdfs to improve throughput<br/>3. Async spill/unspill: When revocable memory crosses threshold and spill is triggered, it blocks accepting the data from the downstream operators. Accepting this and adding to the existing spill would help to complete the pipeline faster<br/>4. Enable spill for right outer & full join for spilling: It don’t spill the build side data when the join type is right outer or full join as it needs the entire data in memory for lookup. This leads to out of  memory when the data size is more. Instead by enable spill and create a Bloom Filter to identify the data spilled and use it during join with probe side |
+| Connector Enhancement | Support data update and delete operator for PostgreSQL and openGauss |
+
+## Known Issues
+
+| Category      | Description                                                  | Gitee issue                                               |
+| ------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
+| Task Recovery | When a snapshot is enabled and a CTAS with transaction is executed, an error is reported in the SQL statement. | [I502KF](https://e.gitee.com/open_lookeng/issues/list?issue=I502KF) |
+|               | An error occurs occasionally when snapshot is enabled and exchange.is-timeout-failure-detection-enabled is disabled. | [I4Y3TQ](https://e.gitee.com/open_lookeng/issues/list?issue=I4Y3TQ) |
+| Star Tree     | In the memory connector, after the star tree is enabled, data inconsistency occurs during query. | [I4QQUB](https://e.gitee.com/open_lookeng/issues/list?issue=I4QQUB) |
+|               | When the reload cube command is executed for 10 different cubes at the same time, some cubes fail to be reloaded. | [I4VSVJ](https://e.gitee.com/open_lookeng/issues/list?issue=I4VSVJ) |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en)
+

hetu-docs/en/releasenotes/releasenotes-1.6.1.md

@@ -0,0 +1,15 @@
+# Release 1.6.1 (27 Apr 2022)
+
+## Key Features
+
+This release is mainly about modification and enhancement of some SPIs, which are used in more scenarios.
+
+| Area                    | Feature                                                      | PR #s                                                        |
+| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| Data Source statistics                | The method of obtaining statistics is added so that statistics can be directly obtained from the Connector. Some operators can be pushed down to the Connector for calculation. You may need to obtain statistics from the Connector to display the amount of processed data.                                               | 1450                                                          |
+| Operator processing extension | Users can customize the physical execution plan of worker nodes. Users can use their own operator pipelines to replace the native implementation to accelerate operator processing. | 1436                                                           |
+| HIVE UDF extension | Adds the adaptation of HIVE UDF function namespace to support the execution of UDFs (including GenericUDF) written based on the HIVE UDF framework. | 1456                                                           |
+
+## Obtaining the Document 
+
+For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en)

hetu-docs/en/security/built-in-system-access-control.md

@@ -1,4 +1,3 @@
-
 Built-in System Access Control
 ==============================
 
@@ -57,7 +56,10 @@ composed of the following fields:
 
 -   `user` (optional): regex to match against user name. Defaults to `.*`.
 -   `catalog` (optional): regex to match against catalog name. Defaults to `.*`.
--   `allow` (required): boolean indicating whether a user has access to the catalog
+-   ``allow`` (required): string indicating whether a user has access to the catalog.
+    This value can be ``all``, ``read-only`` or ``none``, and defaults to ``none``.
+    Setting this value to ``read-only`` has the same behavior as the ``read-only``
+    system access control plugin.
 
 
 **Note**
@@ -65,7 +67,9 @@ composed of the following fields:
 *By default, all users have access to the `system` catalog. You can override this behavior by adding a rule.*
 
 
-For example, if you want to allow only the user `admin` to access the `mysql` and the `system` catalog, allow all users to access the `hive` catalog, and deny all other access, you can use the following rules:
+For example, if you want to allow only the user ``admin`` to access the``mysql`` and the ``system`` catalog, 
+allow all users to access the ``hive`` catalog, allow the user ``alice`` read-only access to the ``postgresql`` 
+catalog, and deny all other access, you can use the following rules:
 
 ``` json
 {
@@ -73,15 +77,20 @@ For example, if you want to allow only the user `admin` to access the `mysql` an
     {
       "user": "admin",
       "catalog": "(mysql|system)",
-      "allow": true
+      "allow": all
     },
     {
       "catalog": "hive",
-      "allow": true
+      "allow": all
+    },
+    {
+      "user": "alice",
+      "catalog": "postgresql",
+      "allow": "read-only"
     },
     {
       "catalog": "system",
-      "allow": false
+      "allow": none
     }
   ]
 }
@@ -195,6 +204,7 @@ If you want to allow users to use the  extactly same name as their Kerberos prin
 ```
 
 ### Node State Rules
+
 These rules govern the node state info particular users can access. The user is granted access to update a node state based on the first matching rule read from top to bottom. If no rule matches, access is denied. Each rule is
 composed of the following fields:
 

hetu-docs/en/security/ranger-access-control.md

@@ -7,7 +7,7 @@ Overview
 
 Apache Ranger delivers a comprehensive approach to security for a Hadoop cluster. It provides a centralized platform to define, administer and manage security policies consistently across Hadoop components. Check [Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index) for detail introduction and user guide.
 
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
+[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is developed based on Ranger 2.1.0, which is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
 
 Build Process
 -------------------------

hetu-docs/en/sql/show-create-cube.md

@@ -0,0 +1,32 @@
+
+SHOW CREATE CUBE
+=================
+
+Synopsis
+--------
+
+``` sql
+SHOW CREATE CUBE cube_name
+```
+
+Description
+-----------
+
+Show the SQL statement that creates the specified cube.
+
+Examples
+--------
+
+Create a cube `orders_cube` on `orders` table as follows
+
+    CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+    GROUP = (custKEY, ORDERkey), format= 'orc')
+
+Use `SHOW CREATE CUBE` command to show the SQL statement that was used to create the cube `orders_cube`:
+
+    SHOW CREATE CUBE orders_cube;
+
+``` sql
+CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+GROUP = (custKEY, ORDERkey), format= 'orc')
+```

hetu-docs/zh/admin/audit-log.md

@@ -1,6 +1,6 @@
 # 审计日志
 
-openLooKeng审计日志记录功能是一个自定义事件监听器，在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
+openLooKeng审计日志记录功能是一个自定义事件监听器，监听openLooKeng集群启停与集群中节点的动态添加与删除事件；监听WebUi用户登录与退出事件；监听查询事件,在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
 
 1. 事件发生时间
 2. 用户ID
@@ -23,15 +23,17 @@ openLooKeng审计日志记录功能是一个自定义事件监听器，在查询
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
 其他审计日志记录属性包括：
 
-`hetu.event.listener.audit.file`：可选属性，用于定义审计文件的绝对文件路径。确保运行openLooKeng服务器的进程对该目录有写权限。
+`hetu.event.listener.type`：用于定义审计日志的记录类型，允许的值为AUDIT和LOGGER。
 
-`hetu.event.listener.audit.filecount`：可选属性，用于定义要使用的文件数。
+`hetu.auditlog.logoutput`：用于定义审计文件的绝对目录路径。确保运行openLooKeng服务器的进程对该目录有写权限。
 
-`hetu.event.listener.audit.limit`：可选属性，用于定义写入任一文件的最大字节数。
+`hetu.auditlog.logconversionpattern`：用于定义审计日志的轮转模式。允许的值为yyyy-MM-dd.HH和yyyy-MM-dd。
 
 配置文件示例：
 
@@ -43,4 +45,6 @@ hetu.event.listener.listen.query.completion=true
 hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
 hetu.event.listener.audit.filecount=1
 hetu.event.listener.audit.limit=100000
+hetu.auditlog.logoutput=/var/log/
+hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```

hetu-docs/zh/admin/extension-execution-planner.md

@@ -0,0 +1,24 @@
+#扩展物理执行计划
+本节介绍openLooKeng如何添加扩展物理执行计划。通过物理执行计划的扩展，openLooKeng可以使用其他算子加速库来加速SQL语句的执行。
+
+##配置
+在配置文件`config.properties`增加如下配置：
+
+``` properties
+extension_execution_planner_enabled=true
+extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
+extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
+```
+
+上述属性说明如下：
+
+- `extension_execution_planner_enabled`：是否开启扩展物理执行计划特性。
+- `extension_execution_planner_jar_path`：指定扩展jar包的文件路径。
+- `extension_execution_planner_class_path`：指定扩展jar包中执行计划生成类的包路径。
+
+
+##使用
+当运行openLooKeng时，可在WebUI或Cli中通过如下命令控制扩展物理执行计划的开启:
+```
+set session extension_execution_planner_enabled=true/false
+```

hetu-docs/zh/admin/properties.md

@@ -115,6 +115,20 @@
 > 
 > 此属性是在JVM堆中为openLooKeng不跟踪的分配留作裕量/缓冲区的内存量。
 
+### `query.suspend-query-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 系统资源不足时，临时挂起运行中的查询。
+
+### `query.max-suspended-queries`
+
+> -   **类型：** `integer`
+> -   **默认值：** `10`
+>
+> 终止查询之前，查询挂起尝试的最大次数。仅当`query.suspend-query-enabled`设置为`true`时，此属性才生效。
+
 ## 溢出属性
 
 ### `experimental.spill-enabled`
@@ -148,6 +162,24 @@
 > 
 > 此配置属性可由`spill_window_operator`会话属性重写。
 
+### `experimental.spill-build-for-outer-join-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 为右外连接和全外连接操作启用溢出功能。
+>
+> 此config属性可被`spill_build_for_outer_join_enabled`会话属性覆盖。
+
+### `experimental.inner-join-spill-filter-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用基于布隆过滤器的构建侧溢出匹配，以进行探查侧溢出决策。
+>
+> 此config属性可被`inner_join_spill_filter_enabled`会话属性覆盖。
+
 ### `experimental.spill-reuse-tablescan`
 
 > - **类型**：`boolean`
@@ -157,13 +189,13 @@
 >
 > 此配置属性可由`spill_reuse_tablescan`会话属性重写。
 
-### experimental.spiller-spill-path`
+### `experimental.spiller-spill-path`
 
 > - **类型：** `string`
 > - **无默认值。** 启用溢出时必须设置。
 > 
 > 溢出内容写入的目录。该属性可以是一个逗号分隔的列表，以同时溢出到多个目录，这有助于利用系统中安装的多个驱动器。
-> 
+> 当`experimental.spiller-spill-to-hdfs`为`true`时，`experimental.spiller-spill-path`必须只包含一个目录。
 > 不建议溢出到系统驱动器上。最重要的是，不要溢出到写入JVM日志的驱动器，因为磁盘过度使用可能导致JVM长时间暂停，从而导致查询失败。
 
 ### `experimental.spiller-max-used-space-threshold`
@@ -208,7 +240,7 @@
 >
 > 用于在Reuse Exchange中缓存页面的内存限制。
 
-### experimental.spill-compression-enabled`
+### `experimental.spill-compression-enabled`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -222,6 +254,69 @@
 > 
 > 允许使用随机生成的密钥（每个溢出文件）来加密和解密溢出到磁盘的数据。
 
+### `experimental.spill-direct-serde-enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 允许将页面直接序列化/读取到流中或从流中序列化/读取页面。
+
+### `experimental.spill-prefetch-read-pages`
+
+> -   **类型：** `integer`
+> -   **默认值：** `1`
+>
+> 设置从溢出文件读取时预取的页数。
+
+
+### `experimental.spill-use-kryo-serialization`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用基于Kryo的序列化以溢出到磁盘，而不使用默认的Java序列化器。
+
+
+### `experimental.revocable-memory-selection-threshold`
+
+> -   **类型：** `data size`
+> -   **默认值：** `512 MB`
+>
+> 设置运算符可撤销内存的内存选择阈值，直接为准备撤销的剩余字节分配可撤销内存。
+
+### `experimental.prioritize-larger-spilts-memory-revoke`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `true`
+>
+> 启用对具有较大可撤销内存的Split进行优先级排序。
+
+### `experimental.spill-non-blocking-orderby`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 开启按照运算符排序，使用异步机制溢出。即使在溢出正在进行时，也可以累积输入，并在次要数据累积超过阈值或主溢出完成时启动次溢出。阈值的默认值是20MB到可用内存的5%之间的最小值。此属性必须与`experimental.spill-enabled`属性结合使用。
+>
+> 此config属性可被`spill_non_blocking_orderby`会话属性覆盖。
+
+### `experimental.spiller-spill-to-hdfs`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 启用溢出到HDFS。当此属性设置为`true`时，必须设置`experimental.spiller-spill-profile`属性，并且`experimental.spiller-spill-path`必须仅包含单个路径。
+
+### `experimental.spiller-spill-profile`
+
+> -   **类型：** `string`
+> -   **无默认值。** 启用溢出到HDFS时必须设置此属性。
+>
+>
+> 此属性定义用于溢出的[filesystem](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果此属性设置为`experimental.spiller-spill-profile=spill-hdfs`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`spill-hdfs.properties`，其中包含必要的信息，包括身份验证类型、config和keytab（如果适用，详情请参见[filesystem](../develop/filesystem.md）)。
+>
+> 当`experimental.spiller-spill-to-hdfs`设置为`true`时，必须配置此属性。所有Coordinator和Worker的配置文件中必须包含此属性。指定的文件系统必须可由所有Worker访问，并且Worker必须能够读取和写入指定文件系统中`experimental.spiller-spill-path`文件夹中指明的路径。
+
 ## 交换属性
 
 在openLooKeng节点之间为查询的不同阶段交换数据。调整这些属性可有助于解决节点间通信问题或提高网络利用率。
@@ -259,21 +354,124 @@
 > 
 > 如果网络延迟较高，增大该值可以提高网络吞吐量。减小该值可以提高大型集群的查询性能，因为它减少了由于交换客户端缓冲区保存了较多任务（而不是保存较少任务中的较多数据）的响应而导致的倾斜。
 
-### `exchange.max-error-duration`
-
-> - **类型：** `duration`
-> - **最小值：** `1m`
-> - **默认值：** `7m`
-> 
-> 交换错误最大缓冲时间，超过该时限则查询失败。
-
 ### `sink.max-buffer-size`
 
 > - **类型：** `data size`
 > - **默认值：** `32MB`
 > 
-> 上游任务等待拉取任务数据的输出缓冲区大小。如果任务输出是经过哈希分区的，那么缓冲区将在所有分区的使用者之间共享。如果网络延迟较高或集群中有多个节点，增加此值可以提高在阶段之间传输的数据的网络吞吐量。
-
+>等待上游任务拉取的任务数据的输出缓冲区大小。如果任务输出是哈希分区的，则缓冲区将在所有分区的消费者之间共享。如果网络延迟高或集群中有许多节点，则增加此值可以提高阶段之间传输数据的网络吞吐量。
+ 
+ ## 故障恢复处理属性
+ 
+ ### 失败重试策略
+ 
+ ### `failure.recovery.retry.profile`
+ 
+ > -   **类型：** `string`
+ > -   **默认值：** `default`
+ >
+ > 此属性定义用于确定HTTP客户端上是否发生故障的故障检测配置文件。此属性的值`<profile-name>`必须对应`etc/failure-retry-policy/`路径中的`<profile-name>.properties`文件。如果没有此类配置文件可用，并且未设置此属性，则使用“default”配置文件。
+ > 例如，`failure.recovery.retry.profile="test"`要求`test.properties`文件存在于`etc/failure-retry-policy`路径中。
+ > `test.properties`文件必须包含指定的`failure.recovery.retry.type`。
+ 
+ 
+ ### `failure.recovery.retry.type`
+ 
+ > -   **类型：** `string`
+ > -   **默认值：** `timeout`
+ >
+ > 此属性用来设置正在使用的故障检测机制。默认值是基于`timeout`的故障检测。
+ 
+ #### 基于`timeout`的故障检测
+ 
+ > 如果使用此机制，HTTP客户端故障将在指定时间段内重试，重试失败则被视为永久故障。
+ >
+ > 可以为此类故障检测定义`max.error.duration`属性。
+ 
+ #### 基于`max-retry`的故障检测
+ 
+ > 如果使用此机制，HTTP客户端故障将在被视为永久故障之前重试指定次数。
+ > 可以为此类故障检测定义`max.retry.count`和`max.error.duration`属性。
+ > 在这种类型的故障检测中，在查询故障检测模块之前，会执行`max.retry.count`次重试。当故障检测器模块检测到远程节点发生故障时，HTTP客户端将此故障视为永久故障。否则，例如，当远程工作节点处于活动状态但没有响应时，在`max.error.duration`指定的时间段内重试，重试失败则被视为永久故障。
+ 
+ ### `max.error.duration`
+ 
+ > -   **类型：** `duration`
+ > -   **默认值：** `300s`
+ >
+ > 被视为永久故障前，协调器等待解决任务间相关错误的最长时间。
+ 
+ ### `max.retry.count`
+ 
+ > -   **类型：** `integer`
+ > -   **默认值：** `100`
+ >
+ > 协调器在向故障检测器模块查询远程节点状态之前，对失败任务执行的最大重试次数。
+ > 此属性指定查询失败检测模块之前的最小重试次数。因此，实际故障数量可能会因为集群大小和集群负载而略有不同。
+ > 此属性仅用于基于`max-retry`的故障检测配置文件。
+ > 最小值为100。
+ 
+ ### 故障检测Gossip协议配置
+ 
+ ### `failure-detection-protocol`
+ 
+ >- **类型：** `string`
+ >- **默认值：** `heartbeat`
+ >
+ >此属性定义正在使用的故障检测器的类型。默认配置为`heartbeat`故障检测器。
+ >在`config.properties`文件中，将此属性配置为`gossip`，可以启用Gossip协议。
+ >集群中的所有节点（即协调器和工作节点）都应在其各自的`etc/config.properties`文件中指定此属性。
+ 
+ ### `failure-detector.heartbeat-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `500ms` （500毫秒）
+ >
+ >集群中两个节点之间的消息散播间隔。
+ >在Gossip协议中，两个工作节点间的消息散播频率高于协调器和一个工作节点间。
+ >在协调器的`config.properties`文件中，可以为此属性配置一个较大的值，例如`5s`（5秒）。
+ >在工作节点中，可以使用默认值。
+ 
+ ### `failure-detector.worker-gossip-probe-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `5s`（5秒）
+ >
+ >Gossip协议使用监控任务（与`heartbeat`故障检测器相同）来监控其他节点。
+ >此属性指定监控任务刷新间隔，以触发工作节点消息散播。
+ >仅可以为工作节点指定默认值以外的任何其他值。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
+ 
+ ### `failure-detector.coordinator-gossip-probe-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `5s`（5秒）
+ >
+ >Gossip协议使用监控任务（与heartbeat故障检测器相同）来监控其他节点。
+ >此属性指定监控任务刷新间隔，以触发协调器参与工作节点消息散播。
+ >仅可以为协调器指定默认值以外的任何其他值。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`和`failure-detector.worker-gossip-probe-interval`的值。
+ 
+ ### `failure-detector.coordinator-gossip-collate-interval`
+ 
+ >- **类型：** `duration`
+ >- **默认值：** `2s`（2秒）
+ >
+ >此属性指定协调器整理从所有工作节点获得的所有散播消息的间隔。
+ >此属性只支持为协调器配置。
+ >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
+ 
+ ### `failure-detector.gossip-group-size`
+ 
+ >- **类型：** `integer`
+ >- **默认值：** `Integer.MAX_VALUE`
+ >
+ >此属性定义单个工作节点在集群中散播消息的工作节点数量。
+ >任何大于集群大小（即工作节点数量）的值都意味着all-to-all消息散播。
+ >要保持较低的网络开销，针对大型集群，请将此属性设置为一个较小的值（例如，100工作节点的集群设置为10）。
+ >每次刷新协调器上的工作节点监视任务时，协调器都会定义工作节点URI列表，其大小由`failure-detector.gossip-group-size`指定，以触发worker-to-worker消息散播。
+ >
+>
 ## 任务属性
 
 ### `task.concurrency`
@@ -517,7 +715,7 @@
 
 ## 启发式索引属性
 
-启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持使用ORC存储格式的表支持蜂巢连接器。
+启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持Hive连接器的ORC存储格式的表。
 
 ### `hetu.heuristicindex.filter.enabled`
 
@@ -618,7 +816,7 @@
 
 ### `hetu.split-cache-map.enabled`
 
-> - **类型：**`boolean`
+> - **类型：** `boolean`
 > - **默认值：** `false`
 > 
 > 此属性启用分片缓存功能。 如果启用了状态存储，则分片缓存映射配置也会自动复制到状态存储中。 在具有多个协调器的HA设置的情况下，状态存储用于在协调器之间共享分片的缓存映射。
@@ -634,7 +832,7 @@
 
 > 自动清空使系统能够通过持续监测需要清空的表来自动管理清空作业，以保持最佳性能。引擎从符合清空条件的数据源获取表，并触发对这些表的清空操作。
 
-### `auto-vacuum.enabled:`
+### `auto-vacuum.enabled`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -661,7 +859,7 @@
 >
 > **注意：** 此属性只能在协调节点中配置。
 
-## **CTE属性**
+## CTE属性
 
 ### `cte.cte-max-queue-size`
 
@@ -700,18 +898,25 @@
 > 
 > 远程任务错误最大缓冲时间，超过该时限则查询失败。
 
-## 分布式快照
+## 查询恢复
+
+### `recovery_enabled`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 此会话属性用于启用或禁用恢复框架，该框架在发生故障时启用或禁用查询重启/恢复。
 
 ### `snapshot_enabled`
 
-> - 类型：`boolean`
-> - **默认值**：`false`
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
 >
-> 此会话属性用于启用或禁用分布式快照功能。
+> 启用恢复框架时，启用此会话属性可以在查询执行期间捕获快照。如果未启用恢复框架，则此属性不生效。
 
 ### `hetu.experimental.snapshot.profile`
 
-> - 类型：`string`
+> - **类型：**`string`
 >
 > 此属性定义用于存储快照的[文件系统](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果将该属性设置为`hetu.experimental.snapshot.profile=snapshot-hdfs1`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`snapshot-hdfs1.properties`，其中包含的必要信息包括身份验证类型、配置和密钥表（如适用）。具体细节请参考[文件系统](../develop/filesystem.md)相关章节。
 >
@@ -719,23 +924,30 @@
 >
 > 作为实验性属性，或可以将快照存储在非文件系统位置，如连接器。
 
-### `hetu.snapshot.maxRetries`
+### `hetu.recovery.maxRetries`
 
-> - 类型：`int`
-> - **默认值**：`10`
+> - **类型：** `integer`
+> - **默认值：** `10`
 >
-> 此属性定义查询的错误恢复尝试的最大次数。达到限制时，查询失败。
+> 此属性定义查询错误恢复尝试的最大次数。当达到限制时，查询失败。
 >
-> 也可以使用`snapshot_max_retries`会话属性在每个查询基础上指定。
+> 也可以使用`recovery_max_retries`会话属性为每个查询指定此属性。
 
-### `hetu.snapshot.retryTimeout`
+### `hetu.recovery.retryTimeout`
 
-> - 类型：`duration`
-> - **默认值：**`10m`（10分钟）
+> - **类型：** `duration`
+> - **默认值：** `10m`（10分钟）
 >
-> 此属性定义系统等待所有任务成功恢复的最大时长。如果在此超时时限内任何任务未就绪，则认为恢复失败，查询将尝试从较早快照恢复（如果可用）。
+> 此属性定义系统等待所有任务成功恢复的最长时间。如果在此时间内有任何任务未就绪，则恢复尝试将被视为失败，查询将尝试从较早的快照恢复（如果可用）。
 >
-> 也可以使用`snapshot_retry_timeout`会话属性在每个查询基础上指定。
+> 也可以使用`recovery_retry_timeout`会话属性为每个查询指定此属性。
+
+### `hetu.snapshot.useKryoSerialization`
+
+> -   **类型：** `boolean`
+> -   **默认值：** `false`
+>
+> 为快照启用基于Kryo的序列化，而不是默认的Java序列化。
 
 ## HTTP客户端属性配置
 
@@ -757,4 +969,13 @@
 > 此参数定义了http客户端接收响应的时间阈值。
 > 当超过所配置时间，客户端没有接收到任何响应，则视为客户端的请求提交失败。
 >
-> （注意： 建议在高负载环境下，该参数配置大一点。）
+> （注意： 建议在高负载环境下，该参数配置大一点。）
+
+## 连接器属性配置
+
+### `case-insensitive-name-matching`
+
+> -   **类型:** `boolean`
+> -   **默认值:** `false`
+>
+> 不区分大小写匹配数据库和集合名称，默认区分大小写。

hetu-docs/zh/admin/reliable-execution.md

@@ -10,9 +10,9 @@
 
 自版本1.2.0起，openLooKeng支持恢复任务和工作节点故障。
 
-## 启用分布式快照
-
-分布式快照适用于长时间运行的查询任务。该功能默认为禁用状态，可以通过会话属性[`snapshot_enabled`](properties.md#snapshot_enabled)启用或禁用。建议仅在对可靠性要求高的复杂查询场景下启用该功能。
+## 启用恢复框架
+     
+恢复框架对于长时间运行的查询最有用。默认禁用，可以使用会话属性[`recovery_enabled`](properties.md#recovery_enabled)启用和禁用恢复框架。建议仅对可靠性要求高的复杂查询启用该功能。
 
 ## 要求
 
@@ -35,7 +35,7 @@
 
 ## 检测
 
-协调节点与远程任务之间的通信长时间失败时，将触发错误恢复，由[`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration)配置控制。
+当协调器与远程任务之间的通信长时间失败时，将触发错误恢复，由[`故障恢复处理属性`](properties.md#故障恢复处理属性)配置控制。
 
 ## 存储注意事项
 
@@ -53,8 +53,20 @@
 
 从错误和快照中恢复需要成本。捕获快照需要时间，时间长短取决于复杂性。因此，需要在性能和可靠性之间进行权衡。
 
-建议仅在必要时启用分布式快照，如运行时间较长的查询任务。对于这些类型的工作负载，捕获快照的开销可以忽略不计。
+建议在必要时打开快照捕获，例如对于长时间运行的查询。对于这些类型的工作负载，拍摄快照的开销可以忽略不计。
+
+## 快照统计信息
+
+在调试模式下启动CLI时，快照捕获信息和恢复信息将与查询结果一起显示在CLI中。
+
+快照捕获统计信息包括捕获的快照数量、捕获的快照大小、捕获快照所需的CPU时间和在查询期间捕获快照所需的挂钟时间。所有快照和最后一个快照的统计信息会分别显示。
+
+快照恢复信息包括查询期间从快照恢复的次数、加载用于恢复的快照大小、从快照恢复所需的CPU时间和从快照恢复所需的挂钟时间。仅当查询期间发生恢复时，才会显示恢复信息。
+
+此外，在查询正在进行时，将显示捕获的快照数量和恢复的快照的ID。更多详细信息，见下图。
+
+![](../images/snapshot_statistics_cn.png)
 
 ## 配置
 
-与分布式快照功能相关的配置可参见[属性参考](properties.md#分布式快照)。
+与恢复框架功能相关的配置，请参见[属性参考](properties.md#查询恢复)。

hetu-docs/zh/admin/spill.md

@@ -31,6 +31,10 @@
 
 openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD )），因此无需使用RAID进行溢出。
 
+## 溢出到HDFS
+
+操作可以直接溢出到HDFS。将`experimental.spiller-spill-to-hdfs`设置为`true`，配置`experimental.spiller-spill-profile`，并且`spiller-spill-path`必须仅包含一个目录。（更多详情请参见`experimental.spiller-spill-to-hdfs`和`experimental.spiller-spill-profile`属性）
+
 ## 溢出压缩
 
 当启用溢出压缩（`tuning-spilling`中的`spill-compression-enabled`属性）时，溢出页将被压缩后再写入磁盘。启用此特性可以减少磁盘I/O，但会牺牲额外的CPU负载来压缩和解压缩溢出页。
@@ -53,6 +57,8 @@ openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipe
 
 通过这种机制，联接操作符使用的峰值内存可以降低到最大构建表分区的大小。假设没有数据倾斜，这个值将是整个构建表大小的`1 / task.concurrency`倍。
 
+注意：spill-to-disk不支持交叉连接。
+
 ### 聚合
 
 聚合函数对一组值执行操作并返回一个值。如果要聚合的组数量很大，可能需要大量内存。当启用溢出到磁盘时，如果没有足够的内存，则中间累积的聚合结果将写入磁盘。结果被重新加载回来，并以较低的内存占用量合并。
@@ -60,6 +66,7 @@ openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipe
 ### 排序
 
 如果尝试对大量数据进行排序，可能需要大量内存。当启用为排序溢出到磁盘时，如果内存不足，则中间排序结果将写入磁盘。结果被重新加载回来，并以较低的内存占用量合并。
+通常，当溢出正在进行时，运算符将被阻止接受输入，但当`experimental.spill-non-blocking-orderby`设置为`true`时，使用异步机制溢出（请参见`experimental.spill-non-blocking-orderby`）。
 
 ### 开窗函数
 

hetu-docs/zh/admin/web-interface.md

@@ -31,3 +31,54 @@ openLooKeng提供了一个用于监视和管理查询的Web界面。Web界面可
 > -   **默认值:** `false`
 >
 > 默认情况下，基于HTTP的非安全环境禁用WEB UI。可以通过配置`etc/config.properties`文件的`hetu.queryeditor-ui.allow-insecure-over-http`属性启用(例子: hetu.queryeditor-ui.allow-insecure-over-http=true)。
+
+### `hetu.queryeditor-ui.execution-timeout`
+
+> -   **类型：** `duration`
+> -   **默认值：** `100 DAYS`
+>
+> UI执行超时默认设置为100天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.execution-timeout`属性修改。
+
+### `hetu.queryeditor-ui.max-result-count`
+
+> - **类型：** `int`
+> - **默认值：** `1000`
+>
+> UI最大结果计数默认设置为1000。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.max-result-count`属性修改。
+
+### `hetu.queryeditor-ui.max-result-size-mb`
+
+>- **类型：** `size`
+>- **默认值：** `1GB`
+>
+>UI最大结果大小默认设置为1 GB。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.max-result-size-mb`属性修改。
+
+### `hetu.queryeditor-ui.session-timeout`
+
+> -   **类型：** `duration`
+> -   **默认值：** `1 DAYS`
+>
+> UI会话超时默认设置为1天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.session-timeout`属性修改。
+
+### `hetu.queryhistory.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `1000`
+>
+> openLooKeng储存的历史查询记录最大数量。可以通过配置`etc/config.properties`文件的`hetu.queryhistory.max-count`属性修改。
+
+### `hetu.collectionsql.max-count`
+
+> -   **Type:** `int`
+> -   **Default value:** `100`
+>
+> 每位用户收藏sql语句条数上限.可以通过配置`etc/config.properties`文件的`hetu.collectionsql.max-count`属性修改。
+
+
+## 备注
+
+收藏sql语句的最大长度默认为600，可通过如下步骤对其进行修改：
+
+1. 根据hetu-metastore.properties文件中jdbc配置信息登录mysql数据库。
+
+2. 选中hetu_favorite表，使用命令`alter table hetu_favorite modify query varchar(2000) not null;`修改收藏语句最大长度为2000。

hetu-docs/zh/connector/Hudi.md

@@ -1,5 +1,8 @@
 # Hudi连接器
 
+### 版本说明
+目前Hudi只支持0.7.0版本。
+
 ### Hudi介绍
 
 Apache Hudi是一个快速迭代的数据湖存储系统，可以帮助企业构建和管理PB级数据湖。它提供在DFS上存储超大规模数据集，同时使得流式处理如果批处理一样，该实现主要是通过如下两个原语实现。

hetu-docs/zh/connector/memory.md

@@ -173,8 +173,8 @@ totaldiskbyteusage | totalmemorybyteusage
 
 | Index ID     | 是否可用于`sorted_by`或`index_columns`   | 支持的运算符                           |
 |--------------|-----------------------------------------|---------------------------------------|
-| Bloom        | 两者都可                                 | `=` `IN`                             |                   
-| MinMax       | 仅`sorted_by`                           | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
+| Bloom        | 仅`index_columns`                                 | `=` `IN`                             |                   
+| MinMax       | 两者都可                           | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 | Sparse       | 仅`sorted_by`                           | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 
 使用统计信息

hetu-docs/zh/connector/opengauss.md

@@ -41,6 +41,95 @@ openGauss连接器为每个openGauss模式提供一个模式。可通过执行`S
 
 如果对目录属性文件使用不同的名称，请使用该目录名称，而不要使用上述示例中的`opengauss`。
 
+## openGauss Update/Delete 支持
+
+### 使用openGauss连接器创建表
+
+示例：
+
+```sql
+CREATE TABLE opengauss_table (
+    id int,
+    name varchar(255));
+```
+
+### 对表执行INSERT
+
+示例：
+
+```sql
+INSERT INTO opengauss_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### 对表执行UPDATE
+
+示例：
+
+```sql
+UPDATE opengauss_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+上述示例将列`id`中值为`1`所在行的列`name`的值更新为`Tim`。
+
+UPDATE前的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+UPDATE后的SELECT结果
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+### 对表执行DELETE
+
+示例：
+
+```sql
+DELETE FROM opengauss_table
+  WHERE id=2;
+```
+
+以上示例删除了值为`2`的列`id`的行。
+
+DELETE前的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+DELETE后的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM opengauss_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 **注意**
 
 > - openGuass数据库兼容类型为O（即DBCOMPATIBILITY = A）时不支持`Date`数据类型。
@@ -59,4 +148,4 @@ openGauss连接器为每个openGauss模式提供一个模式。可通过执行`S
 
 暂不支持以下SQL语句：
 
-[DELETE](../sql/delete.md)、[GRANT](../sql/grant.md)、[REVOKE](../sql/revoke.md)、[SHOW GRANTS](../sql/show-grants.md)、[SHOW ROLES](../sql/show-roles.md)、[SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md)、[REVOKE](../sql/revoke.md)、[SHOW GRANTS](../sql/show-grants.md)、[SHOW ROLES](../sql/show-roles.md)、[SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/zh/connector/postgresql.md

@@ -41,8 +41,97 @@ PostgreSQL连接器为每个PostgreSQL模式提供一个模式。可通过执行
 
 如果对目录属性文件使用不同的名称，请使用该目录名称，而不要使用上述示例中的`postgresql`。
 
+## PostgreSQL Update/Delete 支持
+
+### 使用PostgreSQL连接器创建表
+
+示例：
+
+```sql
+CREATE TABLE postgresql_table (
+    id int,
+    name varchar(255));
+```
+
+### 对表执行INSERT
+
+示例：
+
+```sql
+INSERT INTO postgresql_table
+  VALUES
+     (1, 'Jack'),
+     (2, 'Bob');
+```
+
+### 对表执行UPDATE
+
+示例：
+
+```sql
+UPDATE postgresql_table
+  SET name='Tim'
+  WHERE id=1;
+```
+
+上述示例将列`id`中值为`1`所在行的列`name`的值更新为`Tim`。
+
+UPDATE前的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+id | name
+----+------
+  1 | Jack
+  2 | Bob
+(2 rows)
+```
+
+UPDATE后的SELECT结果
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+### 对表执行DELETE
+
+示例：
+
+```sql
+DELETE FROM postgresql_table
+  WHERE id=2;
+```
+
+以上示例删除了值为`2`的列`id`的行。
+
+DELETE前的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  2 | Bob
+  1 | Tim
+(2 rows)
+```
+
+DELETE后的SELECT结果：
+
+```sql
+lk:default> SELECT * FROM postgresql_table;
+ id | name
+----+------
+  1 | Tim
+(1 row)
+```
+
 ## PostgreSQL连接器限制
 
 暂不支持以下SQL语句：
 
-[DELETE](../sql/delete.md)、[GRANT](../sql/grant.md)、[REVOKE](../sql/revoke.md)、[SHOW GRANTS](../sql/show-grants.md)、[SHOW ROLES](../sql/show-roles.md)、[SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[GRANT](../sql/grant.md)、[REVOKE](../sql/revoke.md)、[SHOW GRANTS](../sql/show-grants.md)、[SHOW ROLES](../sql/show-roles.md)、[SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/zh/connector/redis.md

@@ -0,0 +1,171 @@
+Redis 连接器
+====================
+概述
+--------
+此连接器允许在openlookeng中，将redis中一个kv键值对映射成表的一行数据
+
+**说明**
+
+*kv键值对只能在Reids中映射为string或hash类型。keys可以存储为一个zset,然后keys可以被分割成多个片*
+
+*支持 Redis 2.8.0 或更高版本*
+
+
+配置
+-------------
+要配置Redis连接器，创建具有以下内容的目录属性文件etc/catalog/redis.properties，并适当替换以下属性：
+``` properties
+connector.name=redis
+redis.table-names=schema1.table1,schema1.table2
+redis.nodes=host1:port
+```
+### 多个 Redis Servers
+可以根据需要创建任意多的目录，因此，如果有额外的redis server，只需添加另一个不同的名称的属性文件到etc/catalog中（确保它以.properties结尾）。例如，如果将属性文件命名为sales.properties，openLooKeng将使用配置的连接器创建一个名为sales的目录。
+
+配置属性
+------------------------
+配置属性包括：
+
+| 属性名称                                                       | 说明                                                                          |
+|:-----------------------------------------------------------|:----------------------------------------------------------------------------|
+| `redis.table-names`                                        | catalog 提供的所有表的列表                                                           |
+| `redis.default-schema`                                     | 表的默认schema名 (默认`default`)                                                   |
+| `redis.nodes`                                              | Redis server的节点列表                                                           |
+| `redis.connect-timeout`                                    | 连接Redis server的超时时间 (ms) (默认 2000)                                          |
+| `redis.scan-count`                                         | 每轮scan获得的key的数量 (默认 100)                                                    |
+| `redis.key-prefix-schema-table`                            | Redis keys 是否有 schema-name:table-name的前缀   (默认  false)                      |
+| `redis.key-delimiter`                                      | 如果`redis.key-prefix-schema-table`被启用，那么schema-name和table-name的分隔符为 (默认 `:`) |
+| `redis.table-description-dir`                              | 存放表定义json文件的相对地址 (默认 `etc/redis/`)                                          |
+| `redis.hide-internal-columns`                              | 内部列是否在元数据中隐藏(默认 true)                                                       |
+| `redis.database-index`                                     | Redis database 的索引  (默认 0)                                                  |
+| `redis.password`                                           | Redis server 密码 (默认 null)                                                   |
+| `redis.table-description-interval`                         | flush表定义文件的时间间隔（默认不会flush,意味着Plugin被加载后,表定义一直留存在内存中,不再主动读取json文件）           | 
+
+内部列
+----------------
+
+| 列名        | 类型    | 说明                                                  |
+|:-------------------| :------ |:----------------------------------------------------|
+| `_key`             | VARCHAR  | Redis key.                                          |
+| `_value`           | VARCHAR   | 与key相对应的值                                           |
+| `_key_length`   | BIGINT  | key  的字节大小                                          |
+| `_key_corrupt`     | BOOLEAN  | 如果解码器无法解码此行的key，则为true。当为 true 时，从键映射的数据列应被视为无效。    |
+| `_value_corrupt`   | BOOLEAN | 如果解码器无法解码此行的value，则为true。当为 true 时，从该值映射的数据列应被视为无效。 |
+
+
+
+表定义文件
+----------------------
+对于openLooKeng，每个kv键值对 必须被映射到列中，以便允许对数据查询。这很像kafka connector,所以你可以参考 kafka连接器教程
+
+
+表定义文件由一个表的JSON定义组成。文件名可以任意，但必须以.json结尾。
+
+以nation.json为例
+``` json
+{
+    "tableName": "nation",
+    "schemaName": "tpch",
+    "key": {
+        "dataFormat": "raw",
+        "fields": [
+            {
+                "name": "redis_key",
+                "type": "VARCHAR(64)",
+                "hidden": "true"
+            }
+        ]
+    },
+    "value": {
+        "dataFormat": "json",
+        "fields": [
+            {
+                "name": "nationkey",
+                "mapping": "nationkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "name",
+                "mapping": "name",
+                "type": "VARCHAR(25)"
+            },
+                        {
+                "name": "regionkey",
+                "mapping": "regionkey",
+                "type": "BIGINT"
+            },
+            {
+                "name": "comment",
+                "mapping": "comment",
+                "type": "VARCHAR(152)"
+            }
+       ]
+    }
+}
+```
+在redis,相应的有这样的数据
+```shell
+127.0.0.1:6379> keys tpch:nation:*
+ 1) "tpch:nation:2"
+ 2) "tpch:nation:4"
+ 3) "tpch:nation:16"
+ 4) "tpch:nation:18"
+ 5) "tpch:nation:10"
+ 6) "tpch:nation:17"
+ 7) "tpch:nation:1"
+```
+```shell
+127.0.0.1:6379> get tpch:nation:1
+"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
+```
+我们可以使用redis connector从redis中获取数据，（redis_key没有显示，这是因为我们设置了"hidden": "true" ）
+```shell
+lk> select * from redis.tpch.nation;
+ nationkey |      name      | regionkey |                                                      comment                                                       
+-----------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
+         3 | CANADA         |         1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold              
+         9 | INDONESIA      |         2 |  slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 
+        19 | ROMANIA        |         3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account    
+         2 | BRAZIL         |         1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special   
+```
+**说明**
+
+*如果属性 `redis.key-prefix-schema-table` 是false (默认是false),那么当前所有key的都会被视作的nation表的key,不会发生匹配过滤*
+
+有关各种可用解码器的描述，请参考kafka连接器教程
+
+除了kafka支持的类型，Redis connector对value字段支持hash类型
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "value": {
+        "dataFormat": "hash",
+        "fields": [
+          ...
+        ]
+      }
+    }
+```
+
+Redis connector 支持``zset`` 作为``key``在Redis中存储类型。
+当且仅当``zset`` 作为key的存储格式时，split切片功能才能被真正支持，因为我们可以使用`zrange zsetkey split.start split.end`来得到一个切片的keys
+``` json
+      {
+      "tableName": ...,
+      "schemaName": ...,
+      "key": {
+        "dataFormat": "zset",
+        "name": "zsetkey", //zadd zsetkey score member
+        "fields": [
+            ...
+        ]
+      }
+    }
+```
+
+Redis 连接器的局限性
+---------------------------
+只支持读操作，不支持写操作.
+
+

hetu-docs/zh/develop/_index.md

@@ -1,4 +1,4 @@
 
 # 开发者指南
 
-openLooKeng基于Trino(之前称为PrestoSQL)，是Trino开源项目的一个分支。与Trino开源项目相比，openLooKeng提供了额外的优化和增强功能，可以在任何位置对任何数据（包括远程数据源）进行现场分析。本指南适用于openLooKeng参与者和插件开发人员。
+openLooKeng基于Trino 316版本(之前称为PrestoSQL)，是Trino开源项目的一个分支。与Trino开源项目相比，openLooKeng提供了额外的优化和增强功能，可以在任何位置对任何数据（包括远程数据源）进行现场分析。本指南适用于openLooKeng参与者和插件开发人员。

hetu-docs/zh/develop/connectors.md

@@ -31,7 +31,7 @@
 
 
 
-### ConnectorSplitManger
+### ConnectorSplitManager
 
 分片管理器将表的数据分区成多个块，这些块由 openLooKeng 分发至工作节点进行处理。
 

hetu-docs/zh/develop/getting-started.md

@@ -1,4 +1,4 @@
-``# 入门指南
+# 入门指南
 
 ## 要求
 

hetu-docs/zh/faq/faq.md

@@ -35,6 +35,10 @@
    > openLooKeng微信群：请添加openLooKeng小助手微信号：openLooKengoss，小助手会拉您入群  
    > openLooKeng B站： https://space.bilibili.com/627629884 
 
+8. openLooKeng基于Trino哪个版本进行开发？
+
+   > 基于Trino 316版本进行开发。
+
 ## 功能
 
 1. openLooKeng目前支持哪些连接器?

hetu-docs/zh/index.md

@@ -46,6 +46,8 @@ headless: true
     - [审计日志]({{< relref "./docs/admin/audit-log.md" >}})
     - [可靠查询执行]({{< relref "./docs/admin/reliable-execution.md" >}})
     - [JDBC数据源多分片管理]({{< relref "./docs/admin/multi-split-for-jdbc-data-source.md" >}})
+    - [扩展物理执行计划]({{< relref "./docs/admin/extension-execution-planner.md" >}})
+
 - [查询优化器]("#")
     - [表统计]({{< relref "./docs/optimizer/statistics.md" >}})
     - [EXPLAIN成本]({{< relref "./docs/optimizer/cost-in-explain.md" >}})
@@ -82,6 +84,7 @@ headless: true
     - [JMX]({{< relref "./docs/connector/jmx.md" >}})
     - [Kafka]({{< relref "./docs/connector/kafka.md" >}})
     - [Kafka连接器教程]({{< relref "./docs/connector/kafka-tutorial.md" >}})
+    - [Redis] ({{< relref "./docs/connector/redis.md" >}})
     - [本地文件]({{< relref "./docs/connector/localfile.md" >}})
     - [内存]({{< relref "./docs/connector/memory.md" >}})
     - [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
@@ -171,6 +174,7 @@ headless: true
     - [SHOW CACHE]({{< relref "./docs/sql/show-cache.md" >}})
     - [SHOW CATALOGS]({{< relref "./docs/sql/show-catalogs.md" >}})
     - [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
+    - [SHOW CREATE CUBE]({{< relref "./docs/sql/show-create-cube.md" >}})
     - [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
     - [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
     - [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
@@ -215,6 +219,8 @@ headless: true
     - [任务资源]({{< relref "./docs/rest/task.md" >}})
 
 - [发行说明]("#")
+    - [1.6.1 (2022年4月27日)]({{< relref "./docs/releasenotes/releasenotes-1.6.1.md" >}})
+    - [1.6.0 (2022年3月30日)]({{< relref "./docs/releasenotes/releasenotes-1.6.0.md" >}})
     - [1.5.0 (2021年12月30日)]({{< relref "./docs/releasenotes/releasenotes-1.5.0.md" >}})
     - [1.4.1 (2021年11月12日)]({{< relref "./docs/releasenotes/releasenotes-1.4.1.md" >}})
     - [1.4.0 (2021年10月15日)]({{< relref "./docs/releasenotes/releasenotes-1.4.0.md" >}})

hetu-docs/zh/preagg/overview.md

@@ -61,8 +61,6 @@ AggregationNode
 
    2.1. 克服为更大的数据集创建Cube的限制。
 
-   2.2. 如果源表已更新，则更新Cube。
-
 ## 启用和禁用StarTree Cube
 启用：
 ```sql 
@@ -121,6 +119,14 @@ SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >=
 由于插入Cube的数据是为`nationkey >= 5`，只有匹配此条件的查询才会使用Cube。
 不符合条件的查询将继续工作，但不会使用Cube。
 
+如果Cube的源表更新，则对应的Cube自动过期。为了克服这个问题，我们通过引入**RELOAD CUBE**命令在openLooKeng CLI中添加了支持。如果Cube的状态变为“未激活”或“过期”，用户将能够手动重新加载Cube。重新加载Cube`nation_cube`的语法如下:
+
+```sql 
+RELOAD CUBE nation_cube
+```
+
+请注意，此功能仅在CLI支持。在重新加载过程中，如果发生意外错误，用户可以查看原始SQL语句，手动重新创建Cube。
+
 ## 为大型数据集构建Cube
 当前实现的限制之一是不能一次为更大的数据集构建Cube。这是由于集群内存限制。
 处理大量行需要比集群配置更多的内存。这会导致查询失败并显示消息**Query exceeded per-node user memory limit**，也就是警告查询超出每节点用户内存限制。为了克服这个问题，**INSERT INTO CUBE** SQL支持被添加了。
@@ -178,38 +184,56 @@ SHOW CUBES;
 ```
 
 **注意：**
-1. 系统将尝试将所有类型的Predicates重写为Range以查看它们是否可以合并在一起。
+
+① 系统将尝试将所有类型的Predicates重写为Range以查看它们是否可以合并在一起。
    所有连续谓词将合并为单个范围谓词，其余谓词保持不变。
 
    仅支持以下类型并且可以合并在一起。
-   `Integer, TinyInt, SmallInt, BigInt, Date`
+   `Integer, TinyInt, SmallInt, BigInt, Date, String`
 
-   对于其他数据类型，很难确定两个谓词是否连续，因此它们不能合并在一起。 
-   由于这个问题，即使Cube具有所有必需的数据，在查询优化期间也可能不会使用特定Cube。例如，
+   对于字符串数据类型，谓词合并逻辑仅在字符串以数字结尾，并且所有字符串的长度相同时才能生效。例如，
 
 ```sql
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
    INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
 ```
-   这里这两个谓词不能合并到store_id BETWEEN 'A01' AND 'A20'; 
-   因此，Cube不会用于跨越两个谓词的查询；
-
-```sql
-   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube won't be used for optimizing this query. This is a limitation as of now.
-```
-   由于谓词重写，无法支持以下某些查询
-
-```sql   
-   INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
-```
+   插入后，两个谓词将被合并至`'A01' AND 'A20'`。
+   
+   ```sql
+      SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube 能被这个查询语句所使用
+   ```
+   
+   以下示例中，`store_id`值的长度不相同。
+   
+   ```
+      INSERT INTO CUBE store_sales_cube WHERE store_id = 'A1';
+      INSERT INTO CUBE store_sales_cube WHERE store_id = 'A2' 
+   ```
+   
+   根据varchar谓词合并逻辑，store_id谓词将被重写为`store_id >= 'A1' and store < 'A3'`；
+   
+   ```
+      INSERT INTO CUBE store_sales_cube WHERE store_id = 'A10' 
+   ```
+   
+   上述查询将失败，因为`A10`是范围`store_id >= 'A1' and store < 'A3'`的子集。请用户注意这个问题。
+   
+   对于其他数据类型，很难识别两个谓词是否连续，因此它们无法被合并。因此，即使某些Cube具有所有所需的数据，也可能不会被用来优化查询。
+   
+② 谓词重写也有一些限制。如以下查询：
+   
+   ```sql   
+      INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
+   ```
+   
    谓词重写为ss_sold_date_sk >= 2451912为合并连续谓词做准备。
-   由于谓词被重写，他们使用ss_sold_date_sk > 2451911谓词查询将与Cube谓词不匹配，因此不会使用Cube来优化查询。
-   这同样适用于带有<=运算符的谓词，例如，ss_sold_date_sk <= 2451911改写为ss_sold_date_sk < 2451912。
-
-```sql   
-   SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
-```   
-3. 只能合并单列谓词。
+   由于谓词已重写，使用ss_sold_date_sk > 2451911谓词进行查询将无法匹配到Cube谓词，因此不会使用Cube优化查询。同样的情况也适用于具有<=运算符的谓词。例如 ss_sold_date_sk <= 2451911重写为ss_sold_date_sk < 2451912。
+   
+   ```sql   
+      SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
+   ```
+   
+③ 只能合并单列谓词。
 
 ## 未解决的问题和限制
 1. StarTree Cube仅在按基数分组的数量远小于源表中的行数时有效。
@@ -218,10 +242,11 @@ SHOW CUBES;
 4. 即使源表尚未更新，在事务表上创建的Cubes也可能会自动过期。
    这是由于压缩策略将delta文件合并为单个大型ORC文件，这反过来又更改了表的最后修改时间。
    Cube状态是通过比较创建Cube时表的最后修改时间戳与执行查询时表的最后修改时间来确定的。
-5. OpenLooKeng CLI已经过修改，以简化为更大的数据集创建Cubes的过程。 
+5. openLooKeng CLI已经过修改，以简化为更大的数据集创建Cubes的过程。 
    但是这种实现仍然存在局限性，因为该过程涉及将多个Cube谓词合并为一个。
    只有定义在Integer、Long和Date类型上的Cube谓词才能正确合并。 对Char、String类型的支持仍需实现。
-   
+6. 当Varchar类型的谓词的数值长度是一样时可合并。
+
 ## Star Tree上的性能优化
 1. 对同一个group by列的星型查询重写优化：如果查询语句与Cube组匹配，则会改写查询计划将聚合运算结果重定向到Cube结果，否则将添加其他聚合结果内部应用于重写语句。
 2. 平均聚合函数的star tree表扫描优化：如果查询语句与group by列的Cube匹配，则会改写查询计划将聚合运算结果重定向到Cube的预聚合列的平均值结果，否则语句将在内部重写，以选择star tree预聚合Sum和Count结果，随后计算平均值。

hetu-docs/zh/preagg/statements.md

@@ -150,6 +150,26 @@ SHOW CUBES [ FOR table_name ];
     SHOW CUBES FOR orders;
 ```
 
+## RELOAD CUBE
+
+### 概要
+
+``` sql
+RELOAD CUBE cube_name
+```
+
+### 描述
+
+源表更新后重新加载Cube。
+
+### 示例
+
+如果Cube`orders_cube`的源表`orders`被更新，且`orders_cube`的状态为“过期”，运行`RELOAD CUBE cube_name`命令重新加载Cube：
+
+```sql
+RELOAD CUBE orders_cube
+```
+
 ## DROP CUBE
 
 ### 概要

hetu-docs/zh/releasenotes/releasenotes-1.5.0.md

@@ -7,7 +7,7 @@
 | Star Tree             | 1. 支持优化连接查询，如星型模型的查询。<br/>2. 优化查询计划：由于Cube已经包含了汇总结果，所以改写查询计划将聚合运算结果重定向到Cube结果。当group by子句完全与Cube组匹配时性能收益明显。<br/>3. 问题修复，以进一步增强Cube的可用性和健壮性。 |
 | Memory 连接器          | 1. 通过增加对内存表分区的支持，允许跳过整个分区的数据，从而提高内存连接器的性能。<br/>2. 收集内存表的统计信息，以支持基于openLooKeng代价的优化器。 |
 | Task Recovery         | 修复了几个重要的错误，以解决数据不一致问题，以及在高并发和工作节点故障期间偶尔发生的查询挂起问题。 |
-| Yarn上部署openLooKeng  | 支持在yarn上部署启用HA的openLoKeng集群实例，该实例包含一个反向代理（默认为ngnix）和2个或更多协调节点。通过增加和移除yarn容器，实现手动水平缩放openLooKeng集群。 |
+| Yarn上部署openLooKeng  | 支持在yarn上部署启用HA的openLooKeng集群实例，该实例包含一个反向代理（默认为nginx）和2个或更多协调节点。通过增加和移除yarn容器，实现手动水平缩放openLooKeng集群。 |
 | 数据持久化              | 优化了数据溢出到磁盘的机制，将序列化页面直接写入磁盘，而不是缓存。这样，算子可以释放更多内存，相比之前性能提高30%   |                                  |
 ## 已知问题
 

hetu-docs/zh/releasenotes/releasenotes-1.6.0.md

@@ -0,0 +1,23 @@
+# Release 1.6.0
+
+## 关键特性
+
+| 分类                  | 描述                                                         |
+| --------------------- | ------------------------------------------------------------ |
+| Star Tree             | 支持cube更新命令,允许管理员在基础数据更改时轻松更新现有cube的内容 |
+| Bloom Index | 优化布隆过滤器索引大小使缩小十倍以上 |
+| Task Recovery         | 1. 优化执行失败检测时间：当前需要300秒来确定任务失败,然后继续运行。改进这一点将改善执行流程和整体查询时间<br/> 2. 快照时间和大小优化：当执行过程中使用快照时,当前直接使用Java序列化,速度很慢,而且需要更多的空间。使用kryo序列化方式可以减小文件大小并提升速度来增加总吞吐量 |
+| 数据持久化 | 1. 优化计算过程数据下盘速度和大小：当在Hash Aggregation（聚合算法）和GroupBy（分组）算子执行过程中发生溢出时,序列化到磁盘的数据会很慢,而且大小也会更大。因此可以通过减小大小和提高写入速度来提高整体性能。通过使用kryo序列化可以提高速度并减小溢出写盘文件大小<br/>2. 支持溢出到hdfs上：目前计算过程数据可以溢出到多个磁盘,现在支持溢出到hdfs以提高吞吐量<br/>3. 异步溢出/不溢出机制：当可操作内存超过阈值并触发溢出时,会阻塞接受来自下游运算符的数据。接受数据并加入到现有溢出流程将有助于更快地完成任务<br/>4. 支持右外连接&全连接场景下的溢出写盘：当连接类型为右外连接或全连接时,不会溢出构建侧数据,因为需要所有数据在内存中进行查找。当数据量较大时,这将导致内存溢出。因此，通过启用溢出机制并创建一个布隆过滤器来识别溢出的数据,并在与探查侧连接期间使用它 |
+| 连接器增强            | 增强PostgreSQL和openGauss连接器，支持对数据源进行数据更新和删除操作 |
+## 已知问题
+
+| 分类          | 描述                                                         | Gitee问题                                                 |
+| ------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
+| Task Recovery | 启用快照时，执行带事务的CTAS语句时，SQL语句执行报错          | [I502KF](https://e.gitee.com/open_lookeng/issues/list?issue=I502KF) |
+|               | 启用快照并将exchange.is-timeout-failure-detection-enable关闭时，概率性出现错误 | [I4Y3TQ](https://e.gitee.com/open_lookeng/issues/list?issue=I4Y3TQ) |
+| Star Tree     | 在内存连接器中，启用star tree功能后，查询时偶尔出现数据不一致 | [I4QQUB](https://e.gitee.com/open_lookeng/issues/list?issue=I4QQUB) |
+|               | 当同时对10个不同的cube执行reload cube命令时，部分cube无法重新加载 | [I4VSVJ](https://e.gitee.com/open_lookeng/issues/list?issue=I4VSVJ) |
+
+## 获取文档
+
+请参考： [https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/zh](https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/zh)

hetu-docs/zh/releasenotes/releasenotes-1.6.1.md

@@ -0,0 +1,15 @@
+# Release 1.6.1 (2022年04月27日)
+
+## 关键特性
+
+本次发布主要是一些SPI的修改和增强，为扩展更多的场景使用。
+
+| 类别                    | 特性                                                      | PR #s                                                        |
+| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 数据源统计信息               | 增加统计信息获取方式，支持从Connector直接获取统计信息。有些算子可以下推到Connector里面进行计算，可能需要直接从Connector获取统计信息才能展示真正被处理的数据量。                                               | 1450                                                          |
+| 算子处理扩展 | 增加通过用户自定义worker结点物理执行计划的生成，用户可以实现自己的算子pipeline代替原生实现，加速算子处理。 | 1436                                                           |
+| HIVE UDF扩展 | 增加 HIVE UDF 函数命名空间的适配，以支持执行基于HIVE UDF框架编写的UDF（含GenericUDF）。 | 1456                                                           |
+
+## 获取文档
+
+请参考：[https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/zh](https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/zh )

hetu-docs/zh/security/built-in-system-access-control.md

@@ -58,7 +58,7 @@ security.refresh-period=1s
 
 -   `user`（可选）：用于匹配用户名的正则表达式。默认为`.*`。
 -   `catalog`（可选）:用于匹配目录名的正则表达式。默认为`.*`。
-- `allow`（必选）: 布尔类型参数，表示用户是否有访问目录的权限
+- `allow`（必选）: 字符串参数，表示用户是否有访问目录的权限。这个值可以是all、read-only或none，默认为none。将此值设置为read-only，其行为与只读的系统访问控制插件相同。
 
 
 **注意**
@@ -74,15 +74,20 @@ security.refresh-period=1s
     {
       "user": "admin",
       "catalog": "(mysql|system)",
-      "allow": true
+      "allow": all
     },
     {
       "catalog": "hive",
-      "allow": true
+      "allow": all
+    },
+    {
+      "user": "alice",
+      "catalog": "postgresql",
+      "allow": "read-only"
     },
     {
       "catalog": "system",
-      "allow": false
+      "allow": none
     }
   ]
 }

hetu-docs/zh/security/ranger-access-control.md

@@ -7,7 +7,7 @@ Ranger 访问控制
 
 Apache Ranger 为 Hadoop 集群提供了一种全面的安全框架，以跨组件的、一致性的方式进行定义、授权、管理安全策略。Ranger 详细介绍和用户指导可以参考[Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index )。
 
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) 是为 openLooKeng 开发的 Ranger 插件，用于全面的数据安全监控和权限管理。
+[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) 基于Ranger 2.1.0版本进行开发，是为 openLooKeng 开发的 Ranger 插件，用于全面的数据安全监控和权限管理。
 
 编译过程
 -------------------------

hetu-docs/zh/sql/show-create-cube.md

@@ -0,0 +1,32 @@
+
+SHOW CREATE CUBE
+=================
+
+概要
+--------
+
+``` sql
+SHOW CREATE CUBE cube_name
+```
+
+描述
+-----------
+
+显示创建指定cube的SQL语句。
+
+示例
+--------
+
+在`orders`表上创建cube`orders_cube`
+
+    CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+    GROUP = (custKEY, ORDERkey), format= 'orc')
+
+运行`SHOW CREATE CUBE`命令显示用于创建cube`orders_cube`的SQL语句：
+
+    SHOW CREATE CUBE orders_cube;
+
+``` sql
+CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
+GROUP = (custKEY, ORDERkey), format= 'orc')
+```

hetu-filesystem-client/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

hetu-filesystem-client/src/main/java/io/hetu/core/filesystem/HetuHdfsFileSystemClient.java

@@ -35,6 +35,7 @@ import java.nio.file.NoSuchFileException;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.UUID;
 import java.util.stream.Stream;
 
 import static java.nio.file.StandardOpenOption.CREATE_NEW;
@@ -273,6 +274,40 @@ public class HetuHdfsFileSystemClient
         getHdfs().close();
     }
 
+    @Override
+    public long getUsableSpace(Path path) throws IOException
+    {
+        return getHdfs().getStatus(toHdfsPath(path)).getRemaining();
+    }
+
+    @Override
+    public long getTotalSpace(Path path) throws IOException
+    {
+        return getHdfs().getStatus(toHdfsPath(path)).getCapacity();
+    }
+
+    @Override
+    public Path createTemporaryFile(Path path, String prefix, String suffix) throws IOException
+    {
+        String randomNo = UUID.randomUUID().toString();
+        Path finalPath = Paths.get(String.valueOf(path), prefix + randomNo + suffix);
+        unwrapHdfsExceptions(() -> getHdfs().create(toHdfsPath(finalPath)));
+        return finalPath;
+    }
+
+    @Override
+    public Path createFile(Path path) throws IOException
+    {
+        unwrapHdfsExceptions(() -> getHdfs().create(toHdfsPath(path)));
+        return path;
+    }
+
+    @Override
+    public Stream<Path> getDirectoryStream(Path path, String prefix, String suffix) throws IOException
+    {
+        return list(path).filter(pth -> pth.getFileName().toString().startsWith(prefix) && pth.getFileName().toString().endsWith(suffix));
+    }
+
     /**
      * Getter for filesystem object (lazy instantiation)
      *

hetu-filesystem-client/src/main/java/io/hetu/core/filesystem/HetuLocalFileSystemClient.java

@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.AccessDeniedException;
+import java.nio.file.FileStore;
 import java.nio.file.FileSystemException;
 import java.nio.file.Files;
 import java.nio.file.OpenOption;
@@ -29,6 +30,10 @@ import java.util.Collection;
 import java.util.LinkedList;
 import java.util.Locale;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+import static java.nio.file.Files.getFileStore;
+import static java.nio.file.Files.newDirectoryStream;
 
 /**
  * HetuFileSystemClient implementation for local file system
@@ -216,4 +221,37 @@ public class HetuLocalFileSystemClient
     public void close()
     {
     }
+
+    @Override
+    public long getTotalSpace(Path path) throws IOException
+    {
+        FileStore fileStore = getFileStore(path);
+        return fileStore.getTotalSpace();
+    }
+
+    @Override
+    public long getUsableSpace(Path path) throws IOException
+    {
+        FileStore fileStore = getFileStore(path);
+        return fileStore.getUsableSpace();
+    }
+
+    @Override
+    public Path createTemporaryFile(Path path, String prefix, String suffix) throws IOException
+    {
+        return Files.createTempFile(path, prefix, suffix);
+    }
+
+    @Override
+    public Path createFile(Path path) throws IOException
+    {
+        return Files.createFile(path);
+    }
+
+    @Override
+    public Stream<Path> getDirectoryStream(Path path, String prefix, String suffix) throws IOException
+    {
+        String glob = prefix + "*" + suffix;
+        return StreamSupport.stream(newDirectoryStream(path, glob).spliterator(), false);
+    }
 }

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/TestFileBasedLockOnHdfs.java

@@ -24,6 +24,7 @@ import org.testng.annotations.Test;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.concurrent.TimeUnit;
@@ -72,7 +73,7 @@ public class TestFileBasedLockOnHdfs
         FileBasedLock lock = new FileBasedLock(fs, testDir, 1000L,
                 FileBasedLock.DEFAULT_RETRY_INTERVAL, FileBasedLock.DEFAULT_REFRESH_RATE);
         OutputStream os = fs.newOutputStream(testDir.resolve(".lockFile"));
-        os.write("test".getBytes());
+        os.write("test".getBytes(StandardCharsets.UTF_8));
         os.close();
         assertTrue(lock.isLocked());
         Thread.sleep(1200L);
@@ -88,7 +89,7 @@ public class TestFileBasedLockOnHdfs
         FileBasedLock lock = new FileBasedLock(fs, testDir, 1000L,
                 FileBasedLock.DEFAULT_RETRY_INTERVAL, FileBasedLock.DEFAULT_REFRESH_RATE);
         OutputStream os = fs.newOutputStream(testDir.resolve(".lockInfo"));
-        os.write("test".getBytes());
+        os.write("test".getBytes(StandardCharsets.UTF_8));
         os.close();
         assertFalse(lock.acquiredLock());
         Thread.sleep(1200L);

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/TestFileBasedLockOnLocal.java

@@ -21,6 +21,7 @@ import org.testng.annotations.Test;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.concurrent.TimeUnit;
@@ -55,7 +56,7 @@ public class TestFileBasedLockOnLocal
         FileBasedLock lock = new FileBasedLock(fs, testDir, 1000L,
                 FileBasedLock.DEFAULT_RETRY_INTERVAL, FileBasedLock.DEFAULT_REFRESH_RATE);
         OutputStream os = fs.newOutputStream(testDir.resolve(".lockFile"));
-        os.write("test".getBytes());
+        os.write("test".getBytes(StandardCharsets.UTF_8));
         os.close();
         assertTrue(lock.isLocked());
         Thread.sleep(1200L);
@@ -71,7 +72,7 @@ public class TestFileBasedLockOnLocal
         FileBasedLock lock = new FileBasedLock(fs, testDir, 1000L,
                 FileBasedLock.DEFAULT_RETRY_INTERVAL, FileBasedLock.DEFAULT_REFRESH_RATE);
         OutputStream os = fs.newOutputStream(testDir.resolve(".lockInfo"));
-        os.write("test".getBytes());
+        os.write("test".getBytes(StandardCharsets.UTF_8));
         os.close();
         assertFalse(lock.acquiredLock());
         Thread.sleep(1200L);

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/TestHetuHdfsFileSystemClient.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
@@ -221,11 +222,11 @@ public class TestHetuHdfsFileSystemClient
         assertFalse(fs.exists(path));
         String content = "test content";
         OutputStream os = fs.newOutputStream(path);
-        os.write(content.getBytes());
+        os.write(content.getBytes(StandardCharsets.UTF_8));
         os.close();
         assertTrue(fs.exists(path));
         InputStream is = fs.newInputStream(path);
-        BufferedReader br = new BufferedReader(new InputStreamReader(is));
+        BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
         assertEquals(br.readLine(), content);
     }
 
@@ -249,10 +250,10 @@ public class TestHetuHdfsFileSystemClient
     {
         Path path = Paths.get(rootPath + "/testfileDup");
         OutputStream os = fs.newOutputStream(path);
-        os.write("foo".getBytes());
+        os.write("foo".getBytes(StandardCharsets.UTF_8));
         os.close();
         OutputStream os2 = fs.newOutputStream(path, CREATE_NEW);
-        os2.write("bar".getBytes());
+        os2.write("bar".getBytes(StandardCharsets.UTF_8));
         os2.close();
     }
 

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/TestHetuHdfsFileSystemClientOnLocal.java

@@ -24,6 +24,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
@@ -216,11 +217,11 @@ public class TestHetuHdfsFileSystemClientOnLocal
         assertFalse(fs.exists(path));
         String content = "test content";
         OutputStream os = fs.newOutputStream(path);
-        os.write(content.getBytes());
+        os.write(content.getBytes(StandardCharsets.UTF_8));
         os.close();
         assertTrue(fs.exists(path));
         InputStream is = fs.newInputStream(path);
-        BufferedReader br = new BufferedReader(new InputStreamReader(is));
+        BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
         assertEquals(br.readLine(), content);
     }
 
@@ -244,10 +245,10 @@ public class TestHetuHdfsFileSystemClientOnLocal
     {
         Path path = Paths.get(rootPath + "/testfileDup");
         OutputStream os = fs.newOutputStream(path);
-        os.write("foo".getBytes());
+        os.write("foo".getBytes(StandardCharsets.UTF_8));
         os.close();
         OutputStream os2 = fs.newOutputStream(path, CREATE_NEW);
-        os2.write("bar".getBytes());
+        os2.write("bar".getBytes(StandardCharsets.UTF_8));
         os2.close();
     }
 

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/TestHetuLocalFileSystemClient.java

@@ -27,6 +27,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.DirectoryNotEmptyException;
 import java.nio.file.FileAlreadyExistsException;
@@ -202,10 +203,10 @@ public class TestHetuLocalFileSystemClient
             assertTrue(testFile.delete());
         }
         OutputStream os = fs.newOutputStream(testFile.toPath());
-        os.write(content.getBytes());
+        os.write(content.getBytes(StandardCharsets.UTF_8));
         os.close();
         InputStream is = fs.newInputStream(testFile.toPath());
-        BufferedReader br = new BufferedReader(new InputStreamReader(is));
+        BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8));
         assertEquals(br.readLine(), content);
     }
 
@@ -229,10 +230,10 @@ public class TestHetuLocalFileSystemClient
     {
         Path path = tFolder.getRoot().toPath().resolve("testfileDup");
         OutputStream os = fs.newOutputStream(path);
-        os.write("foo".getBytes());
+        os.write("foo".getBytes(StandardCharsets.UTF_8));
         os.close();
         OutputStream os2 = fs.newOutputStream(path, CREATE_NEW);
-        os2.write("bar".getBytes());
+        os2.write("bar".getBytes(StandardCharsets.UTF_8));
         os2.close();
     }
 

hetu-filesystem-client/src/test/java/io/hetu/core/filesystem/utils/DockerizedHive.java

@@ -113,7 +113,7 @@ public class DockerizedHive
                             "Please refer to READMD.md for set up guide. ##",
                     testName));
             System.out.println("Error message:");
-            e.printStackTrace();
+            System.out.println(e.getStackTrace());
             return null;
         }
     }
@@ -165,8 +165,8 @@ public class DockerizedHive
     {
         // if the service is not up, this will throw an error
         this.hostPortProvider = hostPortProvider;
-        FileSystem fs = getFs();
-        fs.exists(new Path("/"));
+        FileSystem fileSystem = getFs();
+        fileSystem.exists(new Path("/"));
     }
 
     private void checkHostnameResolution(String hostname)
@@ -183,7 +183,14 @@ public class DockerizedHive
     private void checkFileExist(String path)
     {
         File file = new File(path);
-        Preconditions.checkArgument(file.exists(), file.getAbsolutePath() + " is not found");
+        String canonicalPath = "";
+        try {
+            canonicalPath = file.getCanonicalPath();
+        }
+        catch (IOException exception) {
+            // can be ignored
+        }
+        Preconditions.checkArgument(file.exists(), canonicalPath + " is not found");
     }
 
     public synchronized Configuration getHadoopConfiguration()
@@ -248,9 +255,9 @@ public class DockerizedHive
             DocumentBuilder documentBuilder = documentBuilderFactory.newDocumentBuilder();
             Document coreXml = documentBuilder.parse(coreIs);
             coreXml.getDocumentElement().normalize();
-            NodeList properties = coreXml.getElementsByTagName("property");
-            for (int i = 0; i < properties.getLength(); i++) {
-                Node node = properties.item(i);
+            NodeList localProperties = coreXml.getElementsByTagName("property");
+            for (int i = 0; i < localProperties.getLength(); i++) {
+                Node node = localProperties.item(i);
                 node.normalize();
                 if (node.getNodeType() == Node.ELEMENT_NODE) {
                     Element element = (Element) node;

hetu-function-namespace-managers/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.5.0-SNAPSHOT</version>
+        <version>1.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-function-namespace-managers</artifactId>

hetu-function-namespace-managers/src/main/java/io/hetu/core/plugin/functionnamespace/AbstractSqlInvokedFunctionNamespaceManager.java

@@ -89,11 +89,11 @@ public abstract class AbstractSqlInvokedFunctionNamespaceManager
                     @ParametersAreNonnullByDefault
                     public Collection<SqlInvokedFunction> load(QualifiedObjectName functionName)
                     {
-                        Collection<SqlInvokedFunction> functions = fetchFunctionsDirect(functionName);
-                        for (SqlInvokedFunction function : functions) {
+                        Collection<SqlInvokedFunction> sqlInvokedFunctions = fetchFunctionsDirect(functionName);
+                        for (SqlInvokedFunction function : sqlInvokedFunctions) {
                             metadataByHandle.put(function.getRequiredFunctionHandle(), sqlInvokedFunctionToMetadata(function));
                         }
-                        return functions;
+                        return sqlInvokedFunctions;
                     }
                 });
 
@@ -302,9 +302,9 @@ public abstract class AbstractSqlInvokedFunctionNamespaceManager
 
         public synchronized List<SqlInvokedFunction> loadAndGetFunctionsTransactional(QualifiedObjectName functionName)
         {
-            Collection<SqlInvokedFunction> functions = this.functions.computeIfAbsent(functionName, AbstractSqlInvokedFunctionNamespaceManager.this::fetchFunctions);
-            functionHandles.putAll(functions.stream().collect(toImmutableMap(SqlInvokedFunction::getFunctionId, SqlInvokedFunction::getRequiredFunctionHandle)));
-            return new ArrayList<>(functions);
+            Collection<SqlInvokedFunction> sqlInvokedFunctions = this.functions.computeIfAbsent(functionName, AbstractSqlInvokedFunctionNamespaceManager.this::fetchFunctions);
+            functionHandles.putAll(sqlInvokedFunctions.stream().collect(toImmutableMap(SqlInvokedFunction::getFunctionId, SqlInvokedFunction::getRequiredFunctionHandle)));
+            return new ArrayList<>(sqlInvokedFunctions);
         }
 
         public synchronized FunctionHandle getFunctionHandle(SqlFunctionId functionId)