hetu-carbondata/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>io.hetu.core</groupId>
     <artifactId>presto-root</artifactId>
-    <version>1.7.0-SNAPSHOT</version>
+    <version>1.4.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>hetu-carbondata</artifactId>

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataAutoVacuumThread.java

@@ -108,7 +108,8 @@ public class CarbondataAutoVacuumThread
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore)
         {
-            this(metastore, null);
+            this.metastore = metastore;
+            this.schemaName = null;
         }
 
         AutoVacuumScanTask(SemiTransactionalHiveMetastore metastore, String schemaName)
@@ -232,6 +233,7 @@ public class CarbondataAutoVacuumThread
     private void submitTaskScanning(CarbondataAutoVacuumThread instanceAutoVacuum, SemiTransactionalHiveMetastore metastore)
     {
         //trigger task to do scanning of tables
+        //instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore));
         if (enableTracingCleanupTask) {
             queuedTasks.add(instanceAutoVacuum.executorService.submit(new AutoVacuumScanTask(metastore)));
         }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataColumnVectorWrapper.java

@@ -73,17 +73,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putShorts(int rowId, int count, short value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putShort(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putShorts(inputRowId, count, value);
+            columnVector.putShorts(rowId, count, value);
         }
     }
 
@@ -98,17 +97,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putInt(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putInts(inputRowId, count, value);
+            columnVector.putInts(rowId, count, value);
         }
     }
 
@@ -123,17 +121,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putLongs(int rowId, int count, long value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putLong(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putLongs(inputRowId, count, value);
+            columnVector.putLongs(rowId, count, value);
         }
     }
 
@@ -148,12 +145,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDecimal(counter++, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -168,17 +164,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putDoubles(int rowId, int count, double value)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putDouble(counter++, value);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putDoubles(inputRowId, count, value);
+            columnVector.putDoubles(rowId, count, value);
         }
     }
 
@@ -201,12 +196,11 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByteArray(counter++, value);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -229,17 +223,16 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putNulls(int rowId, int count)
     {
-        int inputRowId = rowId;
         if (filteredRowsExist) {
             for (int i = 0; i < count; i++) {
-                if (!filteredRows[inputRowId]) {
+                if (!filteredRows[rowId]) {
                     columnVector.putNull(counter++);
                 }
-                inputRowId++;
+                rowId++;
             }
         }
         else {
-            columnVector.putNulls(inputRowId, count);
+            columnVector.putNulls(rowId, count);
         }
     }
 
@@ -326,72 +319,66 @@ public class CarbondataColumnVectorWrapper
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putFloat(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putShort(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putInt(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putLong(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putDouble(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = srcIndex; i < count; i++) {
-            if (!filteredRows[inputRowId]) {
+            if (!filteredRows[rowId]) {
                 columnVector.putByte(counter++, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataFileWriter.java

@@ -127,16 +127,15 @@ public class CarbondataFileWriter
     private boolean isInitDone;
     private boolean isCommitDone;
 
-    public CarbondataFileWriter(Path paramOutPutPath, List<String> inputColumnNames, Properties properties,
+    public CarbondataFileWriter(Path outPutPath, List<String> inputColumnNames, Properties properties,
                                 JobConf configuration, TypeManager typeManager, Optional<AcidOutputFormat.Options> acidOptions,
                                 Optional<HiveACIDWriteType> acidWriteType, OptionalInt taskId) throws SerDeException
     {
-        Path localOutPutPath = paramOutPutPath;
+        this.outPutPath = requireNonNull(outPutPath, "path is null");
-        this.outPutPath = requireNonNull(localOutPutPath, "path is null");
         // in table creation this can be null
         if (null != properties.getProperty("location")) {
             this.outPutPath = new Path(properties.getProperty("location"));
-            localOutPutPath = new Path(properties.getProperty("location"));
+            outPutPath = new Path(properties.getProperty("location"));
         }
         this.configuration = requireNonNull(configuration, "conf is null");
         this.properties = requireNonNull(properties, "Properties is null");
@@ -212,7 +211,7 @@ public class CarbondataFileWriter
                     Object writer =
                             Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
                     recordWriter = ((MapredCarbonOutputFormat<?>) writer)
-                            .getHiveRecordWriter(this.configuration, localOutPutPath, Text.class, compress,
+                            .getHiveRecordWriter(this.configuration, outPutPath, Text.class, compress,
                                     properties, Reporter.NULL);
                 }
 
@@ -227,25 +226,25 @@ public class CarbondataFileWriter
 
     private FileSinkOperator.RecordWriter getHiveWriter(String segmentId, long taskNo) throws Exception
     {
-        Path finalOutPutPath = this.outPutPath;
+        Path outPutPath = this.outPutPath;
-        Properties finalProperties = this.properties;
+        Properties properties = this.properties;
-        JobConf finalConfiguration = this.configuration;
+        JobConf configuration = this.configuration;
-        boolean compress = HiveConf.getBoolVar(finalConfiguration, COMPRESSRESULT);
+        boolean compress = HiveConf.getBoolVar(configuration, COMPRESSRESULT);
 
-        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(finalProperties, finalConfiguration);
+        CarbonLoadModel carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(properties, configuration);
         carbonLoadModel.setSegmentId(segmentId);
         carbonLoadModel.setTaskNo(String.valueOf(taskNo));
         carbonLoadModel.setFactTimeStamp(Long.parseLong(txnTimeStamp));
         carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
 
-        CarbonTableOutputFormat.setLoadModel(finalConfiguration, carbonLoadModel);
+        CarbonTableOutputFormat.setLoadModel(configuration, carbonLoadModel);
         this.configuration.set(CarbondataConstants.TaskId, getTaskAttemptId(String.valueOf(taskNo)));
 
         Object writer =
                 Class.forName(MapredCarbonOutputFormat.class.getName()).getConstructor().newInstance();
         return ((MapredCarbonOutputFormat<?>) writer)
-                .getHiveRecordWriter(finalConfiguration, finalOutPutPath, Text.class, compress,
+                .getHiveRecordWriter(configuration, outPutPath, Text.class, compress,
-                        finalProperties, Reporter.NULL);
+                        properties, Reporter.NULL);
     }
 
     @Override
@@ -286,7 +285,7 @@ public class CarbondataFileWriter
 
     public void appendRow(Page dataPage, int position)
     {
-        FileSinkOperator.RecordWriter finalRecordWriter = null;
+        FileSinkOperator.RecordWriter recordWriter = null;
         if (HiveACIDWriteType.isUpdateOrDelete(acidWriteType)) {
             try {
                 DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
@@ -335,7 +334,7 @@ public class CarbondataFileWriter
                     return;
                 }
 
-                finalRecordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
+                recordWriter = segmentRecordWriterMap.computeIfAbsent(segmentId, v ->
                 {
                     try {
                         return getHiveWriter(segmentId, CarbonUpdateUtil.getLatestTaskIdForSegment(new Segment(segmentId), tablePath) + 1);
@@ -352,7 +351,7 @@ public class CarbondataFileWriter
             }
         }
         else {
-            finalRecordWriter = this.recordWriter;
+            recordWriter = this.recordWriter;
         }
 
         for (int field = 0; field < fieldCount; field++) {
@@ -366,8 +365,8 @@ public class CarbondataFileWriter
         }
 
         try {
-            if (finalRecordWriter != null) {
+            if (recordWriter != null) {
-                finalRecordWriter.write(serDe.serialize(row, tableInspector));
+                recordWriter.write(serDe.serialize(row, tableInspector));
             }
         }
         catch (SerDeException | IOException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHandleResolver.java

@@ -48,7 +48,6 @@ public class CarbondataHandleResolver
         return CarbonDeleteAsInsertTableHandle.class;
     }
 
-    @Override
     public Class<? extends ConnectorOutputTableHandle> getOutputTableHandleClass()
     {
         return CarbondataOutputTableHandle.class;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuFilterUtil.java

@@ -317,11 +317,11 @@ public class CarbondataHetuFilterUtil
             if (rawData instanceof Slice) {
                 String value = ((Slice) rawData).toStringUtf8();
                 if (type.getTypeInfo() instanceof CharTypeInfo) {
-                    StringBuilder padding = new StringBuilder();
+                    String padding = "";
                     int paddedLength = ((CharTypeInfo) type.getTypeInfo()).getLength();
                     int truncatedLength = value.length();
                     for (int i = 0; i < paddedLength - truncatedLength; i++) {
-                        padding.append(" ");
+                        padding += " ";
                     }
                     return value + padding;
                 }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataHetuOutputFormat.java

@@ -103,6 +103,7 @@ public class CarbondataHetuOutputFormat<T>
         OutputCommitter carbonOutputCommitter = super.getOutputCommitter(context);
         JobContextImpl jobContext = new JobContextImpl(jc, new JobID());
         carbonOutputCommitter.setupJob(jobContext);
+        CarbonLoadModel updatedCarbonLoadModel = CarbonTableOutputFormat.getLoadModel(jc);
         org.apache.hadoop.mapreduce.RecordWriter re = super.getRecordWriter(context);
         return new FileSinkOperator.RecordWriter()
         {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataLocationService.java

@@ -80,6 +80,8 @@ public class CarbondataLocationService
     {
         // TODO: check and make it compatible for cloud scenario
 
+        HdfsEnvironment.HdfsContext context =
+                new HdfsEnvironment.HdfsContext(session, table.getDatabaseName(), table.getTableName());
         Path targetPath = new Path(table.getStorage().getLocation());
 
         return new LocationHandle(targetPath, targetPath, true,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadata.java

@@ -306,13 +306,13 @@ public class CarbondataMetadata
 
     private void setupCommitWriter(Properties hiveSchema, Path outputPath, Configuration initialConfiguration, boolean isOverwrite) throws PrestoException
     {
-        CarbonLoadModel finalCarbonLoadModel;
+        CarbonLoadModel carbonLoadModel;
         TaskAttemptID taskAttemptID = TaskAttemptID.forName(initialConfiguration.get("mapred.task.id"));
         try {
             ThreadLocalSessionInfo.setConfigurationToCurrentThread(initialConfiguration);
-            finalCarbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
+            carbonLoadModel = HiveCarbonUtil.getCarbonLoadModel(hiveSchema, initialConfiguration);
-            finalCarbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
+            carbonLoadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + ",force");
-            CarbonTableOutputFormat.setLoadModel(initialConfiguration, finalCarbonLoadModel);
+            CarbonTableOutputFormat.setLoadModel(initialConfiguration, carbonLoadModel);
         }
         catch (IOException ex) {
             LOG.error("Error while creating carbon load model", ex);
@@ -360,13 +360,13 @@ public class CarbondataMetadata
         this.user = session.getUser();
         return hdfsEnvironment.doAs(user, () -> {
             SchemaTableName tableName = parent.getSchemaTableName();
-            Optional<Table> finalTable =
+            Optional<Table> table =
                     metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-            if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+            if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
                 throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
             }
 
-            this.table = finalTable;
+            this.table = table;
             Path outputPath =
                     new Path(parent.getLocationHandle().getJsonSerializableTargetPath());
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -382,7 +382,7 @@ public class CarbondataMetadata
             }
 
             /* Create committer object */
-            setupCommitWriter(finalTable, outputPath, initialConfiguration, isOverwrite);
+            setupCommitWriter(table, outputPath, initialConfiguration, isOverwrite);
 
             return new CarbondataInsertTableHandle(parent.getSchemaName(),
                     parent.getTableName(),
@@ -416,13 +416,13 @@ public class CarbondataMetadata
         currentState = State.UPDATE;
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -430,8 +430,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -470,13 +470,13 @@ public class CarbondataMetadata
         HiveInsertTableHandle parent = super.beginInsert(session, tableHandle);
         List<HiveColumnHandle> inputColumns = parent.getInputColumns().stream().filter(HiveColumnHandle::isRequired).collect(toList());
         SchemaTableName tableName = parent.getSchemaTableName();
-        Optional<Table> finalTable =
+        Optional<Table> table =
                 this.metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (finalTable.isPresent() && finalTable.get().getPartitionColumns().size() > 0) {
+        if (table.isPresent() && table.get().getPartitionColumns().size() > 0) {
             throw new PrestoException(NOT_SUPPORTED, "Operations on Partitioned CarbonTables is not supported");
         }
 
-        this.table = finalTable;
+        this.table = table;
         this.user = session.getUser();
         hdfsEnvironment.doAs(user, () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
@@ -484,8 +484,8 @@ public class CarbondataMetadata
                             new HdfsEnvironment.HdfsContext(session, parent.getSchemaName(),
                                     parent.getTableName()),
                             new Path(parent.getLocationHandle().getJsonSerializableWritePath())));
-            Properties schema = MetastoreUtil.getHiveSchema(finalTable.get());
+            Properties schema = MetastoreUtil.getHiveSchema(table.get());
-            schema.setProperty("tablePath", finalTable.get().getStorage().getLocation());
+            schema.setProperty("tablePath", table.get().getStorage().getLocation());
             carbonTable = getCarbonTable(parent.getSchemaName(),
                     parent.getTableName(),
                     schema,
@@ -643,7 +643,7 @@ public class CarbondataMetadata
 
         return hdfsEnvironment.doAs(session.getUser(), () -> {
             Properties hiveSchema = MetastoreUtil.getHiveSchema(this.table.get());
-            CarbonTable finalCarbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
+            CarbonTable carbonTable = getCarbonTable(carbondataVacuumTableHandle.getSchemaName(),
                     carbondataVacuumTableHandle.getTableName(),
                     hiveSchema,
                     initialConfiguration);
@@ -705,7 +705,7 @@ public class CarbondataMetadata
                     SegmentFileStore.mergeSegmentFiles(readPath, segmentFileName, CarbonTablePath.getSegmentFilesLocation(carbonLoadModel.getTablePath()));
                     String source;
                     for (String currPartitionName : partitionNames) {
-                        source = finalCarbonTable.getTablePath() + "/" + currPartitionName;
+                        source = carbonTable.getTablePath() + "/" + currPartitionName;
                         moveFromTempFolder(source + "/" + carbonLoadModel.getSegmentId() + "_" + timeStamp + ".tmp", source);
                     }
                     segmentFilesToBeUpdatedLatest.add(new Segment(carbonLoadModel.getSegmentId(), segmentFileName));
@@ -719,7 +719,7 @@ public class CarbondataMetadata
                 for (CarbondataSegmentInfoUtil segmentInfo : newMergedSegmentInfoUtilList) {
                     String mergedLoadNumber = segmentInfo.getDestinationSegment();
                     try {
-                        String segmentFileName = SegmentFileStore.writeSegmentFile(finalCarbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
+                        String segmentFileName = SegmentFileStore.writeSegmentFile(carbonTable, mergedLoadNumber, String.valueOf(carbonLoadModel.getFactTimeStamp()));
                     }
                     catch (IOException e) {
                         throw new PrestoException(GENERIC_INTERNAL_ERROR, "Failed while merging segment files", e);
@@ -900,9 +900,9 @@ public class CarbondataMetadata
     private LocationHandle getCarbonDataTableCreationPath(ConnectorSession session, ConnectorTableMetadata tableMetadata, HiveWriteUtils.OpertionType opertionType) throws PrestoException
     {
         Path targetPath = null;
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String schemaName = schemaTableName.getSchemaName();
-        String tableName = finalSchemaTableName.getTableName();
+        String tableName = schemaTableName.getTableName();
         Optional<String> location = getCarbondataLocation(tableMetadata.getProperties());
         LocationHandle locationHandle;
         FileSystem fileSystem;
@@ -914,32 +914,32 @@ public class CarbondataMetadata
                     throw new PrestoException(NOT_SUPPORTED, format("Setting %s property is not allowed", LOCATION_PROPERTY));
                 }
                 /* if path not having prefix with filesystem type, than we will take fileSystem type from core-site.xml using below methods */
-                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, finalSchemaName), new Path(location.get()));
+                fileSystem = hdfsEnvironment.getFileSystem(new HdfsEnvironment.HdfsContext(session, schemaName), new Path(location.get()));
                 targetLocation = fileSystem.getFileStatus(new Path(location.get())).getPath().toString();
-                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, finalSchemaName, tableName), targetLocation, false);
+                targetPath = getPath(new HdfsEnvironment.HdfsContext(session, schemaName, tableName), targetLocation, false);
             }
             else {
-                updateEmptyCarbondataTableStorePath(session, finalSchemaName);
+                updateEmptyCarbondataTableStorePath(session, schemaName);
                 targetLocation = carbondataTableStore;
-                targetLocation = targetLocation + File.separator + finalSchemaName + File.separator + tableName;
+                targetLocation = targetLocation + File.separator + schemaName + File.separator + tableName;
                 targetPath = new Path(targetLocation);
             }
         }
         catch (IllegalArgumentException | IOException e) {
             throw new PrestoException(NOT_SUPPORTED, format("Error %s store path %s ", e.getMessage(), targetLocation));
         }
-        locationHandle = locationService.forNewTable(metastore, session, finalSchemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
+        locationHandle = locationService.forNewTable(metastore, session, schemaName, tableName, Optional.empty(), Optional.of(targetPath), opertionType);
         return locationHandle;
     }
 
     @Override
     public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting)
     {
-        SchemaTableName localSchemaTableName = tableMetadata.getTable();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String localSchemaName = localSchemaTableName.getSchemaName();
+        String schemaName = schemaTableName.getSchemaName();
-        String tableName = localSchemaTableName.getTableName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = localSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE;
         List<String> partitionedBy = new ArrayList<String>();
         List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
@@ -947,29 +947,29 @@ public class CarbondataMetadata
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
 
-        metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
         Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
 
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                        new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName),
+                        new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                         new Path(locationHandle.getJsonSerializableTargetPath())));
 
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             try {
                 Map<String, String> serdeParameters = initSerDeProperties(tableName);
-                Table localTable = buildTableObject(
+                Table table = buildTableObject(
                         session.getQueryId(),
-                        localSchemaName,
+                        schemaName,
                         tableName,
                         session.getUser(),
                         columnHandles,
@@ -981,11 +981,11 @@ public class CarbondataMetadata
                         true, // carbon table is set as external table
                         prestoVersion,
                         serdeParameters);
-                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
+                PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(table.getOwner());
-                HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
+                HiveBasicStatistics basicStatistics = table.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
                 metastore.createTable(
                         session,
-                        localTable,
+                        table,
                         principalPrivileges,
                         Optional.empty(),
                         ignoreExisting,
@@ -1092,8 +1092,8 @@ public class CarbondataMetadata
     public CarbondataTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName)
     {
         requireNonNull(tableName, "tableName is null");
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent()) {
+        if (!table.isPresent()) {
             return null;
         }
 
@@ -1102,14 +1102,14 @@ public class CarbondataMetadata
             throw new PrestoException(HiveErrorCode.HIVE_INVALID_METADATA, "Unexpected table present in Hive metastore: " + tableName);
         }
 
-        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(finalTable.get()), finalTable.get().getParameters());
+        MetastoreUtil.verifyOnline(tableName, Optional.empty(), MetastoreUtil.getProtectMode(table.get()), table.get().getParameters());
 
         return new CarbondataTableHandle(
                 tableName.getSchemaName(),
                 tableName.getTableName(),
-                finalTable.get().getParameters(),
+                table.get().getParameters(),
-                getPartitionKeyColumnHandles(finalTable.get()),
+                getPartitionKeyColumnHandles(table.get()),
-                HiveBucketing.getHiveBucketHandle(finalTable.get()));
+                HiveBucketing.getHiveBucketHandle(table.get()));
     }
 
     private Optional<ConnectorOutputMetadata> finishUpdateAndDelete(ConnectorSession session,
@@ -1133,12 +1133,12 @@ public class CarbondataMetadata
 
         hdfsEnvironment.doAs(user, () -> {
             if (blockUpdateDetailsList.size() > 0) {
-                CarbonTable finalCarbonTable = getCarbonTable(tableHandle.getSchemaName(),
+                CarbonTable carbonTable = getCarbonTable(tableHandle.getSchemaName(),
                         tableHandle.getTableName(),
                         MetastoreUtil.getHiveSchema(table.get()),
                         initialConfiguration);
 
-                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(finalCarbonTable);
+                SegmentUpdateStatusManager statusManager = new SegmentUpdateStatusManager(carbonTable);
                 SegmentUpdateDetails[] segementDetailsList = statusManager.getUpdateStatusDetails();
                 for (SegmentUpdateDetails segementDetails : segementDetailsList) {
                     segementDetails.getDeletedRowsInBlock();
@@ -1179,26 +1179,26 @@ public class CarbondataMetadata
                                             List<HiveColumnHandle> columnHandles,
                                             Map<String, String> tableProperties)
     {
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String schemaName = schemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        String tableName = schemaTableName.getTableName();
 
         partitionedBy.addAll(CarbondataTableProperties.getPartitionedBy(tableMetadata.getProperties()));
         sortBy.addAll(CarbondataTableProperties.getSortedBy(tableMetadata.getProperties()));
         Optional<HiveBucketProperty> bucketProperty = Optional.empty();
         columnHandles.addAll(getColumnHandles(tableMetadata, ImmutableSet.copyOf(partitionedBy), typeTranslator));
-        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName)));
+        tableProperties.putAll(getEmptyTableProperties(tableMetadata, bucketProperty, new HdfsEnvironment.HdfsContext(session, schemaName, tableName)));
     }
 
     @Override
     public CarbondataOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional<ConnectorNewTableLayout> layout)
     {
         // get the root directory for the database
-        SchemaTableName finalSchemaTableName = tableMetadata.getTable();
+        SchemaTableName schemaTableName = tableMetadata.getTable();
-        String finalSchemaName = finalSchemaTableName.getSchemaName();
+        String schemaName = schemaTableName.getSchemaName();
-        String finalTableName = finalSchemaTableName.getTableName();
+        String tableName = schemaTableName.getTableName();
         this.user = session.getUser();
-        this.schemaName = finalSchemaName;
+        this.schemaName = schemaName;
         currentState = State.CREATE_TABLE_AS;
 
         List<String> partitionedBy = new ArrayList<String>();
@@ -1206,7 +1206,7 @@ public class CarbondataMetadata
         List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
         Map<String, String> tableProperties = new HashMap<String, String>();
         getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
-        metastore.getDatabase(finalSchemaName).orElseThrow(() -> new SchemaNotFoundException(finalSchemaName));
+        metastore.getDatabase(schemaName).orElseThrow(() -> new SchemaNotFoundException(schemaName));
 
         // to avoid type mismatch between HiveStorageFormat & Carbondata StorageFormat this hack no option
         HiveStorageFormat tableStorageFormat = HiveStorageFormat.valueOf("CARBON");
@@ -1222,29 +1222,29 @@ public class CarbondataMetadata
         // it will get final path to create carbon table
         LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE_AS);
         Path targetPath = locationService.getTableWriteInfo(locationHandle, false).getTargetPath();
-        AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
+        AbsoluteTableIdentifier absoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(),
-                new CarbonTableIdentifier(finalSchemaName, finalTableName, UUID.randomUUID().toString()));
+                new CarbonTableIdentifier(schemaName, tableName, UUID.randomUUID().toString()));
 
         hdfsEnvironment.doAs(session.getUser(), () -> {
             initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(
-                                new HdfsEnvironment.HdfsContext(session, finalSchemaName, finalTableName),
+                                new HdfsEnvironment.HdfsContext(session, schemaName, tableName),
                                 new Path(locationHandle.getJsonSerializableTargetPath())));
             // Create Carbondata metadata folder and Schema file
-            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy,
+            CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, absoluteTableIdentifier, partitionedBy,
                     sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
 
             this.tableStorageLocation = Optional.of(targetPath.toString());
             Path outputPath = new Path(locationHandle.getJsonSerializableTargetPath());
-            Properties schema = readSchemaForCarbon(finalSchemaName, finalTableName, targetPath, columnHandles, partitionColumns);
+            Properties schema = readSchemaForCarbon(schemaName, tableName, targetPath, columnHandles, partitionColumns);
             // Create committer object
             setupCommitWriter(schema, outputPath, initialConfiguration, false);
         });
         try {
             CarbondataOutputTableHandle result = new CarbondataOutputTableHandle(
-                    finalSchemaName,
+                    schemaName,
-                    finalTableName,
+                    tableName,
                     columnHandles,
-                    metastore.generatePageSinkMetadata(new HiveIdentity(session), finalSchemaTableName),
+                    metastore.generatePageSinkMetadata(new HiveIdentity(session), schemaTableName),
                     locationHandle,
                     tableStorageFormat,
                     partitionStorageFormat,
@@ -1255,7 +1255,7 @@ public class CarbondataMetadata
                     EncodedLoadModel, jobContext.getConfiguration().get(LOAD_MODEL)));
 
             LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
-            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), finalSchemaTableName);
+            metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), schemaTableName);
             return result;
         }
         catch (RuntimeException ex) {
@@ -1386,7 +1386,7 @@ public class CarbondataMetadata
         List<Segment> segmentFilesToBeUpdated = blockUpdateDetailsList.stream()
                 .map(SegmentUpdateDetails::getSegmentName)
                 .map(Segment::new).collect(Collectors.toList());
-        List<Segment> finalSegmentFilesToBeUpdatedLatest = new ArrayList<>();
+        List<Segment> segmentFilesToBeUpdatedLatest = new ArrayList<>();
         List<Segment> segmentFilesToBeDeleted = blockUpdateDetailsList.stream()
                 .filter(segmentUpdateDetails -> segmentUpdateDetails.getSegmentStatus() != null &&
                         segmentUpdateDetails.getSegmentStatus().equals(SegmentStatus.MARKED_FOR_DELETE))
@@ -1396,12 +1396,12 @@ public class CarbondataMetadata
         for (Segment segment : segmentFilesToBeUpdated) {
             String file =
                     SegmentFileStore.writeSegmentFile(carbonTable, segment.getSegmentNo(), timeStamp.toString());
-            finalSegmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
+            segmentFilesToBeUpdatedLatest.add(new Segment(segment.getSegmentNo(), file));
         }
         if (!(updateSegmentStatusSuccess &&
                 CarbonUpdateUtil.updateTableMetadataStatus(new HashSet<>(segmentFilesToBeUpdated),
                         carbonTable, timeStamp.toString(), true, segmentFilesToBeDeleted,
-                        finalSegmentFilesToBeUpdatedLatest, ""))) {
+                        segmentFilesToBeUpdatedLatest, ""))) {
             CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timeStamp.toString());
         }
     }
@@ -1463,10 +1463,11 @@ public class CarbondataMetadata
                 Properties hiveschema = MetastoreUtil.getHiveSchema(table);
                 Configuration configuration = jobContext.getConfiguration();
                 configuration.set(SET_OVERWRITE, "false");
-                CarbonLoadModel loadModel = HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
+                CarbonLoadModel carbonLoadModel =
-                LoadMetadataDetails loadMetadataDetails = loadModel.getCurrentLoadMetadataDetail();
+                        HiveCarbonUtil.getCarbonLoadModel(hiveschema, configuration);
-                loadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                LoadMetadataDetails loadMetadataDetails = carbonLoadModel.getCurrentLoadMetadataDetail();
-                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, loadModel, false, true);
+                carbonLoadModel.setSegmentId(loadMetadataDetails.getLoadName());
+                CarbonLoaderUtil.recordNewLoadMetadata(loadMetadataDetails, carbonLoadModel, false, true);
             }
             catch (IOException e) {
                 LOG.error("Error occurred while committing the insert job.", e);
@@ -1553,14 +1554,14 @@ public class CarbondataMetadata
         try {
             hdfsEnvironment.doAs(session.getUser(), () -> {
                 metastore.dropTable(session, handle.getSchemaName(), handle.getTableName());
-                Configuration finalInitialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
+                Configuration initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment
                         .getConfiguration(new HdfsEnvironment.HdfsContext(session, handle.getSchemaName(),
                                 handle.getTableName()), new Path(this.tableStorageLocation.get())));
 
                 Properties schema = MetastoreUtil.getHiveSchema(target.get());
                 schema.setProperty("tablePath", this.tableStorageLocation.get());
                 this.carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(),
-                        schema, finalInitialConfiguration);
+                        schema, initialConfiguration);
                 takeLocks(State.DROP_TABLE);
                 AbsoluteTableIdentifier identifier = this.carbonTable.getAbsoluteTableIdentifier();
                 if (SegmentStatusManager.isLoadInProgressInTable(carbonTable)) {
@@ -1569,7 +1570,7 @@ public class CarbondataMetadata
                 try {
                     //Simultaneous case after acquiring locks we should check table exist.
                     //if table is not there clean the lock folders
-                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, finalInitialConfiguration);
+                    carbonTable = getCarbonTable(handle.getSchemaName(), handle.getTableName(), schema, initialConfiguration);
                 }//CarbonFileException
                 catch (RuntimeException e) {
                     try {
@@ -1866,8 +1867,8 @@ public class CarbondataMetadata
     {
         String tableName = absoluteTableIdentifier.getTableName();
         String databaseName = absoluteTableIdentifier.getDatabaseName();
-        TableInfo finalTableInfo = carbonTable.getTableInfo();
+        TableInfo tableInfo = carbonTable.getTableInfo();
-        List<SchemaEvolutionEntry> evolutionEntryList = finalTableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
+        List<SchemaEvolutionEntry> evolutionEntryList = tableInfo.getFactTable().getSchemaEvolution().getSchemaEvolutionEntryList();
         Long updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).getTimeStamp();
         LOG.info("Reverting changes for " + databaseName + "." + tableName);
         List<ColumnSchema> addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).getAdded();
@@ -1879,7 +1880,7 @@ public class CarbondataMetadata
                     break;
                 }
                 case DROP_COLUMN: {
-                    finalTableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
+                    tableInfo.getFactTable().getListOfColumns().forEach(cols -> removedSchemas.forEach(removedCols -> {
                         if (cols.isInvisible() && removedCols.getColumnUniqueId().equals(cols.getColumnUniqueId())) {
                             cols.setInvisible(false);
                         }
@@ -2026,38 +2027,38 @@ public class CarbondataMetadata
     @Override
     protected ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName)
     {
-        Optional<Table> finalTable = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
+        Optional<Table> table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName());
-        if (!finalTable.isPresent() || finalTable.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
+        if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
             throw new TableNotFoundException(tableName);
         }
 
-        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(finalTable.get(), typeManager);
+        Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
         ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             columns.add(metadataGetter.apply(columnHandle));
         }
 
         // External location property
         ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
-        properties.put(LOCATION_PROPERTY, finalTable.get().getStorage().getLocation());
+        properties.put(LOCATION_PROPERTY, table.get().getStorage().getLocation());
 
         // Storage format property
         properties.put(HiveTableProperties.STORAGE_FORMAT_PROPERTY, CarbondataStorageFormat.CARBON);
 
         // Partitioning property
-        List<String> partitionedBy = finalTable.get().getPartitionColumns().stream()
+        List<String> partitionedBy = table.get().getPartitionColumns().stream()
                 .map(Column::getName)
                 .collect(toList());
         if (!partitionedBy.isEmpty()) {
             properties.put(HiveTableProperties.PARTITIONED_BY_PROPERTY, partitionedBy);
         }
 
-        Optional<String> comment = Optional.ofNullable(finalTable.get().getParameters().get(TABLE_COMMENT));
+        Optional<String> comment = Optional.ofNullable(table.get().getParameters().get(TABLE_COMMENT));
 
         // add partitioned columns into immutableColumns
         ImmutableList.Builder<ColumnMetadata> immutableColumns = ImmutableList.builder();
 
-        for (HiveColumnHandle columnHandle : hiveColumnHandles(finalTable.get())) {
+        for (HiveColumnHandle columnHandle : hiveColumnHandles(table.get())) {
             if (columnHandle.getColumnType().equals(HiveColumnHandle.ColumnType.PARTITION_KEY)) {
                 immutableColumns.add(metadataGetter.apply(columnHandle));
             }

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataMetadataFactory.java

@@ -191,7 +191,7 @@ public class CarbondataMetadataFactory
     @Override
     public HiveMetadata get()
     {
-        SemiTransactionalHiveMetastore semiTransactionalHiveMetastore =
+        SemiTransactionalHiveMetastore metastore =
                 new SemiTransactionalHiveMetastore(this.hdfsEnvironment,
                         CachingHiveMetastore.memoizeMetastore(this.metastore, this.perTransactionCacheMaximumSize),
                         this.renameExecution,
@@ -200,7 +200,7 @@ public class CarbondataMetadataFactory
                         this.hiveTransactionHeartbeatInterval,
                         this.heartbeatService, hiveMetastoreClientService, hmsWriteBatchSize);
 
-        return new CarbondataMetadata(semiTransactionalHiveMetastore,
+        return new CarbondataMetadata(metastore,
                 this.hdfsEnvironment,
                 this.partitionManager,
                 this.writesToNonManagedTablesEnabled,
@@ -212,8 +212,8 @@ public class CarbondataMetadataFactory
                 this.segmentInfoCodec,
                 this.typeTranslator,
                 this.hetuVersion,
-                new MetastoreHiveStatisticsProvider(semiTransactionalHiveMetastore, statsCache, samplePartitionCache),
+                new MetastoreHiveStatisticsProvider(metastore, statsCache, samplePartitionCache),
-                this.accessControlMetadataFactory.create(semiTransactionalHiveMetastore),
+                this.accessControlMetadataFactory.create(metastore),
                 carbondataTableReader,
                 this.carbondataTableStore,
                 this.carbondataMajorVacuumSegmentSize,

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSink.java

@@ -167,9 +167,9 @@ public class CarbondataPageSink
     {
         //set flag here if called and change finish accordingly.
         isCompactionCalled = true;
-        HdfsEnvironment finalHdfsEnvironment = connectorPageSource.getHdfsEnvironment();
+        HdfsEnvironment hdfsEnvironment = connectorPageSource.getHdfsEnvironment();
 
-        finalHdfsEnvironment.doAs(session.getUser(), () -> {
+        hdfsEnvironment.doAs(session.getUser(), () -> {
             try {
                 // Worker part: each thread to run this code
                 boolean mergeStatus = false;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSinkProvider.java

@@ -163,7 +163,6 @@ public class CarbondataPageSinkProvider
                 ImmutableMap.of(), handle.getAdditionalConf(), false);
     }
 
-    @Override
     public ConnectorPageSink createPageSink(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorOutputTableHandle tableHandle)
     {
         CarbondataOutputTableHandle handle = (CarbondataOutputTableHandle) tableHandle;

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPageSource.java

@@ -232,15 +232,15 @@ public class CarbondataPageSource
             nanoStart = System.nanoTime();
         }
         CarbondataVectorBatch columnarBatch = null;
-        int columnBatchSize = 0;
+        int batchSize = 0;
         try {
             batchId++;
             if (vectorReader.nextKeyValue()) {
                 Object vectorBatch = vectorReader.getCurrentValue();
                 if (vectorBatch instanceof CarbondataVectorBatch) {
                     columnarBatch = (CarbondataVectorBatch) vectorBatch;
-                    columnBatchSize = columnarBatch.numRows();
+                    batchSize = columnarBatch.numRows();
-                    if (columnBatchSize == 0) {
+                    if (batchSize == 0) {
                         close();
                         return null;
                     }
@@ -256,9 +256,9 @@ public class CarbondataPageSource
 
             Block[] blocks = new Block[columnHandles.size()];
             for (int column = 0; column < blocks.length; column++) {
-                blocks[column] = new LazyBlock(columnBatchSize, new CarbondataBlockLoader(column));
+                blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column));
             }
-            Page page = new Page(columnBatchSize, blocks);
+            Page page = new Page(batchSize, blocks);
             return page;
         }
         catch (PrestoException e) {

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataTableProperties.java

@@ -90,17 +90,16 @@ public class CarbondataTableProperties
 
     private static SortingColumn sortingColumnFromString(String name)
     {
-        String finalName = name;
         SortingColumn.Order order = SortingColumn.Order.ASCENDING;
         String lower = name.toUpperCase(ENGLISH);
         if (lower.endsWith(" ASC")) {
-            finalName = name.substring(0, name.length() - 4).trim();
+            name = name.substring(0, name.length() - 4).trim();
         }
         else if (lower.endsWith(" DESC")) {
-            finalName = name.substring(0, name.length() - 5).trim();
+            name = name.substring(0, name.length() - 5).trim();
             order = SortingColumn.Order.DESCENDING;
         }
-        return new SortingColumn(finalName, order);
+        return new SortingColumn(name, order);
     }
 
     private static String sortingColumnToString(SortingColumn column)

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataWriterFactory.java

@@ -133,7 +133,6 @@ public class CarbondataWriterFactory
     {
     }
 
-    @Override
     protected void setAdditionalSchemaProperties(Properties schema)
     {
         schema.setProperty(META_TABLE_LOCATION, locationService.getTableWriteInfo(locationHandle, false).getTargetPath().toString());

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/ColumnarVectorWrapperDirect.java

@@ -143,15 +143,14 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDecimal(inputRowId, value, precision);
+                columnVector.putDecimal(rowId, value, precision);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
@@ -186,9 +185,8 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putByteArray(int rowId, int count, byte[] value)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            columnVector.putByteArray(inputRowId++, value);
+            columnVector.putByteArray(rowId++, value);
         }
     }
 
@@ -305,90 +303,84 @@ class ColumnarVectorWrapperDirect
     @Override
     public void putFloats(int rowId, int count, float[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putFloat(inputRowId, src[i]);
+                columnVector.putFloat(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putShorts(int rowId, int count, short[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putShort(inputRowId, src[i]);
+                columnVector.putShort(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putInts(int rowId, int count, int[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putInt(inputRowId, src[i]);
+                columnVector.putInt(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putLongs(int rowId, int count, long[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putLong(inputRowId, src[i]);
+                columnVector.putLong(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putDoubles(int rowId, int count, double[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putDouble(inputRowId, src[i]);
+                columnVector.putDouble(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int inputRowId = rowId;
         for (int i = 0; i < count; i++) {
-            if (nullBitSet.get(inputRowId)) {
+            if (nullBitSet.get(rowId)) {
-                columnVector.putNull(inputRowId);
+                columnVector.putNull(rowId);
             }
             else {
-                columnVector.putByte(inputRowId, src[i]);
+                columnVector.putByte(rowId, src[i]);
             }
-            inputRowId++;
+            rowId++;
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/BooleanStreamReader.java

@@ -58,9 +58,8 @@ public class BooleanStreamReader
     @Override
     public void putBytes(int rowId, int count, byte[] src, int srcIndex)
     {
-        int srcIdx = srcIndex;
         for (int i = 0; i < count; i++) {
-            type.writeBoolean(builder, src[srcIdx++] == 1);
+            type.writeBoolean(builder, src[srcIndex++] == 1);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/DecimalSliceStreamReader.java

@@ -76,9 +76,8 @@ public class DecimalSliceStreamReader
     @Override
     public void putDecimals(int rowId, int count, BigDecimal value, int precision)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putDecimal(id++, value, precision);
+            putDecimal(rowId++, value, precision);
         }
     }
 

hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/readers/IntegerStreamReader.java

@@ -58,9 +58,8 @@ public class IntegerStreamReader
     @Override
     public void putInts(int rowId, int count, int value)
     {
-        int id = rowId;
         for (int i = 0; i < count; i++) {
-            putInt(id++, value);
+            putInt(rowId++, value);
         }
     }
 

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java

@@ -15,10 +15,12 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
+import com.esotericsoftware.minlog.Log;
 import com.google.gson.Gson;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import io.prestosql.hive.$internal.au.com.bytecode.opencsv.CSVReader;
 import io.prestosql.spi.PrestoException;
+import io.prestosql.spi.StandardErrorCode;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
@@ -51,6 +53,7 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.SQLException;
 import java.text.DateFormat;
@@ -64,8 +67,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
+import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
@@ -1424,7 +1429,7 @@ public class TestCarbonAllDataType
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtabledrop", false), false);
 
         } catch (IOException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1469,7 +1474,7 @@ public class TestCarbonAllDataType
                     i++;
                 }
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
 
             // Step 3: convert format level TableInfo to code level TableInfo
@@ -1550,7 +1555,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1564,7 +1569,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormat.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     String dateString = outuptformat.format(date);
                     dateString = "date '" + dateString + "'";
@@ -1573,7 +1578,7 @@ public class TestCarbonAllDataType
                 return   "date '" + data + "'";
             }
             case "varchar":
-            {
+            {//'china'
                 return "'" + data + "'";
             }
             case "timestamp":
@@ -1587,7 +1592,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1598,7 +1603,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1609,7 +1614,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1620,7 +1625,7 @@ public class TestCarbonAllDataType
                     try {
                         date = inputFormattime.parse(data);
                     } catch (ParseException e) {
-                        logger.error(e.getMessage());
+                        e.printStackTrace();
                     }
                     dateString = outuptformattime.format(date);
                 }
@@ -1632,10 +1637,9 @@ public class TestCarbonAllDataType
                 return dateString;
             }
             case "smallint":  {
+                // smallint '12'
                 return "smallint '" + data + "'";
             }
-            default:
-                break;
         }
         return data;
     }
@@ -1693,7 +1697,7 @@ public class TestCarbonAllDataType
             hetuServer.execute(inserData);
         }
         catch(Exception e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1732,7 +1736,7 @@ public class TestCarbonAllDataType
 
         }
         catch (IOException | InterruptedException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
     }
 
@@ -1781,15 +1785,14 @@ public class TestCarbonAllDataType
     */
     private boolean checkStatusFileForDeleteMarked(String tableName, int updateNumber, int segmentNumber) throws SQLException
     {
-        BufferedReader reader = null;
         try {
             File dir = new File(storePath + "/carbon.store/testdb/" + tableName + "/Metadata");
             File[] tableUpdateStatusFiles = dir.listFiles((d, name) -> name.startsWith("tableupdatestatus"));
             Arrays.sort(tableUpdateStatusFiles);
             Gson gson = new Gson();
-            reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
+            BufferedReader reader = new BufferedReader(new FileReader(tableUpdateStatusFiles[updateNumber]));
             SegmentUpdateDetails[] segmentUpdateDetails = gson.fromJson(reader, SegmentUpdateDetails[].class);
-            File tableStatusFile = new File(dir.getCanonicalPath() + "/tablestatus");
+            File tableStatusFile = new File(dir.getAbsolutePath() + "/tablestatus");
             reader = new BufferedReader(new FileReader(tableStatusFile));
             LoadMetadataDetails loadMetadataDetails = gson.fromJson(reader, LoadMetadataDetails[].class)[segmentNumber];
             if ((segmentUpdateDetails[0].getSegmentStatus() != null && segmentUpdateDetails[0].getSegmentStatus().toString().equals("Marked for Delete")) &&
@@ -1800,16 +1803,6 @@ public class TestCarbonAllDataType
             hetuServer.execute("drop table if exists testdb." + tableName);
             Assert.fail("Failed to read status files");
         }
-        finally {
-            if (reader != null) {
-                try {
-                    reader.close();
-                }
-                catch (IOException e) {
-                    logger.error(e.getMessage());
-                }
-            }
-        }
         return false;
     }
 
@@ -1829,7 +1822,7 @@ public class TestCarbonAllDataType
                     "/carbon.store/testdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable");
@@ -1853,7 +1846,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.mytesttable2");
@@ -1886,7 +1879,7 @@ public class TestCarbonAllDataType
         }
         catch (IOException | InterruptedException e) {
             hetuServer.execute("DROP TABLE if exists testdb.myectable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists testdb.myectable");
@@ -1911,7 +1904,7 @@ public class TestCarbonAllDataType
                 FileFactory.mkdirs( storePath + "/carbon.store/mytestDb");
             }
         } catch (IOException e) {
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         String location = "'" + "file:///" + storePath + "/carbon.store/mytestDb" + "')" ;
@@ -2494,44 +2487,91 @@ public class TestCarbonAllDataType
     @Test
     public void test_writer_count() throws SQLException
     {
+        hetuServer.execute("drop table if exists testdb.testorders1");
+        hetuServer.execute("drop table if exists testdb.testorders_bak");
         hetuServer.execute("set session task_writer_count=32");
+        hetuServer.execute("set session implicit_conversion=true");
+        hetuServer.execute("CREATE TABLE testdb.testorders1(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("INSERT INTO testdb.testorders1 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
 
-        hetuServer.execute("CREATE TABLE testdb.testWriterCount_32(orderkey int, orderstatus STRING, totalprice double, orderdate date)");
+        hetuServer.execute("CREATE TABLE testdb.testorders_bak(orderkey bigint, orderstatus varchar(7), totalprice double, orderdate date)");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(10,'SUCCESS', 125.15, DATE'2919-05-17')");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(20,'SUCCESS', 125.15, DATE'2919-05-17')");
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 VALUES(30,'SUCCESS', 125.15, DATE'2919-05-17')");
 
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 6);
-
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 12);
-
-        hetuServer.execute("INSERT INTO testdb.testWriterCount_32 SELECT * FROM testdb.testWriterCount_32");
-        verifyRowCount("testdb.testWriterCount_32", 24);
-
-        String filePath = storePath + "/carbon.store/testdb/testwritercount_32";
-        try {
-            hetuServer.execute("VACUUM TABLE testdb.testWriterCount_32 AND WAIT");
-            assertTrue(FileFactory.isFileExist(filePath + "/Fact/Part0/Segment_0.1"));
-        }
-        catch (IOException e) {
-            assertTrue(false, "Unable to read file from table path");
-        }
-        finally {
-            hetuServer.execute("drop table if exists testdb.testWriterCount_32");
         hetuServer.execute("set session task_writer_count=1");
-        }
+        List<Map<String, Object>>  actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
-    }
 
-    private void verifyRowCount(String tableName, int rowCount) throws SQLException
-    {
-        String query = String.format("SELECT COUNT(*) AS result FROM %s", tableName);
-        List<Map<String, Object>>  actualResult = hetuServer.executeQuery(query);
         List<Map<String, Object>>  expectedResult = new ArrayList<Map<String, Object>>() {{
-            add(new HashMap<String, Object>() {{    put("result", rowCount); }});
+            add(new HashMap<String, Object>() {{    put("RESULT", 3); }});
         }};
         Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        // checking correct number of files created in Fact
+        File files1 = null;
+        files1 = new File(storePath + "/carbon.store/testdb/testorders_bak/Fact");
+        File[] fileListPart0 = files1.listFiles();//Part0
+
+        if (fileListPart0.length == 1) {
+            File[] Segment_0 = fileListPart0[0].listFiles();
+            if (Segment_0.length == 1) {
+                File[] fileListSegment_0 = Segment_0[0].listFiles();
+                if (fileListSegment_0.length == 6) {
+                    assertEquals("true", "true");
+                }
+                assertEquals(fileListSegment_0.length, 6);
+            }
+            assertEquals(Segment_0.length, 1);
+        }
+        assertEquals(fileListPart0.length, 1);
+
+        //test with 16
+        hetuServer.execute("set session task_writer_count=16");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 6); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        //test with 8
+        hetuServer.execute("set session task_writer_count=8");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 9); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        //test with 2
+        hetuServer.execute("set session task_writer_count=2");
+        hetuServer.execute("insert into testdb.testorders_bak(orderkey, orderstatus, totalprice) select orderkey, orderstatus, totalprice from testorders1");
+
+        hetuServer.execute("set session task_writer_count=1");
+        actualResult = hetuServer.executeQuery("Select count (*) as RESULT from testdb.testorders_bak");
+
+        expectedResult = new ArrayList<Map<String, Object>>() {{
+            add(new HashMap<String, Object>() {{    put("RESULT", 12); }});
+        }};
+        Assert.assertEquals(actualResult.toString(), expectedResult.toString());
+
+        try {
+            hetuServer.execute("set session task_writer_count=32");
+            hetuServer.execute("VACUUM TABLE testdb.testorders_bak AND WAIT");
+            assertEquals(FileFactory.isFileExist(storePath +
+                    "/carbon.store/testdb/testorders_bak/Fact/Part0/Segment_0.1", false), true);
+        } catch (IOException e) {
+        }
+
+        hetuServer.execute("drop table if exists testdb.testorders1");
+        hetuServer.execute("drop table if exists testdb.testorders_bak");
+        hetuServer.execute("set session task_writer_count=1");
     }
 
     private TableInfo getTableInfoFromSchemaFile(String tableName)

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAutoVacuum.java

@@ -40,7 +40,6 @@ import java.util.List;
 import java.util.Map;
 
 import static org.testng.Assert.assertTrue;
-import static org.testng.AssertJUnit.assertNotNull;
 
 public class TestCarbonAutoVacuum
 {
@@ -100,6 +99,7 @@ public class TestCarbonAutoVacuum
     @AfterClass
     public void tearDown() throws SQLException, IOException, InterruptedException
     {
+        //hetuServer.execute("drop table if exists hive.default.demotable");
         logger.info("TearDown begin: " + this.getClass().getSimpleName());
         hetuServer.stopServer();
         CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath));
@@ -145,12 +145,11 @@ public class TestCarbonAutoVacuum
 
         try {
             CarbondataAutoVacuumThread.enableTracingVacuumTask(true);
-            assertNotNull(catalog);
             connector = catalog.getConnector(catalog.getConnectorCatalogName());
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();
@@ -216,7 +215,7 @@ public class TestCarbonAutoVacuum
             connectorMetadata = connector.getConnectorMetadata();
             connectorMetadata.getTablesForVacuum();
         } catch (Exception e) {
-            logger.debug(e.getMessage());
+
         }
 
         CarbondataAutoVacuumThread.waitForSubmittedVacuumTasksFinish();

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java

@@ -62,6 +62,7 @@ public class TestCarbondataAutoCleanup
     public void setup() throws Exception
     {
         logger.info("Setup begin: " + this.getClass().getSimpleName());
+        String dataPath = rootPath + "/src/test/resources/alldatatype.csv";
 
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest");
         CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0");
@@ -130,7 +131,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -159,7 +160,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -189,7 +190,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -221,7 +222,7 @@ public class TestCarbondataAutoCleanup
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_2", false), false);
                 assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanupwithpushdown/Fact/Part0/Segment_3", false), false);
             } catch (IOException exception) {
-                logger.debug(exception.getMessage());
+
             }
         }
         finally {
@@ -254,7 +255,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -285,7 +286,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false);
         }
         catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -314,7 +315,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -344,7 +345,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -374,7 +375,7 @@ public class TestCarbondataAutoCleanup
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false);
             assertEquals(FileFactory.isFileExist(storePath  + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false);
         } catch (IOException exception) {
-            logger.debug(exception.getMessage());
+
         }
 
         CarbondataMetadata.enableTracingCleanupTask(false);
@@ -403,7 +404,7 @@ public class TestCarbondataAutoCleanup
                 content = content.replaceFirst(modificationOrdeletionTimesStamp, replace);
                 Files.write(path, content.getBytes(charset));
             } catch (IOException e) {
-                logger.error(e.getMessage());
+                e.printStackTrace();
             }
         }
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataMinorConfig.java

@@ -99,7 +99,7 @@ public class TestCarbondataMinorConfig
                     "/carbon.store/mytestdb/mytesttable/Fact/Part0/Segment_0.1", false), true);
         } catch (IOException e) {
             hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");
-            logger.error(e.getMessage());
+            e.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE if exists mytestdb.mytesttable");

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestsWithHiveConnector.java

@@ -15,7 +15,6 @@
 
 package io.hetu.core.plugin.carbondata.integrationtest;
 
-import io.airlift.log.Logger;
 import io.hetu.core.plugin.carbondata.server.HetuTestServer;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -37,7 +36,6 @@ import static org.testng.Assert.assertTrue;
 
 public class TestsWithHiveConnector
 {
-    private static final Logger log = Logger.get(TestsWithHiveConnector.class);
     private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..")
             .getCanonicalPath();
 
@@ -116,7 +114,7 @@ public class TestsWithHiveConnector
         assertEquals(FileFactory.isFileExist(storePath +
                 "hive.store/default/parttable/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
         hetuServer.execute("DROP TABLE hive.default.parttable");
     }
@@ -143,7 +141,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2013", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
         hetuServer.execute("insert into hive.default.parttable2 values (4,2014)");
@@ -152,7 +150,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "hive.store/default/parttable2/year=2014", false), false);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
 
         hetuServer.execute("DROP TABLE hive.default.parttable2");
@@ -180,7 +178,7 @@ public class TestsWithHiveConnector
             assertEquals(FileFactory.isFileExist(storePath +
                     "/hive.store/default/parttable3/year=2013", false), true);
         } catch (IOException exception) {
-            log.error(exception.getMessage());
+            exception.printStackTrace();
         }
         hetuServer.execute("DROP TABLE hive.default.parttable3");
     }

hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/server/HetuTestServer.java

@@ -91,11 +91,11 @@ public class HetuTestServer
         carbonProperties.putAll(properties);
 
         logger.info("------------ Starting Presto Server -------------");
-        DistributedQueryRunner distributedQueryRunner = createQueryRunner(hetuProperties);
+        DistributedQueryRunner queryRunner = createQueryRunner(hetuProperties);
         Connection connection = createJdbcConnection(dbName);
         statement = (PrestoStatement) connection.createStatement();
 
-        logger.info("STARTED SERVER AT :" + distributedQueryRunner.getCoordinator().getBaseUrl());
+        logger.info("STARTED SERVER AT :" + queryRunner.getCoordinator().getBaseUrl());
     }
 
     public void stopServer() throws SQLException
@@ -123,20 +123,14 @@ public class HetuTestServer
     public List<Map<String, Object>> executeQuery(String query) throws SQLException
     {
         logger.info(">>>>> Executing Query: " + query);
-        ResultSet rs = null;
         try {
-            rs = statement.executeQuery(query);
+            ResultSet rs = statement.executeQuery(query);
             return convertResultSetToList(rs);
         }
         catch (SQLException e) {
             logger.error("Exception Occured: " + e.getMessage() + "\n Failed Query: " + query);
             throw e;
         }
-        finally {
-            if (rs != null) {
-                rs.close();
-            }
-        }
     }
 
     private List<Map<String, Object>> convertResultSetToList(ResultSet rs) throws SQLException
@@ -192,7 +186,7 @@ public class HetuTestServer
     {
         try {
             queryRunner.installPlugin(new CarbondataPlugin());
-            Map<String, String> carbonPropertiesMap = ImmutableMap.<String, String>builder()
+            Map<String, String> carbonProperties = ImmutableMap.<String, String>builder()
                     .putAll(this.carbonProperties)
                     .put("carbon.unsafe.working.memory.in.mb", "512")
                     .build();
@@ -203,7 +197,7 @@ public class HetuTestServer
                     .build();
 
             // CreateCatalog will create a catalog for CarbonData in etc/catalog.
-            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonPropertiesMap);
+            queryRunner.createCatalog(carbonDataCatalog, carbonDataConnector, carbonProperties);
             queryRunner.createCatalog(carbonDataCatalogLocationDisabled, carbonDataConnector, carbonPropertiesLocationDisabled);
         }
         catch (RuntimeException e) {

hetu-clickhouse/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-clickhouse</artifactId>

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/ClickHouseClient.java

@@ -345,9 +345,8 @@ public class ClickHouseClient
     }
 
     @Override
-    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String inputNewColumnName)
+    public void renameColumn(JdbcIdentity identity, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName)
     {
-        String newColumnName = inputNewColumnName;
         try (Connection connection = connectionFactory.openConnection(identity)) {
             if (connection.getMetaData().storesUpperCaseIdentifiers()) {
                 newColumnName = newColumnName.toUpperCase(ENGLISH);

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseApplyRemoteFunctionPushDown.java

@@ -41,7 +41,6 @@ public class ClickHouseApplyRemoteFunctionPushDown
     /**
      * rewrite the remote function to a executable function in the data source.
      */
-    @Override
     public Optional<String> rewriteRemoteFunction(CallExpression callExpression, BaseJdbcRowExpressionConverter rowExpressionConverter, JdbcConverterContext jdbcConverterContext)
     {
         if (!isConnectorSupportedRemoteFunction(callExpression)) {

hetu-clickhouse/src/main/java/io/hetu/core/plugin/clickhouse/optimization/ClickHouseSqlStatementWriter.java

@@ -37,9 +37,8 @@ public class ClickHouseSqlStatementWriter
     }
 
     @Override
-    public String aggregation(String inputFunctionName, List<String> arguments, boolean isDistinct)
+    public String aggregation(String functionName, List<String> arguments, boolean isDistinct)
     {
-        String functionName = inputFunctionName;
         if (functionName.toUpperCase(Locale.ENGLISH).equals("VARIANCE")) {
             functionName = "varPop";
         }

hetu-clickhouse/src/test/java/io/hetu/core/plugin/clickhouse/ClickHouseServerTest.java

@@ -205,7 +205,7 @@ public final class ClickHouseServerTest
     {
         String actualTable = tablePattern;
 
-        for (String table : tables) {
+        for (String table : tables) { //tableName + _ + UUID
             int lastIndex = table.lastIndexOf("_");
             if (lastIndex == -1) {
                 continue;

hetu-common/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-common</artifactId>

hetu-common/src/main/java/io/hetu/core/common/util/SslSocketUtil.java

@@ -47,6 +47,16 @@ public class SslSocketUtil
         if (!tlsEnabled) {
             return Optional.empty();
         }
+
+        // https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html#CustomizingStores
+        // as per link above, the default SSLContext will be constructed using the default KeyManager and
+        // default TrustManager. Those can be configured using the following system properties:
+        // javax.net.ssl.keyStore
+        // javax.net.ssl.keyStorePassword
+        // javax.net.ssl.keyStoreType
+        // javax.net.ssl.trustStore
+        // javax.net.ssl.trustStorePassword
+        // see link above for more details
         return Optional.of(SSLContext.getDefault());
     }
 

hetu-common/src/main/java/io/hetu/core/common/util/TrustStore.java

@@ -13,7 +13,6 @@
  */
 package io.hetu.core.common.util;
 
-import io.airlift.log.Logger;
 import io.airlift.security.pem.PemReader;
 
 import javax.security.auth.x500.X500Principal;
@@ -30,8 +29,6 @@ import java.util.Optional;
 
 public class TrustStore
 {
-    private static final Logger LOGGER = Logger.get(TrustStore.class);
-
     private TrustStore() {}
 
     public static KeyStore loadTrustStore(File trustStorePath, Optional<String> trustStorePassword)
@@ -51,7 +48,6 @@ public class TrustStore
             }
         }
         catch (IOException | GeneralSecurityException ignored) {
-            LOGGER.error("loadTrustStore error : %s", ignored.getMessage());
         }
 
         try (InputStream in = new FileInputStream(trustStorePath)) {

hetu-common/src/test/java/io/hetu/core/common/filesystem/TestTempFolder.java

@@ -34,9 +34,9 @@ public class TestTempFolder
             root = folder.getRoot();
             assertTrue(root.exists());
             File newFile = folder.newFile("aNewFile");
-            assertEquals(newFile.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFile");
+            assertEquals(newFile.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFile");
             File newFolder = folder.newFile("aNewFolder");
-            assertEquals(newFolder.getCanonicalPath(), folder.getRoot().getCanonicalPath() + "/aNewFolder");
+            assertEquals(newFolder.getAbsolutePath(), folder.getRoot().getAbsolutePath() + "/aNewFolder");
         }
         assertFalse(root.exists());
     }

hetu-cube/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-cube</artifactId>

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeFilter.java

@@ -36,7 +36,8 @@ public class CubeFilter
 
     public CubeFilter(String sourceTablePredicate)
     {
-        this(sourceTablePredicate, null);
+        this.sourceTablePredicate = sourceTablePredicate;
+        this.cubePredicate = null;
     }
 
     public String getSourceTablePredicate()

hetu-cube/src/main/java/io/hetu/core/spi/cube/CubeStatement.java

@@ -140,13 +140,13 @@ public class CubeStatement
             return this;
         }
 
-        public Builder groupByAddString(String column)
+        public Builder groupBy(String column)
         {
             this.groupBy.add(column);
             return this;
         }
 
-        public Builder groupByAddStringList(String... columns)
+        public Builder groupBy(String... columns)
         {
             this.groupBy.addAll(Arrays.asList(columns));
             return this;

hetu-cube/src/test/java/io/hetu/core/spi/cube/TestCubeStatement.java

@@ -34,8 +34,8 @@ public class TestCubeStatement
                 .select("name", "address", "nationkey")
                 .aggregate(AggregationSignature.count())
                 .from("tpch.tiny.customer")
-                .groupByAddString("address")
+                .groupBy("address")
-                .groupByAddStringList("name", "nationkey")
+                .groupBy("name", "nationkey")
                 .build();
 
         assertEquals(statement.getFrom(), "tpch.tiny.customer", "incorrect from table");

hetu-datacenter/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>io.hetu.core</groupId>
         <artifactId>presto-root</artifactId>
-        <version>1.7.0-SNAPSHOT</version>
+        <version>1.4.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>hetu-datacenter</artifactId>

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterColumnHandle.java

@@ -55,7 +55,6 @@ public final class DataCenterColumnHandle
     }
 
     @JsonProperty
-    @Override
     public String getColumnName()
     {
         return columnName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterTableHandle.java

@@ -56,11 +56,11 @@ public final class DataCenterTableHandle
      */
     public DataCenterTableHandle(String catalogName, String schemaName, String tableName, OptionalLong limit)
     {
-        this(catalogName,
+        this.catalogName = catalogName;
-                requireNonNull(schemaName, "schemaName is null"),
+        this.schemaName = requireNonNull(schemaName, "schemaName is null");
-                requireNonNull(tableName, "tableName is null"),
+        this.tableName = requireNonNull(tableName, "tableName is null");
-                requireNonNull(limit, "limit is null"),
+        this.limit = requireNonNull(limit, "limit is null");
-                "");
+        this.pushDownSql = "";
     }
 
     /**
@@ -125,7 +125,6 @@ public final class DataCenterTableHandle
         return new SchemaTableName(schemaName, tableName);
     }
 
-    @Override
     public String getSchemaPrefixedTableName()
     {
         return catalogName + SPLIT_DOT + schemaName + SPLIT_DOT + tableName;

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterPlanOptimizer.java

@@ -179,7 +179,7 @@ public class DataCenterPlanOptimizer
             List<RowExpression> pushable = new ArrayList<>();
             List<RowExpression> nonPushable = new ArrayList<>();
 
-            for (RowExpression conjunct : LogicalRowExpressions.extractConjuncts(node.getPredicate())) {
+            for (RowExpression conjunct : logicalRowExpressions.extractConjuncts(node.getPredicate())) {
                 try {
                     conjunct.accept(queryGenerator.getConverter(), new JdbcConverterContext());
                     pushable.add(conjunct);

hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/optimization/DataCenterQueryGenerator.java

@@ -116,20 +116,6 @@ public class DataCenterQueryGenerator
                     .setSchemaTableName(Optional.of(new SchemaTableName(dcTableHandle.getSchemaName(), dcTableHandle.getTableName())))
                     .setSelections(selections)
                     .setFrom(Optional.of(table.toString()));
-
-            String catalogName = dcTableHandle.getCatalogName();
-            if (catalogName != null) {
-                contextBuilder.setRemoteCatalogName(catalogName);
-            }
-            String schemaName = dcTableHandle.getSchemaName();
-            if (schemaName != null) {
-                contextBuilder.setRemoteSchemaName(schemaName);
-            }
-            String tableName = dcTableHandle.getTableName();
-            if (tableName != null) {
-                contextBuilder.setRemoteTablename(tableName);
-            }
-
             // If LIMIT has been push down, add it to context
             if (dcTableHandle.getLimit().isPresent()) {
                 contextBuilder.setLimit(dcTableHandle.getLimit());

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestCrossRegionDynamicFilter.java

@@ -1392,24 +1392,24 @@ public class TestCrossRegionDynamicFilter
         hetuServer.installPlugin(new StateStoreManagerPlugin());
         hetuServer.loadStateSotre();
 
-        DistributedQueryRunner distributedQueryRunner = null;
+        DistributedQueryRunner queryRunner = null;
         try {
-            distributedQueryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
+            queryRunner = DistributedQueryRunner.builder(testSessionBuilder().build())
                     .setNodeCount(1)
                     .build();
 
             Map<String, String> connectorProperties = new HashMap<>(properties);
             connectorProperties.putIfAbsent("connection-url", hetuServer.getBaseUrl().toString());
             connectorProperties.putIfAbsent("connection-user", "root");
-            distributedQueryRunner.installPlugin(new DataCenterPlugin());
+            queryRunner.installPlugin(new DataCenterPlugin());
-            distributedQueryRunner.createDCCatalog("dc", "dc", connectorProperties);
+            queryRunner.createDCCatalog("dc", "dc", connectorProperties);
-            distributedQueryRunner.installPlugin(new TpchPlugin());
+            queryRunner.installPlugin(new TpchPlugin());
-            distributedQueryRunner.createCatalog("tpch", "tpch", properties);
+            queryRunner.createCatalog("tpch", "tpch", properties);
 
-            return distributedQueryRunner;
+            return queryRunner;
         }
         catch (Throwable e) {
-            closeAllSuppress(e, distributedQueryRunner);
+            closeAllSuppress(e, queryRunner);
             throw e;
         }
     }

hetu-datacenter/src/test/java/io/hetu/core/plugin/datacenter/TestDataCenterClient.java

@@ -212,31 +212,31 @@ public class TestDataCenterClient
     @Test(expectedExceptions = RuntimeException.class)
     public void testPasswordWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setConnectionPassword("root")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testKerberosWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setKerberosRemoteServiceName("kerberos")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)
     public void testAccessTokenWithoutSSL()
     {
-        DataCenterConfig dataCenterConfig = new DataCenterConfig().setConnectionUrl(this.baseUri)
+        DataCenterConfig config = new DataCenterConfig().setConnectionUrl(this.baseUri)
                 .setConnectionUser("root")
                 .setAccessToken("token")
                 .setSsl(false);
-        DataCenterStatementClientFactory.newHttpClient(dataCenterConfig);
+        DataCenterStatementClientFactory.newHttpClient(config);
     }
 
     @Test(expectedExceptions = RuntimeException.class)

hetu-docs/en/admin/audit-log.md

@@ -1,6 +1,6 @@
 # Audit Log
 
-openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
+openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
 An audit log contains the following information:
 
 1. time when an event occurs
@@ -24,17 +24,15 @@ To enable audit logging feature, the following configs must be present in `etc/e
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
-The following is a detailed description of audit logging properties:
+Other audit logging properties include: 
 
-`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
+`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
 
-`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
+`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
 
-`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
+`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
 
 Example configuration file:
 
@@ -43,6 +41,7 @@ event-listener.name=hetu-listener
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
+hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
+hetu.event.listener.audit.filecount=1
+hetu.event.listener.audit.limit=100000
 ```

hetu-docs/en/admin/extension-execution-planner.md

@@ -1,25 +0,0 @@
-#Extension Physical Execution Planner
-This section describes how to add an extension physical execution planner in openLooKeng. With the extension physical execution planner, openLooKeng can utilize other operator acceleration libraries to speed up the execution of SQL statements.
-
-##Configuration
-To enable extension physical execution feature, the following configs must be added in
-`config.properties`：
-
-``` properties
-extension_execution_planner_enabled=true
-extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
-extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
-```
-
-The above attributes are described below:
-
-- `extension_execution_planner_enabled`: Enable extension physical execution feature.
-- `extension_execution_planner_jar_path`: Set the file path of the extension physical execution jar package.
-- `extension_execution_planner_class_path`: Set the package path of extension physical execution generated class in jar。
-
-
-##Usage
-The below command can control the enablement of extension physical execution feature in WebUI or Cli while running openLooKeng:
-```
-set session extension_execution_planner_enabled=true/false
-```

hetu-docs/en/admin/properties.md

@@ -118,20 +118,6 @@ This section describes the most important config properties that may be used to
 >
 > This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
 
-### `query.suspend-query-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables running query temporary suspension when system is in low resource situation.
-
-### `query.max-suspended-queries`
-
-> -   **Type:** `integer`
-> -   **Default value:** `10`
->
-> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
-
 ## Spilling Properties
 
 ### `experimental.spill-enabled`
@@ -175,30 +161,6 @@ This section describes the most important config properties that may be used to
 >
 > This config property can be overridden by the `spill_window_operator` session property.
 
-
-### `experimental.spill-build-for-outer-join-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spill feature for right-outer and full-outer join operations.
->
->
->
-> This config property can be overridden by the `spill_build_for_outer_join_enabled` session property.
-
-### `experimental.inner-join-spill-filter-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables bloom filter based build-side spill matching for probe side spill decision.
->
->
->
-> This config property can be overridden by the `inner_join_spill_filter_enabled` session property.
-
-
 ### `experimental.spill-reuse-tablescan`
 
 > -   **Type:** `boolean`
@@ -217,7 +179,7 @@ This section describes the most important config properties that may be used to
 >
 > Directory where spilled content will be written. It can be a comma separated list to spill simultaneously to multiple directories, which helps to utilize multiple drives installed in the system.
 >
-> When `experimental.spiller-spill-to-hdfs` is to `true`, `experimental.spiller-spill-path` must contain only  a single directory.
+>
 >
 > It is not recommended to spill to system drives. Most importantly, do not spill to the drive on which the JVM logs are written, as disk overutilization might cause JVM to pause for lengthy periods, causing queries to fail.
 
@@ -278,71 +240,6 @@ This section describes the most important config properties that may be used to
 >
 > Enables using a randomly generated secret key (per spill file) to encrypt and decrypt data spilled to disk
 
-### `experimental.spill-direct-serde-enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables to serialize/read the page directly to/from the stream.
-
-### `experimental.spill-prefetch-read-pages`
-
-> -   **Type:** `integer`
-> -   **Default value:** `1`
->
-> Sets number of pages prefetched while reading from spilled files.
-
-
-### `experimental.spill-use-kryo-serialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for spill to disk, instead of default java serializer.
-
-
-### `experimental.revocable-memory-selection-threshold`
-
-> -   **Type:** `data size`
-> -   **Default value:** `512 MB`
->
-> Sets memory selection threshold for revocable memory of operator to directly allocate revocable memory for remaining bytes ready to revoke.
-
-### `experimental.prioritize-larger-spilts-memory-revoke`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `true`
->
-> Enables to prioritize splits with larger revocable memory.
-
-### `experimental.spill-non-blocking-orderby`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables order by operator to use asynchronous mechanism to spill, i.e it can accumulate input even when a spill is in progress and initiate a secondary spill when the secondary data accumulate exceeds a threshold or when the primary spill is completed, the default value of the threshold is the minimum between 20MB and 5% of available free memory. This property must be used in conjunction with the `experimental.spill-enabled` property.
->
->
->
-> This config property can be overridden by the `spill_non_blocking_orderby` session property.
-
-### `experimental.spiller-spill-to-hdfs`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables spilling into HDFS. When this property is set to `true` the property `experimental.spiller-spill-profile` must be set and also `experimental.spiller-spill-path` must contain only a single path.
-
-### `experimental.spiller-spill-profile`
-
-> -   **Type:** `string`
-> -   **No default value.** Must be set when spilling to hdfs is enabled
->
->
-> This property defines the [filesystem](../develop/filesystem.md) profile used to spill. The corresponding profile must exist in `etc/filesystem`. For example, if this property is set as `experimental.spiller-spill-profile=spill-hdfs`, a profile describing this filesystem `spill-hdfs.properties` must be created in `etc/filesystem` with necessary information including authentication type, config, and keytabs (if applicable, refer [filesystem](../develop/filesystem.md) for details).
->
-> This property is required when `experimental.spiller-spill-to-hdfs` is set to `true`. It must be included in configuration files for all coordinators and all workers. The specified file system must be accessible by all workers, and they must be able to read from and write to the path declared in `experimental.spiller-spill-path` folder in the specified file system.
-
 ## Exchange Properties
 
 Exchanges transfer data between openLooKeng nodes for different stages of a query. Adjusting these properties may help to resolve inter-node communication issues or improve network utilization.
@@ -378,8 +275,19 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
 >
+>
+>
 > Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
 
+### `exchange.max-error-duration`
+
+> -   **Type:** `duration`
+> -   **Minimum value:** `1m`
+> -   **Default value:** `7m`
+>
+> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a failure.
+
+
 ### `sink.max-buffer-size`
 
 > -   **Type:** `data size`
@@ -387,113 +295,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
 >
 > Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
 
-## Failure Recovery handling Properties
-
-### Failure Retry Policies
-
-### `failure.recovery.retry.profile`
-
-> -   **Type:** `String`
-> -   **Default value:** `default`
->
-> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
-> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
-> The file `test.properties` must contain `failure.recovery.retry.type` specified.
-
-
-### `failure.recovery.retry.type`
-
-> -   **Type:** `String`
-> -   **Default value:** `timeout`
->
-> The failure detection mechanism in use. Default is timeout based failure detection.
->
-#### `timeout` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
-> Additional properties `max.error.duration` can be defined for this type of failure detection.
->
-#### `max-retry` based failure detection. 
-> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
-> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection. 
-> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
-
-### `max.error.duration`
-> -   **Type:** `duration`
-> -   **Default value:** `300s`
->
-> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
-
-
-### `max.retry.count`
-
-> -   **Type:** `integer`
-> -   **Default value:** `100`
->
-> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. 
-> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster. 
-> This property is used only for `max-retry` based failure detection profiles. 
-> The minimum value for this parameter is 100.
-
-### Gossip Protocol Configurations for Failure Detection
-
-### `failure-detection-protocol`
-
->- **Type:** String
->- **Default value:** `heartbeat`
->
-> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
-> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
-> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
-
-### `failure-detector.heartbeat-interval`
-
->- **Type:** Duration
->- **Default value:** `500ms` (500 miliseconds)
->
-> This is the interval of gossip between two nodes in the cluster.
-> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
-> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
-> In workers, this property can be left to use the default value.
->
-### `failure-detector.worker-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.coordinator-gossip-probe-interval`
->
-> - **Type:** Duration
->- **Default value:** `5s` (5 seconds)
->
-> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
-> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
-> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
->
-### `failure-detector.coordinator-gossip-collate-interval`
->
-> - **Type:** Duration
->- **Default value:** `2s` (2 seconds)
->
-> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
-> This property has to be specified only for the coordinator.
-> This parameter should have higher value than `failure-detector.heartbeat-interval`.
->
-### `failure-detector.gossip-group-size`
->
-> - **Type:** Integer
->- **Default value:** `Integer.MAX_VALUE`
->
-> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
-> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
-> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
-> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
-
 ## Task Properties
 
 ### `task.concurrency`
@@ -878,7 +679,7 @@ helps with cache affinity scheduling.
 > Auto-Vacuum enables the system to automatically manage vacuum jobs by constantly monitoring the tables which needs vacuum in order to maintain optimal performance.
 > Engine gets the tables from data sources that are eligible for vacuum and trigger vacuum operation for those tables.
 
-### `auto-vacuum.enabled`
+### `auto-vacuum.enabled:`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
@@ -948,25 +749,15 @@ helps with cache affinity scheduling.
 > -   **Default value:** `5m`
 >
 > The maximum time coordinator waits for remote-task related error to be resolved before it's considered a failure.
->
-> Note:
-> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
 
-## Query Recovery
+## Distributed Snapshot
-
-### `recovery_enabled`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
 
 ### `snapshot_enabled`
 
 > -   **Type:** `boolean`
 > -   **Default value:** `false`
 >
-> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
+> This session property is used to enable or disable the distributed snapshot functionality.
 
 ### `hetu.experimental.snapshot.profile`
 
@@ -978,67 +769,20 @@ helps with cache affinity scheduling.
 >
 > This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
 
-### `hetu.recovery.maxRetries`
+### `hetu.snapshot.maxRetries`
 
 > -   **Type:** `int`
 > -   **Default value:** `10`
 >
 > This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
 >
-> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
+> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
 
-### `hetu.recovery.retryTimeout`
+### `hetu.snapshot.retryTimeout`
 
 > -   **Type:** `duration`
 > -   **Default value:** `10m` (10 minutes)
 >
 > This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
 >
-> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
+> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.
-
-### `hetu.snapshot.useKryoSerialization`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables Kryo based serialization for snapshot, instead of default java serializer.
-
-### `experimental.eliminate-duplicate-spill-files`
-
-> -   **Type:** `boolean`
-> -   **Default value:** `false`
->
-> Enables elimination of duplicate spill files storage as part of snapshot capture.
-
-
-
-## HTTP Client Configurations
-
-### `http.client.idle-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `30s` (30 seconds)
->
-> This property defines the time for which a given http client shall stay connected without any operations performed over it.
-> After the specified time elapse with no activity, then the client connection is closed and related resources are released.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-### `http.client.request-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `10s` (10 seconds)
->
-> This property defines the time threshold for a given http client for which response should be received.
-> After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
->
-> (Note: this parameter should be configured with higher time when in high load environment)
-
-## Connector Properties configuration
-
-### `case-insensitive-name-matching`
->
-> -    **Type:** `boolean`
-> -    **Default value:** `false`
->
-> Case-insensitive matching between database and collection names. The default is case sensitive.

hetu-docs/en/admin/reliable-execution.md

@@ -11,9 +11,9 @@ To achieve better performance while maintaining execution reliability, the *dist
 
 As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
 
-## Enable Recovery framework
+## Enable Distributed Snapshot
 
-Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
+Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
 
 ## Requirements
 
@@ -37,7 +37,7 @@ When a query that does not meet the above requirements is submitted with distrib
 
 ## Detection
 
-Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
+Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration) configuration.
 
 ## Storage Considerations
 
@@ -55,20 +55,8 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
 
 The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
 
-It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
+It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
-
-## Snapshot statistics
-
-Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
-
-Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query.  These statistics are displayed for all snapshots and for last snapshot separately.
-
-Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots.  Restore statistics are displayed only when there is restore(recovery) happened during the query.
-
-Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details 
-
-![](../images/snapshot_statistics.png)
 
 ## Configurations
 
-Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).
+Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).

hetu-docs/en/admin/spill.md

@@ -34,10 +34,6 @@ saturation of the configured spill paths.
 
 openLooKeng treats spill paths as independent disks (see [JBOD](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#JBOD)), so there is no need to use RAID for spill.
 
-
-## Spill To HDFS
-Spilling directly into HDFS is also possible for that `experimental.spiller-spill-to-hdfs` needs to be set to `true`, `experimental.spiller-spill-profile` needs to be set and `spiller-spill-path` must contain only a single directory when we intend to spill into HDFS. (refer `experimental.spiller-spill-to-hdfs` and  `experimental.spiller-spill-profile` properties for more details )
-
 ## Spill Compression
 
 
@@ -65,17 +61,14 @@ When the build table is partitioned, the spill-to-disk mechanism can decrease th
 
 With this mechanism, the peak memory used by the join operator can be decreased to the size of the largest build table partition. Assuming no data skew, this will be `1 / task.concurrency` times the size of the whole build table.
 
-Note: spill-to-disk is not supported for Cross Join.
-
 ### Aggregations
 
 Aggregation functions perform an operation on a group of values and return one value. If the number of groups you\'re aggregating over is large, a significant amount of memory may be needed. When spill-to-disk
-is enabled, if there is not enough memory, intermediate accumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
+is enabled, if there is not enough memory, intermediate cumulated aggregation results are written to disk. They are loaded back and merged with a lower memory footprint.
 
 ### Order By
 
 If you're trying to sort a larger amount of data, a significant amount of memory may be needed. When spill to disk for order by is enabled, if there is not enough memory, intermediate sorted results are written to disk. They are loaded back and merged with a lower memory footprint.
-Generally when a spill is in progress the operator is blocked from taking inputs, but when `experimental.spill-non-blocking-orderby` is set to `true` order by uses asynchronous mechanism to spill (see`experimental.spill-non-blocking-orderby`).
 
 ### Window functions
 

hetu-docs/en/admin/web-interface.md

@@ -33,54 +33,4 @@ and statistics about the query is available by clicking the *JSON* link. These v
 > -   **Allowed values:** `true`, `false`
 > -   **Default value:** `false`
 >
-> Insecure authentication over HTTP is disabled by default. This could be overridden via `hetu.queryeditor-ui.allow-insecure-over-http` property of `etc/config.properties` (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
+> Insecure authentication over HTTP is disabled by default. This could be overridden via "hetu.queryeditor-ui.allow-insecure-over-http" property of "etc/config.properties" (e.g. hetu.queryeditor-ui.allow-insecure-over-http=true).
-
-### `hetu.queryeditor-ui.execution-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `100 DAYS`
->
-> UI Execution timeout is set to 100 days as default. This could be overridden via `hetu.queryeditor-ui.execution-timeout` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.max-result-count`
-
-> - **Type:** `int`
-> - **Default value:** `1000`
->
-> UI max result count is set to 1000 as default. This could be overridden via `hetu.queryeditor-ui.max-result-count` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.max-result-size-mb`
-
->- **Type:** `size`
->- **Default value:** `1GB`
->
-> UI max result size is set to 1 GB as default. This could be overridden via `hetu.queryeditor-ui.max-result-size-mb` of `etc/config.properties`
-
-### `hetu.queryeditor-ui.session-timeout`
-
-> -   **Type:** `duration`
-> -   **Default value:** `1 DAYS`
->
-> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
-
-### `hetu.queryhistory.max-count`
-
-> -   **Type:** `int`
-> -   **Default value:** `1000`
->
-> The maximum number of query history stored by openLooKeng. This could be overridden via "hetu.queryhistory.max-count" of "etc/config.properties".
-
-### `hetu.collectionsql.max-count`
-
-> -   **Type:** `int`
-> -   **Default value:** `100`
->
-> The Maximum number of SQL collected by each user. This could be overridden via "hetu.collectionsql.max-count" of "etc/config.properties".
-
-## Remarks
-
-The max length of the favorite SQL is 600 by default. You can modify it through the following steps:
-
-1. Login MySQL database according to the JDBC configuration of `hetu-metastore.properties`
-
-2. Select table hetu_favorite, execute script `alter table hetu_favorite modify query varchar(2000) not null;` to modify the max length of the favorite SQL.

hetu-docs/en/connector/Hudi.md

@@ -1,8 +1,5 @@
 # Hudi  Connector
 
-### Release Notes
-Currently Hudi only supports version 0.7.0.
-
 ### Hudi Introduction
 
 Apache Hudi is a fast growing data lake storage system that helps organizations build and manage petabyte-scale data lakes. Hudi enables storing vast amounts of data on top of existing DFS compatible storage while also enabling stream processing in addition to typical batch-processing. This is made possible by providing two new primitives. Specifically,

hetu-docs/en/connector/elasticsearch.md

@@ -192,4 +192,4 @@ The array fields of this structure can be defined by using the following command
     
 ## Limitations
 1. openLooKeng does not support to query table in Elasticsearch which has duplicated columns, such as column "name" and "NAME";
-2. openLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.
+2. opneLooKeng does not support to query the table in Elasticsearch that the name has special characters, such as '-', '.', etc.

hetu-docs/en/connector/hive.md

@@ -620,36 +620,6 @@ The following operations are not supported when `avro_schema_url` is set:
     columns are not supported in `CREATE TABLE`.
 -   `ALTER TABLE` commands modifying columns are not supported.
 
-## Drop Column Behavior
-
-Syntax supported for Drop Column is as follows:
-
-```sql
-ALTER TABLE 'name' DROP COLUMN 'column_name'
-```
-
-In case of Hive connector, DROP COLUMN drops column which is at the end of existing columns. Hive doesn't support DROP COLUMN syntax, however closest DDL supported by Hive which enumerates the above is REPLACE COLUMNS. REPLACE COLUMNS removes all existing columns and adds the new set of columns whereas DROP COLUMN does remove all existing columns and add the same set of columns excluding column specified in the query, manifesting as it dropped the column.
-
-For example, consider a table with columns **a**, **b** and **c**.
-
-```sql
-lk:default> SELECT * FROM hive_table;
- a | b  |  c
----+----+-----
- 1 | 10 | 100
-(1 row)
-```
-
- On dropping column **a**:
-
-```sql
-lk:default> SELECT * FROM hive_table;
-  b | c
----+----
- 1 | 10
-(1 row)
-```
-
 ## Procedures
 
 
@@ -746,7 +716,7 @@ Drop a schema:
 DROP SCHEMA hive.web
 ```
 
-## Metastore Cache
+## Metastore Cache:
 
 Hive connector maintains a metastore cache to service the metastore request faster to various operations. Loading, reloading and retention times of the cache entries can be configured in `hive.properties`.
 
@@ -770,7 +740,7 @@ REFRESH META CACHE
 Additionally, metadata cache refresh command can be used to reload the metastore cache by user.
 
 
-## Performance tuning notes
+## Performance tuning notes:
 
 #### INSERT
 

hetu-docs/en/connector/kafka-tutorial.md

@@ -565,4 +565,4 @@ lk:default> SELECT created_at, raw_date FROM (
 (5 rows)
 ```
 
-The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or milliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.
+The Kafka connector contains converters for ISO 8601, RFC 2822 text formats and for number-based timestamps using seconds or miilliseconds since the epoch. There is also a generic, text-based formatter which uses Joda-Time format strings to parse text columns.

hetu-docs/en/connector/memory.md

@@ -37,7 +37,7 @@ memory.spill-path=/opt/hetu/data/spill
   hetu.metastore.cache.type=local
   ```
   ##### Multi-Node Setup
-  - This section will give an example configuration for Memory Connector and a cluster with more than one node.
+  - This section will give an example configuration for Memory Connector an a cluster with more than one node.
   - Create a file `etc/catalog/memory.properties` with the following information:
   ``` properties
   connector.name=memory
@@ -80,10 +80,10 @@ memory.spill-path=/opt/hetu/data/spill
 **Note:**
 - `spill-path` should be set to a directory with enough free space to hold
  the table data.
-- See [**Configuration Properties**](#configuration-properties) section for additional properties and
+- See **Configuration Properties** section for additional properties and
  details.
-- In `etc/config.properties` ensure that `task.writer-count` is set
+- In `etc/config.properties` ensure that `task.writer-count` is set to
- `>=` to number of nodes in the cluster running openLooKeng. This will help
+ `>=` number of nodes in the cluster running openLooKeng. This will help
   distribute the data uniformly between all the workers.
 
 Examples
@@ -112,45 +112,17 @@ Create a table using the Memory Connector with sorting, indices and spill compre
     CREATE TABLE memory.default.nation
     WITH (
         sorted_by=array['nationkey'],
-        partitioned_by=array['regionkey'],
+        index_columns=array['name', 'regionkey'],
-        index_columns=array['name'],
         spill_compression=true
     )
     AS SELECT * from tpch.tiny.nation;
 
 After table creation completes, the Memory Connector will start building indices and sorting data in the background. Once the processing is complete any queries using the sort or index columns will be faster and more efficient.
 
-For now, `sorted_by` and `partitioned_by` only accepts a single column.
+For now, `sorted_by` only accepts a single column.
 
-Memory and Disk Usage via JMX
------------------------------
-JMX can be used to show memory and disk usage of memory connector tables
-Please refer to [JMX Connector](./jmx.md) for setup
 
-The `io.prestosql.plugin.memory.data:name=MemoryTableManager` table of `jmx.current` contains information on all the tables' memory and disk usage size in bytes
+Configuration Properties
-
-    SELECT * FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
-
-```
- currentbytes | alltablesdiskbyteusage | alltablesmemorybyteusage |   node   |                       object_name                       
---------------+------------------------+--------------------------+----------+---------------------------------------------------------
-           23 |                   3456 |                       23 | example1 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
-          253 |                   8713 |                      667 | example2 | io.prestosql.plugin.memory.data:name=MemoryTableManager 
-```
-
-Not all tables will be in memory since some may have been spilled to disk. `currentbytes` column will show the current memory occupied by tables which are in memory now.
-
-The usage for each node is shown as a separate row, aggregation functions can be utilized to show total usage across the cluster. For example, to view total disk or memory usage on all nodes, run:
-
-    SELECT sum(alltablesdiskbyteusage) as totaldiskbyteusage, sum(alltablesmemorybyteusage) as totalmemorybyteusage FROM jmx.current."io.prestosql.plugin.memory.data:name=MemoryTableManager";
-
-```
-totaldiskbyteusage | totalmemorybyteusage
--------------------+---------------------
-             12169 |                  690
-```
-
-# Configuration Properties
 ------------------------
 
 | Property Name                         | Default Value   | Required| Description               |
@@ -161,50 +133,31 @@ totaldiskbyteusage | totalmemorybyteusage
 | `memory.max-page-size                `  | 512KB         | No      | Memory limit for each page. Default value is recommended.|
 | `memory.logical-part-processing-delay`  | 5s            | No      | The delay between when the table is created/updated and LogicalPart processing starts. Default value is recommended.|
 | `memory.thread-pool-size             `  | Half of threads available to the JVM | No      | Maximum threads to allocate for background processing (e.g. sorting, index creation, cleanup, etc)|
-| `memory.table-statistics-enabled`       | False         | No      | When enabled, user can run analyze to collect statistics and leverage that information for accelerating queries.|
 
-Path whitelist:  `["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
+Path whitelist：`["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]`
 
 Additional WITH properties
---------------------------
+--------------
 Use these properties when creating a table with the Memory Connector to make queries faster.
 
 | Property Name            | Argument type             | Requirements                     | Description|
 |--------------------------|---------------------------|----------------------------------|------------|
 | sorted_by                | `array['col']`            | Maximum of one column. Column type must be comparable.  | Sort and create indexes on the given column|
-| partitioned_by           | `array['col']`            | Maximum of one column. | Partition the table on the given column|
 | index_columns            | `array['col1', 'col2']`   | None                             | Create indexes on the given column|
 | spill_compression        | `boolean`           | None                             | Compress data when spilling to disk|
 
 
 Index Types
 --------------
-These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`.
+These are the types of indices that are built on the columns you specify in `sorted_by` or `index_columns`. If a query operator is not supported by a particular index you can still
-If a query operator is not supported by a particular index, you can still use that operator, but the query will not benefit from the index.
+use that operator, but the query will not benefit from the index.
 
 | Index ID                          |Built for Columns In | Supported query operators             |
 |-----------------------------------|----------------------------------------|---------------------------------------|
-| Bloom   | `index_columns`                                 | `=` `IN`                             |                   
+| Bloom   | `sorted_by,index_columns`                                 | `=` `IN`                             |                   
 | MinMax  | `sorted_by,index_columns`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 | Sparse  | `sorted_by`                            | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` |
 
-
-Using statistics
------------------
-If the statistic configuration is enabled, you can refer to the example below to use it.
-
-Create a table using the Memory Connector:
-
-    CREATE TABLE memory.default.nation AS
-    SELECT * from tpch.tiny.nation;
-
-Run Analyze to collect the statistic information:
-
-    ANALYZE memory.default.nation;
-
-And then run the queries. Note that currently we do not support automatic statistic update, so you will need to run ANALYZE again if the table is updated.
-
-
 Developer Information
 ----------------------------
 
@@ -213,11 +166,20 @@ This section outlines the overall design of the Memory Connector, as shown in th
 ![Memory Connector Overall Design](../images/memory-connector-design.png)
 
 ### Scheduling Process
-The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng. In the Memory Connector, each worker has several LogicalParts. During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion. Table data will be automatically spilled to disk as part of a background process as well.  If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule. HetuMetastore is used to persist table metadata. At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
+The data to be processed are stored in pages, which are distributed to different worker nodes in openLooKeng.
+In the Memory Connector, each worker has a number of LogicalParts.
+During table creation, LogicalParts in the workers are filled with the input pages in a round-robin fashion.
+Table data will be automatically spilled to disk as part of a background process as well. 
+If there is not enough memory to hold the entire data, the tables can be released from memory according to LRU rule.
+HetuMetastore is used to persist table metadata.
+At query time, when Tablescan operation is scheduled, the LogicalParts will be scheduled.
 
 ### LogicalPart
-As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data. The sorting and indexing are handled in a background process allowing faster querying,
+As shown in the lower part of the design figure, LogicalPart is the data structure that contains both indexes and data.
-but the table is still queriable during processing. LogicalParts have a maximum configurable size (default 256 MB).  New LogicalParts are created once the previous one is full.
+The sorting and indexing are handled in a background process allowing faster querying,
+but the table is still queriable during processing.
+LogicalParts have a maximum configurable size (default 256 MB). 
+New LogicalParts are created once the previous one is full.
 
 
 ### Indices
@@ -252,4 +214,3 @@ Limitations and known Issues
 - Without State Store and Hetu Metastore with global cache, after `DROP TABLE`, memory is not released immediately on the workers. It is released on the next `CREATE TABLE` operation.
 - Currently only a single column in ascending order is supported by `sorted_by`
 - If a CTAS (CREATE TABLE AS) query fails or is cancelled, an invalid table will remain. This table must be dropped manually.
-- And we support BOOLEAN, All INT Types, CHAR, VARCHAR, DOUBLE, REAL, DECIMAL, DATE, TIME, UUID types as partition keys.

hetu-docs/en/connector/omnidata.md

@@ -1,103 +0,0 @@
-
-OmniData Connector
-==============
-
-## Overview
-
-The OmniData connector allows querying data stored in the remote Hive data warehouse.
-It pushes the operators of openLooKeng down to the storage node to achieve near-data calculation, thereby reducing the amount of network transmission data and improving computing performance.
-
-For more information, please see: [OmniData](https://www.hikunpeng.com/en/developer/boostkit/big-data?accelerated=3) and [OmniData connector](https://github.com/kunpengcompute/omnidata-openlookeng-connector).
-
-## Supported File Types
-
-
-The following file types are supported for the OmniData connector:
-
--   ORC
--   Parquet
--   Text
-
-## Configuration
-
-Create `etc/catalog/omnidata.properties` with the following configurations, replacing `example.net:9083` with the correct host and port for your Hive metastore Thrift service:
-
-``` properties
-connector.name=omnidata-openlookeng
-hive.metastore.uri=thrift://example.net:9083
-```
-
-### HDFS Configuration
-
-For basic setups, openLooKeng configures the HDFS client automatically and does not require any configuration files. In some cases, such as when using federated HDFS or NameNode high availability, it is necessary to specify additional HDFS client options in order to access your HDFS cluster. To do so, add the `hive.config.resources` property to reference your HDFS config files:
-
-``` properties
-hive.config.resources=/etc/hadoop/conf/core-site.xml,/etc/hadoop/conf/hdfs-site.xml
-```
-
-Only specify additional configuration files if necessary for your setup. We also recommend reducing the configuration files to have the minimum set of required properties, as additional properties may cause problems.
-
-The configuration files must exist on all openLooKeng nodes. If you are referencing existing Hadoop config files, make sure to copy them to any openLooKeng nodes that are not running Hadoop.
-
-## OmniData Configuration Properties
-
-| Property Name                   | Description                                                  | Default |
-| ------------------------------- | ------------------------------------------------------------ | ------- |
-| hive.metastore                  | The type of Hive metastore                                   | thrift  |
-| hive.config.resources           | An optional comma-separated list of HDFS configuration files. These files must exist on the machines running openLooKeng. Only specify this if absolutely necessary to access HDFS. Example: `/etc/hdfs-site.xml` |         |
-| hive.omnidata-enabled           | Allows push-down operators to execute on the storage side. If disabled, all operators will not be pushed down. | true    |
-| hive.min-offload-row-number     | If the number of rows in the table is less than the threshold, all operators of the table will not be pushed down. | 500     |
-| hive.filter-offload-enabled     | Allows the filter operator to be pushed down to the storage side. If disabled, the filter operator will not be pushed down. | true    |
-| hive.filter-offload-factor      | Only when the selection rate of the filter operator is less than the threshold, it will be pushed down. | 0.25    |
-| hive.aggregator-offload-enabled | Allows the aggregator operator to be pushed down to the storage side. If disabled, the aggregator operator will not be pushed down. | true    |
-| hive.aggregator-offload-factor  | Only when the aggregation rate of the aggregator operator is less than the threshold, it will be pushed down. | 0.25    |
-
-For more configuration, please refer to the [Hive Configuration Properties](./hive.md#Hive Configuration Properties) chapter.
-
-### Querying OmniData
-
-The SQL query plan after some operators are pushed down:
-
-```sql
-lk:tpch_flat_orc_date_1000> explain select sum(l_extendedprice * l_discount) as revenue
-				 		 -> from
-				 		 -> lineitem
-				 		 -> where
-				 		 -> l_shipdate >= DATE '1993-01-01'
-				 		 -> and l_shipdate < DATE '1994-01-01'
-				 		 -> and l_discount between 0.06 - 0.01 and 0.06 + 0.01
-				 		 -> and l_quantity < 25;
-				 							Query Plan
-------------------------------------------------------------------------------------------------------
-Output[revenue]
-│ Layout: [sum:double]
-│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
-│ revenue := sum
-└─ Aggregate(FINAL)
-│ Layout: [sum:double]
-│ Estimates: {rows: 4859991664 (40.74GB), cpu: 246.43G, memory: 86.00GB, network: 45.26GB}
-│ sum := sum(sum_4)
-└─ LocalExchange[SINGLE] ()
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
-└─ RemoteExchange[GATHER]
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 45.26GB}
-└─ Aggregate(PARTIAL)
-│ Layout: [sum_4:double]
-│ Estimates: {rows: 5399990738 (45.26GB), cpu: 201.17G, memory: 45.26GB, network: 0B}
-│ sum_4 := sum(expr)
-└─ ScanProject[table = hive:tpch_flat_orc_date_1000:lineitem offload={ filter=[AND(AND(BETWEEN(l_discount, 0.05, 0.07), LESS_THAN(l_quantity, 25.0)), AND(GREATER_THAN_OR_EQUAL(l_shipdate, 8401), LESS_THAN(l_shipdate, 8766)))]} ]
- Layout: [expr:double]
- Estimates: {rows: 5999989709 (50.29GB), cpu: 100.58G, memory: 0B, network: 0B}/{rows: 5999989709 (50.29GB), cpu: 150.87G, memory: 0B, network: 0B}
- expr := (l_extendedprice) * (l_discount)
- l_extendedprice := l_extendedprice:double:5:REGULAR
- l_discount := l_discount:double:6:REGULAR
-```
-
-
-## OmniData Connector Limitations
-
-
-- The OmniData service needs to be deployed on the storage node.
-- Only the pushdown of Filter, Aggregator, and Limit operators are supported.

hetu-docs/en/connector/opengauss.md

@@ -45,95 +45,6 @@ Finally, you can access the `hetutb` table in the `public` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `opengauss` in the above examples.
 
-## openGauss Update/Delete Support
-
-### Create openGauss Table
-
-Example：
-
-```sql
-CREATE TABLE opengauss_table (
-    id int,
-    name varchar(255));
-```
-
-### INSERT on openGauss tables
-
-Example：
-
-```sql
-INSERT INTO opengauss_table
-  VALUES
-     (1, 'Jack'),
-     (2, 'Bob');
-```
-
-### UPDATE on openGauss tables
-
-Example：
-
-```sql
-UPDATE opengauss_table
-  SET name='Tim'
-  WHERE id=1;
-```
-
-Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
-
-SELECT result before UPDATE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
-id | name
-----+------
-  1 | Jack
-  2 | Bob
-(2 rows)
-```
-
-SELECT result after UPDATE
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
-id | name
-----+------
-2 | Bob
-1 | Tim
-(2 rows)
-```
-
-### DELETE on openGauss tables
-
-Example：
-
-```sql
-DELETE FROM opengauss_table
-  WHERE id=2;
-```
-
-Above example delete the rows with column `id` having value `2`.
-
-SELECT result before DELETE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
- id | name
-----+------
-  2 | Bob
-  1 | Tim
-(2 rows)
-```
-
-SELECT result after DELETE:
-
-```sql
-lk:default> SELECT * FROM opengauss_table;
- id | name
-----+------
-  1 | Tim
-(1 row)
-```
-
 ****Note:****
 
 > - When the compatibility type of the openGuass database is O (DBCOMPATIBILITY = A), the `Date` data type is not supported.
@@ -153,4 +64,4 @@ openGauss Connector Limitations
 
 The following SQL statements are not yet supported:
 
-[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/postgresql.md

@@ -46,98 +46,9 @@ Finally, you can access the `clicks` table in the `web` schema:
 
 If you used a different name for your catalog properties file, use that catalog name instead of `postgresql` in the above examples.
 
-## PostgreSQL Update/Delete Support
-
-### Create PostgreSQL Table
-
-Example：
-
-```sql
-CREATE TABLE postgresql_table (
-    id int,
-    name varchar(255));
-```
-
-### INSERT on PostgreSQL tables
-
-Example：
-
-```sql
-INSERT INTO postgresql_table
-  VALUES
-     (1, 'Jack'),
-     (2, 'Bob');
-```
-
-### UPDATE on PostgreSQL tables
-
-Example：
-
-```sql
-UPDATE postgresql_table
-  SET name='Tim'
-  WHERE id=1;
-```
-
-Above example updates the column `name`'s value to `Tim` of rows with column `id` having value `1`.
-
-SELECT result before UPDATE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
-id | name
-----+------
-  1 | Jack
-  2 | Bob
-(2 rows)
-```
-
-SELECT result after UPDATE
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
-id | name
-----+------
-2 | Bob
-1 | Tim
-(2 rows)
-```
-
-### DELETE on PostgreSQL tables
-
-Example：
-
-```sql
-DELETE FROM postgresql_table
-  WHERE id=2;
-```
-
-Above example delete the rows with column `id` having value `2`.
-
-SELECT result before DELETE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
- id | name
-----+------
-  2 | Bob
-  1 | Tim
-(2 rows)
-```
-
-SELECT result after DELETE:
-
-```sql
-lk:default> SELECT * FROM postgresql_table;
- id | name
-----+------
-  1 | Tim
-(1 row)
-```
-
 PostgreSQL Connector Limitations
 --------------------------------
 
 The following SQL statements are not yet supported:
 
-[GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)
+[DELETE](../sql/delete.md), [GRANT](../sql/grant.md), [REVOKE](../sql/revoke.md), [SHOW GRANTS](../sql/show-grants.md), [SHOW ROLES](../sql/show-roles.md), [SHOW ROLE GRANTS](../sql/show-role-grants.md)

hetu-docs/en/connector/redis.md

@@ -1,173 +0,0 @@
-Redis Connector
-====================
-Overview
---------
-this connector allows the use of Redis key/value pair is presented as a single row in openLooKeng.
-
-**Note**
-
-*In Redis,key/value pair can only be mapped to string or hash value types.keys can be stored in a zset,then keys can split into multiple slice*
-
-*Support Redis 2.8.0 or higher*
-
-Configuration
--------------
-To configure the Redis connector, create a catalog properties file `etc/catalog/redis.properties` with the following contents, replacing the properties as appropriate:
-``` properties
-connector.name=redis
-redis.table-names=schema1.table1,schema1.table2
-redis.nodes=host1:port
-```
-### Multiple Redis Servers
-You can have as many catalogs as you need. If you have additional
-Redis servers, simply add another properties file to ``etc/catalog``
-with a different name, making sure it ends in ``.properties``.
-For example, if you name the property file `sales.properties`, openLooKeng will create a catalog named `sales` using the configured connector.
-
-Configuration properties
-------------------------
-The following configuration properties are available:
-
-| Property Name                                                    | Description                                                                                                           |
-|:-----------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
-| `redis.table-names`                                              | List of all tables provided by the catalog                                                                            |
-| `redis.default-schema`                                           | Default schema name for tables  (default `default`)                                                                   |
-| `redis.nodes`                                                    | List of nodes in the Redis server                                                                                     |
-| `redis.connect-timeout`                                          | Timeout for connecting to the  Redis server (ms) (default 2000)                                                       |
-| `redis.scan-count`                                               | The number of keys obtained from each scan for string and hash value types (default 100)                              |
-| `redis.key-prefix-schema-table`                                  | Redis keys have schema-name:table-name prefix   (default  false)                                                      |
-| `redis.key-delimiter`                                            | Delimiter separating schema_name and table_name if redis.key-prefix-schema-table is used (default `:`)                |
-| `redis.table-description-dir`                                    | Directory containing table description files (default `etc/redis/`)                                                   |
-| `redis.hide-internal-columns`                                    | Whether internal columns are shown in table metadata or not. (default true)                                           |
-| `redis.database-index`                                           | Redis database index  (default 0)                                                                                     |
-| `redis.password`                                                 | Redis server password  (default null)                                                                                 |
-| `redis.table-description-interval`                               | the interval of flush description files (ms) (default no flush,table description will be memoized without expiration) |
-
-
-Internal columns
-----------------
-
-| Column name        | Type    | Description                                                                                                                              |
-|:-------------------| :------ |:-----------------------------------------------------------------------------------------------------------------------------------------|
-| `_key`             | VARCHAR  | Redis key.                                                                                                                               |
-| `_value`           | VARCHAR   | Redis value corresponding to the key                                                                                                     |
-| `_key_length`   | BIGINT  | Number of bytes in the key.                                                                                                              |
-| `_key_corrupt`     | BOOLEAN  | True if the decoder could not decode the key for this row. When true, data columns mapped from the key should be treated as invalid.     |
-| `_value_corrupt`   | BOOLEAN | True if the decoder could not decode the value for this row. When true, data columns mapped from the value should be treated as invalid. |
-
-
-Table Definition Files
-----------------------
-For openLooKeng, every key/value pair  must be mapped into columns to allow queries against the data. It is like kafka conntector,so you can refer to kafka-tutorial
-
-A table definition file consists of a JSON definition for a table. The name of the file can be arbitrary but must end in `.json`.
-
-for example,there is a nation.json
-``` json
-{
-    "tableName": "nation",
-    "schemaName": "tpch",
-    "key": {
-        "dataFormat": "raw",
-        "fields": [
-            {
-                "name": "redis_key",
-                "type": "VARCHAR(64)",
-                "hidden": "true"
-            }
-        ]
-    },
-    "value": {
-        "dataFormat": "json",
-        "fields": [
-            {
-                "name": "nationkey",
-                "mapping": "nationkey",
-                "type": "BIGINT"
-            },
-            {
-                "name": "name",
-                "mapping": "name",
-                "type": "VARCHAR(25)"
-            },
-                        {
-                "name": "regionkey",
-                "mapping": "regionkey",
-                "type": "BIGINT"
-            },
-            {
-                "name": "comment",
-                "mapping": "comment",
-                "type": "VARCHAR(152)"
-            }
-       ]
-    }
-}
-```
-In redis,such data exists
-```shell
-127.0.0.1:6379> keys tpch:nation:*
- 1) "tpch:nation:2"
- 2) "tpch:nation:4"
- 3) "tpch:nation:16"
- 4) "tpch:nation:18"
- 5) "tpch:nation:10"
- 6) "tpch:nation:17"
- 7) "tpch:nation:1"
-```
-```shell
-127.0.0.1:6379> get tpch:nation:1
-"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
-```
-Now we can use redis connector get data from redis,(redis_key don't show,because we set "hidden": "true" ) 
-```shell
-lk> select * from redis.tpch.nation;
- nationkey |      name      | regionkey |                                                      comment                                                       
------------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
-         3 | CANADA         |         1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold              
-         9 | INDONESIA      |         2 |  slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull 
-        19 | ROMANIA        |         3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account    
-         2 | BRAZIL         |         1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special   
-```
-**Note**
-
-*if redis.key-prefix-schema-table is false (default is false),all keys in redis will be mapped to table's key,no matching occurs*
-
-Please refer to the `kafka-tutorial` for the description of the ``dataFormat`` as well as various available decoders.
-
-In addition to the above Kafka types, the Redis connector supports ``hash`` type for the ``value`` field which represent data stored in the Redis hash.
-Redis connector use `hgetall key` to get data.
-``` json
-      {
-      "tableName": ...,
-      "schemaName": ...,
-      "value": {
-        "dataFormat": "hash",
-        "fields": [
-          ...
-        ]
-      }
-    }
-```
-
-the Redis connector supports ``zset`` type for the ``key`` field which represent key stored in the Redis zset. 
-if and only if ``zset`` is used as key datafomart,the split is truly supported , because we can use `zrange zsetkey split.start split.end` to get keys of a split.
-``` json
-      {
-      "tableName": ...,
-      "schemaName": ...,
-      "key": {
-        "dataFormat": "zset",
-        "name": "zsetkey", //zadd zsetkey score member
-        "fields": [
-            ...
-        ]
-      }
-    }
-```
-
-Redis Connector Limitations
----------------------------
-only support read operation,don't support write operation.
-
-

hetu-docs/en/develop/_index.md

@@ -1,5 +1,5 @@
 
 # Developer Guide
 
-openLooKeng is based on Trino 316(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
+openLooKeng is based on Trino(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
 

hetu-docs/en/develop/connectors.md

@@ -21,7 +21,7 @@ This interface is too big to list in this documentation, but if you are interest
 connector. If your underlying data source supports schemas, tables and columns, this interface should be straightforward to implement. If you are attempting to adapt something that is not a relational database (as
 the Example HTTP connector does), you may need to get creative about how you map your data source to openLooKeng\'s schema, table, and column concepts.
 
-### ConnectorSplitManager
+### ConnectorSplitManger
 
 The split manager partitions the data for a table into the individual chunks that openLooKeng will distribute to workers for processing. For example, the Hive connector lists the files for each Hive partition and creates
 one or more split per file. For data so urces that don\'t have partitioned data, a good strategy here is to simply return a single split for the entire table. This is the strategy employed by the Example HTTP connector.

hetu-docs/en/develop/externalfunction-registration-pushdown.md

@@ -3,7 +3,7 @@ External Function Registration and Push Down
 
 Introduction
 ------------
-The connector can register `external function` into openLooKeng. In Jdbc connector, openLooKeng can push them down to data source which support to execute those functions.
+The connector can register `external function` into openLooKeng. In Jdbc connector, openLookeng can push them down to data source which support to execute those functions.
 
 Function Registration in Connector
 ----------------------------------

hetu-docs/en/develop/functions.md

@@ -216,7 +216,7 @@ An in-depth look at the various annotations relevant to writing an aggregation f
     
 -   `@CombineFunction`:
 
-    The `@CombineFunction` annotation declares the function used to combine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
+    The `@CombineFunction` annotation declares the function used to ombine two state objects. This function is used to merge all the partial aggregation states. It takes two state objects, and merges
     the results into the first one (in the above example, just by adding them together).
     
 -   `@OutputFunction`:

hetu-docs/en/develop/getting-started.md

@@ -5,7 +5,7 @@
 
 * Mac OS X or Linux
 * Java 8 Update 161 or higher (8u161+), 64-bit. Both Oracle JDK and OpenJDK are supported.
-* AArch64 ([Bisheng JDK 1.8.262 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
+* AArch64 ([Bisheng JDK 11 or higher](https://www.hikunpeng.com/developer/devkit/compiler?data=JDK))
 * Maven 3.3.9+ (for building)
 * Python 2.4+ (for running with the launcher script)
 

hetu-docs/en/develop/star-tree-cube.md

@@ -0,0 +1,148 @@
+# Star-Tree 
+
+Star tree cubing is a pre-aggregation technique to achieve low latency runtime for iceberg queries. An iceberg query computes an aggregate function over 
+an attribute ( or set of attributes) in order to find aggregate values above a specified threshold. Using this technique, User is provided with an option to 
+create a cube with necessary aggregations and dimensions. Then, when aggregation queries are executed, the cube is used to executing the query instead of the 
+original table. The actual performance gain is achieved during the TableScan operation as cubes are pre-computed and pre-aggregated. 
+
+For this reason, the cubing technique is highly effective when the group by cardinality results in lesser rows than the original table.
+
+## Supported functions
+    COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
+
+## Enabling and Disabling Star-tree
+To enable:
+```sql 
+SET SESSION enable_star_tree_index=true;
+```
+To disable:
+```sql 
+SET SESSION enable_star_tree_index=false;
+```
+
+## Configuration Properties
+| Property Name                                     | Default Value       | Required| Description|
+|---------------------------------------------------|---------------------|---------|--------------|
+| optimizer.enable-star-tree-index                  | false               | No      | Enables star-tree index|
+| cube.metadata-cache-size                          | 5                   | No      | The maximum number of metadata for star-trees that could be loaded into cache before eviction happens|
+| cube.metadata-cache-ttl                           | 1h                  | No      | The maximum time to live of star-trees that are be loaded into cache before eviction happens |
+
+## Examples
+
+Creating a star-tree cube:
+```sql 
+CREATE CUBE nation_cube 
+ON nation 
+WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
+GROUP=(nationkey),
+format='orc', partitioned_by=ARRAY['nationkey']);
+```
+Next, to add data to the cube:
+```sql 
+INSERT INTO CUBE nation_cube WHERE nationkey >= 5;
+```
+Creating a star-tree cube with CLI with WHERE clause:
+```sql 
+CREATE CUBE nation_cube 
+ON nation 
+WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
+GROUP=(nationkey),
+format='orc', partitioned_by=ARRAY['nationkey'])
+WHERE nationkey >= 5;
+```
+
+To use the new cube, just query the original table using aggregations that were included in the cube:
+```sql 
+SELECT count(*) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
+SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
+```
+
+Since the data inserted into the cube was for nationkey >= 5, only queries matching this condition will utilize the cube. 
+Queries not matching the condition will still work as usual.
+
+## Optimizer Changes
+
+The star tree aggregation rule is an Iterative optimizer that optimizes the logical plan by replacing the original aggregation sub-tree
+and original table scan with pre-aggregation table scan. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can
+be supported by the Cubes. The exact rows are not queried to check if Cube is applicable or not.
+
+## Dependencies
+
+Star Tree index relies on Hetu metastore to store the cube related metadata.
+Please check [Hetu Metastore](../admin/meta-store.md) for more information.
+
+### Building Cube for Large dataset
+One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
+Processing large number rows requires more memory than cluster is configured with. This results in query failing with message "Query exceeded per-node user memory 
+limit". To overcome this issue, "Insert into cube" query support was added. The user has ability to build a cube for larger data by executing multiple 
+insert into cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into cube.
+
+This section explains the steps to build a cube for larger dataset. 
+
+Let us take an example of TPCDS dataset and `store_sales` table. The table has 10 years worth of data 
+and user wants to build a cube for year 2001 and due to the cluster memory limit, the entire data set for year 2001 cannot be processed at once.
+
+```sql
+
+CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk));
+
+SELECT min(d_date_sk) as year_start, max(d_date_sk) as year_end FROM date_dim WHERE d_year = 2001;
+ year_start | year_end 
+------------+----------
+    2451911 |  2452275 
+(1 row)
+
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275; 
+-- This could result in query failure if the number of rows need to be processed is huge and the query memory exceeds the configured limit.
+
+To overcome this issue, multiple insert statements can be used into process rows and insert into cube and the number of rows canbe controlled by using where clause;
+
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2452010;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk >= 2452011 AND ss_sold_date_sk <= 2452110;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452111 AND 2452210;
+INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452211 AND 2452275;
+
+Alternative to overcome this issue, use CLI if expression is Between Predicate or Comparison Expression. CLI internally queries multiple insert statements.
+
+CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk)) WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275;
+
+Internally the system will rewrite and merge all continuous range predicates into a single predicate;
+
+SHOW CUBES;
+
+           Cube Name             |         Table Name         | Status |         Dimensions          |                     Aggregations                      |                                     Where Clause                                     
+---------------------------------+----------------------------+--------+-----------------------------+-------------------------------------------------------+-------------------------------------------------------+------------------------------
+ hive.tpcds_sf1.store_sales_cube | hive.tpcds_sf1.store_sales | Active | ss_sold_date_sk,ss_store_sk | sum(ss_sales_price),sum(ss_net_paid),sum(ss_quantity) | (("ss_sold_date_sk" >= BIGINT '2451911') AND ("ss_sold_date_sk" < BIGINT '2452276')) 
+
+Note:
+1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together. 
+   All continous predicates will be merged into a single range predicate and remainining predicates are untouched.
+   Only the following types are supported and can be merged together. 
+   Integer, TinyInt, SmallInt, BigInt, Date;
+   For other types, its difficult to identify if two predicates are continous therefore they cannot be merged together. And because of this issue, there is 
+   possibility that particular cube may not be used during query optimisation even if the cube has all the required data. For example,
+   
+   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
+   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
+   
+   Here these two predicates cannot be merged into store_id BETWEEN 'A01' AND 'A20'; So the cube won't be used 
+   for queries that are spanning over two the predicates;
+   
+   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube won't be used for optimizing this query. This is a limitation as of now.
+
+2. Only Single column predicates can be merged. 
+```
+
+#CLI changes
+
+The CLI supports the create cube statement with where clause. The where clause used in the create cube statement is the range of data to be inserted into the cube.
+Once the user runs create cube statement with where clause then the cli internally runs the insert cube statements. This process improves the user experience and
+improves the memory footprint based on the cluster memory limits. As of now, only Between Predicate and Comparison Expressions are supported. We support only Integer and Long Literals.
+
+## Limitation
+
+1. Star tree cube is only effective when the group by cardinality is considerably lower than the number of rows in 
+   source table.
+2. A significant amount of user effort required in maintaining Cubes for large datasets.
+3. Only incremental insert into cube is supported. Cannot delete specific rows from Cube.
+4. For the create cube statement in CLI, we only support Between Predicate or Comparison Expression. We support only Integer and Long Literal.

hetu-docs/en/develop/types.md

@@ -13,7 +13,7 @@ Below is a high level overview of the `Type` interface, for more details see the
     
 -   Native encoding:
 
-    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the Unix epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
+    The interpretation of a value in its native container type form is defined by its `Type`. For some types, such as `BigintType`, it matches the Java interpretation of the native container type (64bit 2\'s complement). However, for other types such as `TimestampWithTimeZoneType`, which also uses `long` for its native container type, the value stored in the `long` is a 8byte binary value combining the timezone and the milliseconds since the unix  epoch. In particular, this means that you cannot compare two native values and expect a meaningful result, without knowing the native encoding.
     
 -   Type signature:
 

hetu-docs/en/faq/faq.md

@@ -40,10 +40,6 @@
    >
    > openLooKeng Bilibili channel:  https://space.bilibili.com/627629884   
 
-8. Which version of Trino is openLooKeng developed on?
-
-   > Based on Trino 316 version development.
-
 ## Functions
 
 1. What connectors does the openLooKeng support?

hetu-docs/en/functions/conversion.md

@@ -106,7 +106,7 @@ Obviously, not all data types are compatible with each other, below table lists
 
 **Note:**
 
-- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. Please refer to below item.
+- Y or Y(#): standard for support implicit convert. But there might be some limitation need your attention. please refer to below item.
 - N: standard for not support implicit convert
 
 (1): BOOLEAN-\>NUMBER the converted result can be only 0 or 1
@@ -123,7 +123,7 @@ Obviously, not all data types are compatible with each other, below table lists
 
 (7): VARCHAR-\>BOOLEAN only \'0\',\'1\',\'TRUE\',\'FALSE\' can be converted. Others will be failed
 
-(8): VARCHAR-\>DECIMAL conversion will fail when it's not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
+(8): VARCHAR-\>DECIMAL conversion will fail when its not an numeric or the converted value is out of range of DECIMAL. Scale will be cut off when out of range.
 
 (9): VARCHAR-\>CHAR if length of VARCHAR is larger than CHAR, it will be cut off.
 

hetu-docs/en/functions/geospatial.md

@@ -245,7 +245,7 @@ Returns `true` if this Geometry is an empty geometrycollection, polygon, point e
 
 **ST\_IsSimple(Geometry)** -\> boolean
 
-Returns `true` if this Geometry has no anomalous geometric points, such as self-intersection or self-tangency.
+Returns `true` if this Geometry has no anomalous geometric points, such as self intersection or self tangency.
 
 **ST\_IsRing(Geometry)** -\> boolean
 

hetu-docs/en/index.md

@@ -46,7 +46,6 @@ headless: true
     - [Audit Log]({{< relref "./docs/admin/audit-log.md" >}})
     - [Reliable Execution]({{< relref "./docs/admin/reliable-execution.md" >}})
     - [JDBC Data Source Multi-Split Management]({{< relref "./docs/admin/multi-split-for-jdbc-data-source.md" >}})
-    - [Extension Physical Execution Planner]({{< relref "./docs/admin/extension-execution-planner.md" >}})
 
 - [Query Optimizer]("#")
     - [Table Statistics]({{< relref "./docs/optimizer/statistics.md" >}})
@@ -64,11 +63,6 @@ headless: true
     - [HIndex Statements]({{< relref "./docs/indexer/hindex-statements.md" >}})
     - [New Index]({{< relref "./docs/indexer/new-index.md" >}})
 
-- [Star Tree Cubes](#)
-    - [Overview]({{< relref "./docs/preagg/overview.md" >}})
-    - [Join Support]({{< relref "./docs/preagg/join-queries.md" >}})
-    - [Statements]({{< relref "./docs/preagg/statements.md" >}})
-  
 - [Connectors]({{< relref "./docs/connector/_index.md" >}})
     - [Carbondata]({{< relref "./docs/connector/carbondata.md" >}})
     - [ClickHouse]({{< relref "./docs/connector/clickhouse.md" >}})
@@ -84,7 +78,6 @@ headless: true
     - [JMX]({{< relref "./docs/connector/jmx.md" >}})
     - [Kafka]({{< relref "./docs/connector/kafka.md" >}})
     - [Kafka Connector Tutorial]({{< relref "./docs/connector/kafka-tutorial.md" >}})
-    - [Redis] ({{< relref "./docs/connector/redis.md" >}})
     - [Local File]({{< relref "./docs/connector/localfile.md" >}})
     - [Memory]({{< relref "./docs/connector/memory.md" >}})
     - [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
@@ -99,7 +92,6 @@ headless: true
     - [TPCH]({{< relref "./docs/connector/tpch.md" >}})
     - [VDM]({{< relref "./docs/connector/vdm.md" >}})
     - [Kylin]({{< relref "./docs/connector/kylin.md" >}})
-    - [OmniData]({{< relref "./docs/connector/omnidata.md" >}})
 
 - [Functions and Operators]("#")
     - [Logical Operators]({{< relref "./docs/functions/logical.md" >}})
@@ -140,6 +132,7 @@ headless: true
     - [CALL]({{< relref "./docs/sql/call.md" >}})
     - [COMMENT]({{< relref "./docs/sql/comment.md" >}})
     - [COMMIT]({{< relref "./docs/sql/commit.md" >}})
+    - [CREATE CUBE]({{< relref "./docs/sql/create-cube.md" >}})
     - [CREATE ROLE]({{< relref "./docs/sql/create-role.md" >}})
     - [CREATE SCHEMA]({{< relref "./docs/sql/create-schema.md" >}})
     - [CREATE TABLE]({{< relref "./docs/sql/create-table.md" >}})
@@ -151,6 +144,7 @@ headless: true
     - [DESCRIBE INPUT]({{< relref "./docs/sql/describe-input.md" >}})
     - [DESCRIBE OUTPUT]({{< relref "./docs/sql/describe-output.md" >}})
     - [DROP CACHE]({{< relref "./docs/sql/drop-cache.md" >}})
+    - [DROP CUBE]({{< relref "./docs/sql/drop-cube.md" >}})
     - [DROP ROLE]({{< relref "./docs/sql/drop-role.md" >}})
     - [DROP SCHEMA]({{< relref "./docs/sql/drop-schema.md" >}})
     - [DROP TABLE]({{< relref "./docs/sql/drop-table.md" >}})
@@ -162,6 +156,8 @@ headless: true
     - [GRANT ROLES]({{< relref "./docs/sql/grant-roles.md" >}})
     - [INSERT]({{< relref "./docs/sql/insert.md" >}})
     - [INSERT OVERWRITE]({{< relref "./docs/sql/insert-overwrite.md" >}})
+    - [INSERT CUBE]({{< relref "./docs/sql/insert-cube.md" >}})
+    - [INSERT OVERWRITE CUBE]({{< relref "./docs/sql/insert-overwrite-cube.md" >}})
     - [JMX]({{< relref "./docs/sql/jmx.md" >}})
     - [PREPARE]({{< relref "./docs/sql/prepare.md" >}})
     - [RESET SESSION]({{< relref "./docs/sql/reset-session.md" >}})
@@ -174,9 +170,9 @@ headless: true
     - [SHOW CACHE]({{< relref "./docs/sql/show-cache.md" >}})
     - [SHOW CATALOGS]({{< relref "./docs/sql/show-catalogs.md" >}})
     - [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
-    - [SHOW CREATE CUBE]({{< relref "./docs/sql/show-create-cube.md" >}})
     - [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
     - [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
+    - [SHOW CUBES]({{< relref "./docs/sql/show-cubes.md" >}})
     - [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
     - [SHOW EXTERNAL FUNCTION]({{< relref "./docs/sql/show-external-function.md" >}})
     - [SHOW GRANTS]({{< relref "./docs/sql/show-grants.md" >}})
@@ -210,6 +206,7 @@ headless: true
     - [Filesystem Access Utilities]({{< relref "./docs/develop/filesystem.md" >}})
     - [Hive ORC Cache]({{< relref "./docs/develop/hive-orc-cache.md" >}})
     - [External Function Registration and Push Down]({{< relref "./docs/develop/externalfunction-registration-pushdown.md" >}})
+    - [Star Tree Cube]({{< relref "./docs/develop/star-tree-cube.md" >}})
 
 - [openLooKeng REST API]({{< relref "./docs/rest/_index.md" >}})
     - [Node Resource]({{< relref "./docs/rest/node.md" >}})
@@ -219,11 +216,6 @@ headless: true
     - [Task Resource]({{< relref "./docs/rest/task.md" >}})
 
 - [Release Notes]("#")
-    - [1.6.1 (27 Apr 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.1.md" >}})
-    - [1.6.0 (30 Mar 2022)]({{< relref "./docs/releasenotes/releasenotes-1.6.0.md" >}})
-    - [1.5.0 (30 Dec 2021)]({{< relref "./docs/releasenotes/releasenotes-1.5.0.md" >}})
-    - [1.4.1 (12 Nov 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.1.md" >}})
-    - [1.4.0 (15 Oct 2021)]({{< relref "./docs/releasenotes/releasenotes-1.4.0.md" >}})
     - [1.3.0 (30 Jun 2021)]({{< relref "./docs/releasenotes/releasenotes-1.3.0.md" >}})
     - [1.2.0 (31 Mar 2021)]({{< relref "./docs/releasenotes/releasenotes-1.2.0.md" >}})
     - [1.1.0 (30 Dec 2020)]({{< relref "./docs/releasenotes/releasenotes-1.1.0.md" >}})

hetu-docs/en/indexer/hindex-statements.md

@@ -31,7 +31,7 @@ CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WITH ("bloom.
 CREATE INDEX index_name USING bloom ON hive.schema.table (column1) WHERE p in (part1, part2, part3);
 ```
 
-**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), for BTree index, only index creation on the **first** level is supported (colA). Bloom, Bitmap and Minmax index creation on either (colA or colB) is supported.
+**Note:** If the table is multi-partitioned (for example, partitioned by colA and colB), only index creation on the **first** level is supported (colA).
 
 ## SHOW
 

hetu-docs/en/indexer/new-index.md

@@ -10,13 +10,17 @@ The `getID()` method in the `Index` interface returns the ID of this index type
 
 ### Level
 
-A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e.  it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();`  which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
+A heuristic index stores additional and usually partial information of a dataset in a more compact way to speed up lookups in various ways. Therefore, each index must 
+have a domain on which it is applied. For instance, if an index marks the max value of a data set, we must know how big the data set is when we define the "max" value (i.e. 
+it can be the max of a group of rows, a data partition, or even a whole table). When a new index type is created, it must implement a method `Set<Level> getSupportedIndexLevels();` 
+which returns the data set level it can support. The levels are defined as an enum in `Index` interface.
 
 ## Interface overlook
 
 ### Indexing methods
 
-Apart from the methods mentioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete document on `Index` interface, please refer to the Java Doc of the source code.
+Apart from the methods metioned above, this section gives a quick guide on the most important methods needed to create a new index type. For the complete 
+document on `Index` interface, please refer to the Java Doc of the source code.
 
 There are two main functionalities in the `Index` interface:
 

hetu-docs/en/installation/deployment-auto.md

@@ -3,13 +3,13 @@
 
 
 
-In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete the deployment faster and easier. The script is friendly to most of Linux OS. However, to Ubuntu, you need to manually install the following dependencies:
+In addition to the manual deployment of openLooKeng Sever, you can follow below guide to complete deployment faster and easier. The script is friendly to most of linux OS. However, to Ubuntu, you need to manually install the following dependencies:
 
 > sshpass1.06 or above
 
 ## Deploying openLooKeng on a Single Node
 
-Executing the below command can help you download the necessary packages and deploy openLooKeng server in one-click:
+Execute below command can help you download the necessary packages and deploy openLooKeng server in one-click:
 
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh)
@@ -21,7 +21,7 @@ or:
 wget -O - https://download.openlookeng.io/install.sh|bash
 ```
 
-Normally, you don\'t need to do anything, except waiting for the installation to complete. It will automatically start the service.
+Normally, you don\'t need to do any thing, except for the installation to complete. It will automatically start the service.
 
 Execute below command to stop openLooKeng service.:
 
@@ -51,13 +51,13 @@ or:
 bash <(wget -qO- https://download.openlookeng.io/install.sh) --multi-node
 ```
 
-First, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
+First of all, this command will download scripts and packages required by openLooKeng service. After the download is completed, it will check whether the dependent packages `expect` and `sshpass` are installed. If not, those dependencies will be installed automatically.
 
 Besides, jdk version is required to be greater than 1.8.0\_151. If not, jdk1.8.0\_201 will be installed in the cluster. It is recommended to manually install these dependencies before installing openLooKeng service.
 
 Secondly, the script will download openLooKeng-server tarball and copy that tarball to all the nodes in the cluster. Then install the openLooKeng-server by using this tarball.
 
-Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and for built-in catalogs like `tpch`, `tpcds`, `memory connector`.
+Lastly, the script will setup openLooKeng server with the standard configurations, includes configurations for JVM, Node and also for build-in catalogs like `tpch`, `tpcds`, `memory connector`.
 
 By design, the script will check if there are existing configuration under directory:
 `/home/openlkadmin/.openlkadmin/cluster_node_info`
@@ -79,9 +79,9 @@ The general configurations for openLooKeng\'s coordinator, workers are taken fro
 `/home/openlkadmin/.openlkadmin/cluster_config_info` and configurations for connectors are taken from the directory `/home/openlkadmin/.openlkadmin/catalog` respectively. If these directories or any required configuration files are absent during the deploy script running, default configuration files will be generated
 automatically and deployed to all nodes.
 
-Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customize the deployment.
+Which means, alternatively, you can add those configuration files before running this deploy script, if you want to customized the deployment.
 
-If all the above processes succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
+If above process all succeed, the deploy script will automatically start the openLooKeng Service for you. Execute below command to stop openLooKeng service.:
 
 ```shell
 /opt/openlookeng/bin/stop.sh
@@ -108,7 +108,7 @@ bash <(wget -qO- https://download.openlookeng.io/install.sh) --file <cluster_nod
 ```
 
 
-For more help, execute below command to deploy single node cluster:
+For more help message,execute below command to deploy single node cluster:
 ```shell
 bash <(wget -qO- https://download.openlookeng.io/install.sh) -h
 ```
@@ -150,9 +150,9 @@ execute below command to deploy the configurations to openLooKeng cluster:
 bash /opt/openlookeng/bin/configuration_deploy.sh
 ```
 
-Note, if you want to add more configurations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
+Note, if you want to add more configrations or customize the configurations, you can add properties to the templates into file located at `/home/openlkadmin/.openlkadmin/.etc_template/coordinator` or `/home/openlkadmin/.openlkadmin/.etc_template/worker`.
 
-The property format must be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it is a dynamic value. For example:
+The property format has to be key=\<value\>, where value is wrapped with \'\<\' and \'\>\', which means it it a dynamic value. For example:
 
 ``` properties
 http-server.http.port=<http-server.http.port>
@@ -174,7 +174,7 @@ It is very easy and straight forward to uninstall openLooKeng Service, simply ru
 bash /opt/openlookeng/bin/uninstall.sh
 ```
 
-This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you want to delete user and configuration files, you need to run the below command:
+This will uninstall openLooKeng Service by removing directory `/opt/openlookeng` and all files inside it. However, the `openlkadmin` user and all the configuration files under`/home/openlkadmin/` will not be removed. If you wan to delete user and configuration files, you need to run the below command:
 
 ```shell
 bash /opt/openlookeng/bin/uninstall.sh --all
@@ -190,7 +190,7 @@ If you can't access the download URL from the machine where you want to install
 
 1. Also save third party dependencies under `/opt/openlookeng/resource`. That is, download all files from either `https://download.openlookeng.io/auto-install/third-resource/x86/` or `https://download.openlookeng.io/auto-install/third-resource/aarch64/`, depending on the machine's architecture. This should include 1 `OpenJDK` file and 2 `sshpass` files.
 
-1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
+1. If you plan to perform multi-node installation, and some nodes in the cluster have a different architecture type from the current machine, then also download the `OpenJDK` file for the other architecture, and save it under `/opt/openlookeng/resource/<arch>`, where `<arch>` is either `x86` or `aarch64`, corresponding to the other architecture.
 
 After all resources are available, execute below command to deploy single node cluster:
 

hetu-docs/en/installation/odbc.md

@@ -61,7 +61,7 @@ Before an application uses the openLooKeng ODBC driver, the data source DSN must
 
 ### Opening the ODBC Data Source Administrator (64-bit)
 
-1. Click **Start** and choose **Control Panel**.
+1. Click **Start**, and choose **Control Panel**.
 
 2. In **Control Panel**, click **System and Security**, and then click **Administrative Tools**.
 
@@ -171,6 +171,6 @@ You can obtain the details about data types by calling **SQLGetTypInfo** in **Ca
 
 The openLooKeng ODBC driver supports **both ANSI and Unicode** applications. The default connection character set is the system default character set for ANSI applications and utf8 for Unicode applications. If the character set used by the application is different from the above-mentioned character set,  it may cause garbled characters. For this, the user should specify the connection character set to adapt to the character set required by the application. The corresponding configuration of the connection character set is described as follows.
 
-When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by default, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
+When calling the ODBC API to retrieve data, if bound to the SQL_C_WCHAR C data type buffer, the driver will return the Unicode encoded result for both ANSI and Unicode applications. When bound to the SQL_C_CHAR C data type buffer, by deafult, the driver will return to the ANSI application the result  encoded in system default character set, and for Unicode application the driver will return the result encoded in utf8. If the encoding character set used by the application does not match the default, the result may be garbled. To this end, the user should configure the connection character set to specify the encoding of the result. For example, if the application has garbled Chinese characters, you can try to configure the connection character set to GBK or GB2312.
 
 While configuring data source all connection character sets supported by the openLooKeng ODBC driver can be set in the **Character Set** drop-down box on the page 3 of the User interface. User can select the connection character from the drop-down box after the **Test DSN** is success.

hetu-docs/en/preagg/join-queries.md

@@ -1,65 +0,0 @@
-## Join Query Support
-StarTree Cube can help optimize aggregation over join queries as well. The optimizer looks for aggregation subtree pattern in the logical plan that typically looks like following.
-```
-  AggregationNode
-  |- ProjectNode[Optional]
-  .  |- ProjectNode[Optional]
-  .  .  |- FilterNode[Optional]
-  .  .  .  |- JoinNode
-  .  .  .  .  [More Joins]
-  .  .  .  .  .
-  .  .  .  .  |- ProjectNode[Optional] - Left
-  .  .  .  .  .  |- TableScanNode [Fact Table]
-  .  .  .  .  |- ProjectNode[Optional] - Right
-  .  .  .  .  .  |- TableScanNode [Dim Table]
-```
-
-If the query matches the pattern, the optimizer rewrites the logical plan by replacing the Fact TableScanNode with Cube TableScanNode. This is similar to the single 
-table rewrite.
-
-### Star Schema Support
-Join Query optimizer supports star schema only. A star schema is a data warehousing architecture model where one fact table references multiple dimension tables, which, when viewed as a diagram, 
-looks like a star with the fact table in the center and the dimension tables radiating from it. All kinds of joins are supported.
-
-![star-schema](../images/star-schema.png "star schema")
-
-### Cube Management
-`Create Cube` can be still be used to define Cubes to optimize Join queries as well. The difficult part is identifying GROUP construct while building the Cubes. With single table 
-queries, the GROUP BY clause will contain columns only from same the table. But with join queries, especially star schema queries, the GROUP BY contain columns from Dimension tables and not the Fact table. 
-Let's analyze more with following query
-```sql
-SELECT SUM(lo_revenue) AS lo_revenue, d_year, p_brand
-FROM lineorder
-LEFT JOIN dates ON lo_orderdate = d_datekey
-LEFT JOIN part on lo_partkey = p_partkey
-LEFT JOIN supplier on lo_suppkey = s_suppkey
-WHERE p_category = 'MFGR#12' AND s_region = 'AMERICA'
-GROUP BY d_year, p_brand
-ORDER BY d_year, p_brand;
-```
-
-Here `lineorder` is the Fact table and `dates`, `part`, `supplier` are the Dimension tables. Cubes will be defined on the `lineorder` table. The group by columns `d_year`, `p_brand` are part 
-of the dimension tables `dates` and `part` appropriately. They cannot be used directly in `CREATE CUBE` statement. The proper solution is to use the foreign key columns of `lineorder` table in 
-the GROUP construct while building Cubes.
-```sql
-CREATE CUBE lineorder_cube ON lineorder WITH(
-AGGREGATIONS = (sum(lo_revenue)),
-GROUP = (lo_orderdate, lo_partkey, lo_suppkey));
-```
-The optimizer parses the join conditions and uses those columns to identify the matching Cubes. The performance gain is realized if Cube size is smaller than fact table.
-
-### Limitations
-* Only star schema is supported.
-* Count distinct not supported because Cube does not store actual dimension values.
-* Queries won't be optimized if Cubes are defined on both Fact and Dimension as the optimizer does not have capability to differentiate between two.
-* If Cubes are defined on more than one table of the Join query - then Optimizer does not work. Assumption is that Cubes are defined only the Fact table.
-* Supports only simple aggregation like SUM, COUNT, AVG, MIN, MAX - defined on Single column. Cube does not support SUM(revenue - supplycost) aggregation. The following query cannot be optimized using Cube.
-```
-   SELECT sum(lo_extendedprice * lo_discount) AS revenue
-   FROM lineorder
-   WHERE toYear(lo_orderdate) = 1993 AND lo_discount BETWEEN 1 AND 3 AND lo_quantity < 25;
-```
-
-### Future
-* Support for snowflake schema
-* Building a single cube over multiple tables

hetu-docs/en/preagg/overview.md

@@ -1,261 +0,0 @@
-# StarTree Cube
-## Introduction
-StarTree Cubes are materialized pre-aggregation results stored as tables. This technique is built to optimize low latency iceberg queries. 
-Iceberg queries are a special case of SQL queries involving **GROUP BY** and **HAVING** clauses, wherein the answer set is small relative to the data scanned size. 
-Queries can be characterized by their huge input-small output. 
-
-This technique allows user to build Cubes on an existing table with aggregates and dimensions that are intended to optimize specific queries. 
-Cubes are rollup pre-aggregations that have fewer dimensions and rows compared to the original table. Smaller number of rows means the time spent 
-on table scan is significantly reduced which in turn reduces query latency. If a query is a subset of dimensions and measures of the pre-aggregated table, 
-then Cube can be used to calculate the query without accessing the original table. 
-
-Few of the Cube properties are
- - Cubes are stored in tabular format
- - Generally speaking, Cubes can be created for any table in any connector and stored in another connector
- - Query latency is reduced by rewriting the logical plan to use Cube instead of the original table.
-
-## Cube Optimizer Rule
-As part of logical plan optimization, Cube optimizer rule analyzes and optimizes the aggregation subtree of the logical plan with Cubes. 
-The rule looks for the aggregation subtree that typically looks like the following
-
-```
-AggregationNode
-|- ProjectNode[Optional]
-  |- ProjectNode[Optional]
-    |- FilterNode[Optional]
-      |- ProjectNode[Optional]
-        |- TableScanNode
-```
-
-The rule parses through the subtree and identifies the table name, aggregate functions, where clause, group by clause that is matched with Cube metadata 
-to identify any Cube that can help optimize the query. In case of multiple match, recently created Cube is selected for optimization. If any match found, entire 
-aggregation subtree is rewritten using the Cube. This optimizer uses the TupleDomain construct to match if predicates provided in the Query can be supported by the 
-Cubes.
-
-The following picture depicts the change in the logical plan after the optimization. 
-
-![img](../images/cube-logical-plan-optimizer.png)
-
-## Recommended Usage
-1. Cubes are most useful for iceberg queries that takes huge input and produces small output.
-2. Query performance is best when size of the Cube is less that on the actual table on which Cube was built.
-3. Cubes need to be rebuilt if the source table is updated. 
-
-**Note:** 
-If the source table is updated once the Cubes are built, Cube optimizer ignores the set of Cubes created on the table. Reason being, any 
-operation on the update is considered as a change in the existing data even if only new rows are inserted on the original table. Since inserts and updates
-can't be differentiated, Cubes can't be used as it might result in incorrect result. We are working on a solution to overcome this limitation.
-
-## Supported Connectors
-Star Tree Cube can be stored in following Connectors
-1. Hive
-2. Memory
-   
-Tables from following Connectors can be used as source to build a StarTree Cube.
-1. Hive
-2. Memory
-3. Clickhouse
-
-## Future Work
-1. Support for more JDBC connectors
-2. Simplify Cube management 
-
-   2.1. Overcome the limitation of Creating Cube for larger dataset.
-
-## Enabling and Disabling StarTree Cube
-To enable:
-```sql 
-SET SESSION enable_star_tree_index=true;
-```
-To disable:
-```sql 
-SET SESSION enable_star_tree_index=false;
-```
-
-## Configuration Properties
-| Property Name                                     | Default Value       | Required| Description|
-|---------------------------------------------------|---------------------|---------|--------------|
-| optimizer.enable-star-tree-index                  | false               | No      | Enables StarTree Cube |
-| cube.metadata-cache-size                          | 50                  | No      | The maximum number of metadata for StarTree Cubes that could be loaded into cache before eviction happens |
-| cube.metadata-cache-ttl                           | 1h                  | No      | The maximum time to live of StarTree Cubes that are be loaded into cache before eviction happens |
-
-## Dependencies
-
-StarTree Cube relies on Hetu Metastore to store the Cube related metadata.
-Please check [Hetu Metastore](../admin/meta-store.md) for more information.
-
-## Examples
-
-Creating a StarTree Cube:
-```sql 
-CREATE CUBE nation_cube 
-ON nation 
-WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
-GROUP=(nationkey),
-format='orc', partitioned_by=ARRAY['nationkey']);
-```
-Next, to add data to the Cube:
-```sql 
-INSERT INTO CUBE nation_cube WHERE nationkey >= 5;
-```
-Creating a StarTree Cube with WHERE clause:
-Please note that the following query is only supported via the CLI
-
-```sql 
-CREATE CUBE nation_cube 
-ON nation 
-WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
-GROUP=(nationkey),
-format='orc', partitioned_by=ARRAY['nationkey'])
-WHERE nationkey >= 5;
-```
-
-To use the new Cube, just query the original table using aggregations that were included in the Cube:
-
-```sql 
-SELECT count(*) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
-SELECT nationkey, avg(nationkey), max(regionkey) FROM nation WHERE nationkey >= 5 GROUP BY nationkey;
-```
-
-Since the data inserted into the Cube was for `nationkey >= 5`, only queries matching this condition will utilize the Cube.
-Queries not matching the condition would continue to work but won't use the Cube.
-
-If the source table of a Cube gets updated, the corresponding Cube gets expired automatically. In order to overcome
-this issue, we have added support in openLooKeng CLI by introducing **RELOAD CUBE** command. The user will have the
-ability to manually reload a cube if the status of the Cube becomes INACTIVE or EXPIRED. The syntax to reload the
-Cube nation_cube is as follows,
-
-```sql 
-RELOAD CUBE nation_cube
-```
-Please note that this feature is only supported via the CLI. During this reload process if an unexpected error occurs, the user will get to see the original SQL statement
-to recreate the cube manually.
-
-## Building Cube for Large Dataset
-One of the limitations with the current implementation is that Cube cannot be built for a larger dataset at once. This is due to the cluster memory limitation.
-Processing large number of rows requires more memory than cluster is configured with. This results in query failing with message **Query exceeded per-node user memory
-limit**. To overcome this issue, **INSERT INTO CUBE** SQL support was added. The user has ability to build a Cube for larger data by executing multiple
-insert into Cube statements. The insert statement accepts a where clause, and it can be used to limit the number of processed and inserted into Cube.
-
-This section explains the steps to build a Cube for larger dataset.
-
-Let us take an example of TPCDS dataset and `store_sales` table. The table has 10 years worth of data and user wants to build a Cube for year 2001 
-and due to the cluster memory limit, the entire data set for year 2001 cannot be processed at once.
-
-```sql
-CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk));
-
-SELECT min(d_date_sk) as year_start, max(d_date_sk) as year_end FROM date_dim WHERE d_year = 2001;
- year_start | year_end 
-------------+----------
-    2451911 |  2452275 
-(1 row)
-```
-The following query could result in a failure if the number of rows need to be processed is huge and the query memory exceeds 
-the limit configured for the cluster.
-
-```sql
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275; 
-```
-
-### Solution 1)
-To overcome this issue, multiple insert statements can be used to process rows and insert into Cube and the number of rows can be limited by using where clause;
-
-```sql
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2451911 AND 2452010;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk >= 2452011 AND ss_sold_date_sk <= 2452110;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452111 AND 2452210;
-INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk BETWEEN 2452211 AND 2452275;
-```
-
-### Solution 2)
-CLI has been modified to support creating Cubes for larger dataset and without need for multiple insert statements. CLI internally handles this process.
-Once the user runs create Cube statement with where clause, the CLI takes care of creating the Cube as well as inserting the data into it. This process improves the user experience and
-improves the memory footprint based on the cluster memory limits. CLI internally parses the converts the statement into one create Cube statement followed by 
-one or more insert statements. This change is only works if user executes the command from CLI and not via any other means i.e. JDBC, etc...
-
-```sql
-CREATE CUBE store_sales_cube ON store_sales WITH (AGGREGATIONS = (sum(ss_net_paid), sum(ss_sales_price), sum(ss_quantity)), GROUP = (ss_sold_date_sk, ss_store_sk)) WHERE ss_sold_date_sk BETWEEN 2451911 AND 242275;
-```
-
-Internally the system will rewrite and merge all continuous range predicates into a single predicate;
-
-```sql
-SHOW CUBES;
-
-           Cube Name             |         Table Name         | Status |         Dimensions          |                     Aggregations                      |                                     Where Clause                                     
----------------------------------+----------------------------+--------+-----------------------------+-------------------------------------------------------+-------------------------------------------------------+------------------------------
- hive.tpcds_sf1.store_sales_cube | hive.tpcds_sf1.store_sales | Active | ss_sold_date_sk,ss_store_sk | sum(ss_sales_price),sum(ss_net_paid),sum(ss_quantity) | (("ss_sold_date_sk" >= BIGINT '2451911') AND ("ss_sold_date_sk" < BIGINT '2452276')) 
-```
-
-**Note:**
-1. The system will try to rewrite all type of Predicates into a Range to see if they can be merged together.
-   All continuous predicates will be merged into a single range predicate and remaining predicates are untouched.
-
-   Only the following types are supported and can be merged together.
-   `Integer, TinyInt, SmallInt, BigInt, Date, String`
-
-   For String data type, predicate merge logic functionally works only if the Strings are ending with a digit and all are of same length. 
-   For example,
-
-```sql
-   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A01' AND 'A10';
-   INSERT INTO CUBE store_sales_cube WHERE store_id BETWEEN 'A11' AND 'A20';
-```
-After the insertion, the two predicates will be merged into `'A01' AND 'A20'`
-
-```sql
-   SELECT ss_store_id, sum(ss_sales_price) WHERE ss_store_id BETWEEN 'A05' AND 'A15'; - Cube would be used for this query.
-```
-
-   Consider the following example where `store_id` values are not of same length.
-
-```sql
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A1';
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A2' 
-```   
-   store_id predicate will be rewritten as `store_id >= 'A1' and store < 'A3'` as per the varchar predicate merge logic;
-
-```sql   
-   INSERT INTO CUBE store_sales_cube WHERE store_id = 'A10' 
-```
-   The above query would fail because `A10` is subset of the range `store_id >= 'A1' and store < 'A3'`. So Users should be wary of this issue. 
-
-For other data types, it is difficult to identify if two predicates are continuous therefore they cannot be merged together. And because of this issue, there is
-possibility that particular Cube may not be used during query optimization even if the Cube has all the required data.
-
-2. Predicate rewrite has some limitations as well. Consider the following query
-
-```sql   
-   INSERT INTO CUBE store_sales_cube WHERE ss_sold_date_sk > 2451911; 
-```
-The predicate is rewritten as ss_sold_date_sk >= 2451912 to support merging continuous predicates.
-Since the predicate is rewritten, they query using ss_sold_date_sk > 2451911 predicate will not match with Cube predicate so Cube won't be used to
-optimize the query. The same is applicable for predicates with <= operator. ie. ss_sold_date_sk <= 2451911 is rewritten as ss_sold_date_sk < 2451912
-
-```sql   
-   SELECT ss_sold_date_sk, .... FROM hive.tpcds_sf1.store_sales WHERE ss_sold_date_sk > 2451911
-```   
-3. Only single column predicates can be merged.
-
-
-## Open issues and Limitations
-1. StarTree Cube is only effective when the group by cardinality is considerably fewer than the number of rows in source table.
-2. A significant amount of user effort required in maintaining Cubes for large datasets.
-3. Only incremental insert into Cube is supported. Cannot delete specific rows from Cube.
-4. Cubes created on a transaction table may expire automatically even if the source table has not been updated. This is due to the compaction policy which
-   merges delta files into single large ORC file which in turn changes the last modified of time of the table. Cube status is determined by comparing last modified
-   timestamp of table when Cube was created with the last modified time of the table when queries are executed.
-5. OpenLooKeng CLI has been modified to ease the process of creating Cubes for larger datasets. But still there are limitations with this implementation
-   as the process involves merging multiple Cube predicates into one. Only Cube predicates defined on Integer, Long and Date types can be merged properly. Support for Char, 
-   String types still need to be implemented.
-6. Varchar predicates can be merged only if the values are of same length.  
-
-## Performance Optimizations on Star Tree
-1. Star Tree Query re-write optimization for same group by columns: If the group by columns of the cube and query matches, the query is 
-re-written internally to select the pre-aggregated data. If the group by columns does not matches, the additional aggregations are 
-internally applied on the re-written query.
-2. Star Tree table scan optimization for Average aggregation function: If the group by columns of the cube and query matches, the select query 
-is re-written internally to select the startree cube's pre-aggregated Average column data. If the group by columns does not match, 
-the select query is re-written internally to select the startree cube's pre-aggregated Sum and Count column data, from which the 
-average is later calculated.

hetu-docs/en/preagg/statements.md

@@ -1,195 +0,0 @@
-# Usage
-Cubes can be managed using any of the supported clients, such as hetu-cli located under the `bin` directory in the installation.
-
-## CREATE CUBE
-### Synopsis
-
-``` sql
-CREATE CUBE [ IF NOT EXISTS ]
-cube_name ON table_name WITH (
-   AGGREGATIONS = ( expression [, ...] ),
-   GROUP = ( column_name [, ...])
-   [, FILTER = (expression)]
-   [, ( property_name = expression [, ...] ) ] 
-)
-[WHERE predicate]
-```
-### Description
-Create a new, empty Cube with the specified group and aggregations. Use `INSERT INTO CUBE (see below)` to insert into data.
-
-The optional `IF NOT EXISTS` clause causes the error to be suppressed if the Cube already exists.
-The optional `property_name` section can be used to set properties on the newly created Cube. 
-
-To list all available table properties, run the following query:
-
-    SELECT * FROM system.metadata.table_properties
-
-**Note:** These properties are limited to the Connector which the Cube is being created for.
-
-### Examples
-Create a new Cube `orders_cube` on `orders`:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
-      GROUP = ( orderstatus, orderdate ),
-      format = 'ORC'
-    )
-
-Create a new partitioned Cube `orders_cube`:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
-      GROUP = ( orderstatus, orderdate ),
-      format = 'ORC',
-      partitioned_by = ARRAY['orderdate']
-    )
-
-Create a new Cube `orders_cube` with some source data filter:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    )
-
-Create a new Cube `orders_cube` with some additional predicate on Cube columns:
-
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    ) WHERE orderstatus = 'PENDING';
-
-This is same as following:
-    
-    CREATE CUBE orders_cube ON orders WITH (
-      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
-      GROUP = ( orderstatus ),
-      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
-    );
-    INSERT INTO CUBE orders_cube WHERE orderstatus = 'PENDING';
-
-The `FILTER` property can be used to filter out data from the source table while building the Cube. Cube is built on the data after 
-applying the `orderdate BETWEEN 2512450 AND 2512460` predicate on the source table. The columns used in the filter predicate must not be part the Cube.
-
-### Limitations
-- Cubes can be created with only following aggregation functions.  
-  In other words, Queries using the following functions can only be optimized using Cubes.
-  **COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG**
-- Different connector might support different data type, and different table/column properties.
-
-## INSERT INTO CUBE
-
-### Synopsis
-``` sql
-INSERT INTO CUBE cube_name [WHERE condition]
-```
-
-### Description
-`CREATE CUBE` statement creates Cube without any data. To insert data into Cube, use `INSERT INTO CUBE` SQL.
-The `WHERE` clause is optional. If predicate is provided, only data matching the given predicate are processed from the source table and inserted into the Cube. 
-Otherwise, entire data from the source table is processed and inserted into Cube.
-
-### Examples
-Insert data into the `orders_cube` Cube:
-
-    INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
-    INSERT INTO CUBE order_all_cube;
-
-### Limitations
-1. Subsequent inserts to the same Cube need to use same set of columns
-
-```sql
-   CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (count(*)), GROUP = (orderdate));
-   
-   INSERT INTO CUBE orders_cube WHERE orderdate BETWEEN date '1999-01-01' AND date '1999-01-05';
-   
-   -- This statement would fail because its possible the Cube already contain rows matching the given predicate.
-   INSERT INTO CUBE orders_cube WHERE location = 'Canada';
-```
-**Note:** This means that columns used in the first insert must be used in every insert predicate following the first to avoid inserting duplicate data.
-
-## INSERT OVERWRITE CUBE
-
-### Synopsis
-``` sql
-INSERT OVERWRITE CUBE cube_name [WHERE condition]
-```
-
-### Description
-Similar to `INSERT INTO CUBE` statement but with this statement the existing data is overwritten. Predicates
-are optional.`INSERT OVERWRITE CUBE` is not supported on partitioned cubes. Cubes are essentially stored as tables and so `INSERT OVERWRITE` only 
-replaces the matching partitions and does not overwrite the entire table. So this operation is blocked on partitioned cube. 
-Drop and recreate cube if needed.
-
-### Examples
-Insert data based on condition into the `orders_cube` Cube:
-
-    INSERT OVERWRITE CUBE orders_cube WHERE orderdate > date '1999-01-01';
-    INSERT OVERWRITE CUBE orders_cube;
-
-## SHOW CUBES
-
-### Synopsis
-```sql
-SHOW CUBES [ FOR table_name ];
-```
-
-### Description
-`SHOW CUBES` lists all Cubes. Adding the optional `table_name` lists only the Cubes for that table.
-
-### Examples
-
-Show all Cubes:
-```sql
-    SHOW CUBES;
-```
-
-Show Cubes for `orders` table:
-
-```sql
-    SHOW CUBES FOR orders;
-```
-## RELOAD CUBE
-
-### Synopsis
-
-``` sql
-RELOAD CUBE cube_name
-```
-
-### Description
-Reloads the Cube if the source table has been updated.
-
-### Examples
-
-If the source table `orders` of the cube `orders_cube` gets updated then the status of the cube `orders_cube` 
-gets EXPIRED. Use the command `RELOAD CUBE cube_name` to overcome this issue as follows:
-
-```sql
-    RELOAD CUBE orders_cube
-```
-## DROP CUBE
-
-### Synopsis
-
-``` sql
-DROP CUBE  [ IF EXISTS ] cube_name
-```
-
-### Description
-Drop an existing Cube.
-
-The optional `IF EXISTS` clause causes the error to be suppressed if the Cube does not exist.
-
-### Examples
-
-Drop the Cube `orders_cube`:
-
-    DROP CUBE orders_cube
-
-Drop the Cube `orders_cube` if it exists:
-
-    DROP CUBE IF EXISTS orders_cube
-
-

hetu-docs/en/releasenotes/releasenotes-1.4.1.md

@@ -1,14 +0,0 @@
-# Release 1.4.1 (12 Nov 2021)
-
-## Key Features
-
-This release mainly adds the introduction of OmniData Connector and jdk8 support under arm architecture.
-
-| Area                    | Feature                                                      | PR #s                                                        |
-| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Data Source Connector                | openLooKeng supports offload operators to the near data side for process through omnidata connector, reducing the transmission of invalid data on the network and effectively improving the performance of big data computing.                                               | 1219                                                          |
-| Arm architecture | Eliminate the mandatory requirements for Java version under arm architecture caused by JDK paused problem, and support jdk1.8.262 and above under arm architecture. | 1214                                                           |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.4.1/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.5.0.md

@@ -1,23 +0,0 @@
-# Release 1.5.0
-
-## Key Features
-
-| Area             | Feature                                                      |
-| ---------------- | ------------------------------------------------------------ |
-| Star Tree        | 1. Support for optimizing join queries such as star schema queries.<br/>2. Optimized the query plan by eliminating unnecessary aggregations on top of the cube since the cube already contains the rolled up results. The performance optimizations benefits queries whose group by clause exactly matches the cube’s group.<br/>3. Bug fixes to further enhance the usability, and robustness of cubes. |
-| Memory Connector | 1. Improved performance of memory connector by adding support for memory table partitioning to allow data skipping of entire partitions<br/>2. Collect statistics on memory table to support openLooKeng cost based optimizers. |
-| Task Recovery    | Fixed several important bugs to address data inconsistency issues, and query hanging issues that occasionally occur during high concurrency, and during worker failures. |
-| OLK-on-Yarn      | Support deploying an HA-enabled openLooKeng cluster instance on-yarn, that contains a reverse proxy (ngnix by default), and 2 or more coordinator nodes. The cluster can be horizontally scaled manually by adding and removing yarn containers to the coordinator and worker components.|
-| Spill to Disk    | Optimized the spill to disk mechansim to directly write Pages to disk instead of buffering it. Changed the strategy to spill those operators which can free up maximum memory. This resulted in improvement of spill to disk performance by 30% |
-
-## Known Issues
-
-| Category              | Description                                                  | Gitee issue                                                       |
-| --------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Task Recovery         |When the query process reaches stage1,a worker is killed result some values become smaller(occasionally).   |[I4M2LW](https://e.gitee.com/open_lookeng/issues/list?issue=I4M2LW) |
-| Memory Connector      |When a table with the index_columns parameter is queried, an error message is displayed：java.lang.NullPointerException. | [I4NVW3](https://e.gitee.com/open_lookeng/issues/list?issue=I4NVW3)|
-|                       |When drop and then create a same partitioned table with data type double, query result rows is greater than expected. | [I4LUDF](https://e.gitee.com/open_lookeng/issues/list?issue=I4LUDF) |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.5.0/hetu-docs/en)

hetu-docs/en/releasenotes/releasenotes-1.6.0.md

@@ -1,25 +0,0 @@
-# Release 1.6.0
-
-## Key Features
-
-| Area                  | Feature                                                      |
-| --------------------- | ------------------------------------------------------------ |
-| Star Tree             | Support update cube command to allow admin to easily update an existing cube when the underlying data changes |
-| Bloom Index           | Hindex-Optimize Bloom Index Size-Reduce bloom index size by 10X+ times |
-| Task Recovery         | 1. Improve failure detection time: It need take 300s to determine a task is failed and resume after that. Improving this would improve the resume & also the overall query time<br/>2. snapshotting speed & size: When sql execute takes a snapshot, now use direct Java serialization which is slow and also takes more size. Using kryo serialization would reduce size and also increase speed there by increasing the overall throughput |
-| Spill to Disk         | 1. Spill to disk speed & size improvement: When spill happens during HashAggregation & GroupBy, the data serialized to disk is slow and also size is more. It can improve the overall performance by reducing size and also improving  the writing speed. Using kryo serialization improves both speed and reduces size<br/>2. Support spilling to hdfs: Currently data can spill to multiple disks, now support spill to hdfs to improve throughput<br/>3. Async spill/unspill: When revocable memory crosses threshold and spill is triggered, it blocks accepting the data from the downstream operators. Accepting this and adding to the existing spill would help to complete the pipeline faster<br/>4. Enable spill for right outer & full join for spilling: It don’t spill the build side data when the join type is right outer or full join as it needs the entire data in memory for lookup. This leads to out of  memory when the data size is more. Instead by enable spill and create a Bloom Filter to identify the data spilled and use it during join with probe side |
-| Connector Enhancement | Support data update and delete operator for PostgreSQL and openGauss |
-
-## Known Issues
-
-| Category      | Description                                                  | Gitee issue                                               |
-| ------------- | ------------------------------------------------------------ | --------------------------------------------------------- |
-| Task Recovery | When a snapshot is enabled and a CTAS with transaction is executed, an error is reported in the SQL statement. | [I502KF](https://e.gitee.com/open_lookeng/issues/list?issue=I502KF) |
-|               | An error occurs occasionally when snapshot is enabled and exchange.is-timeout-failure-detection-enabled is disabled. | [I4Y3TQ](https://e.gitee.com/open_lookeng/issues/list?issue=I4Y3TQ) |
-| Star Tree     | In the memory connector, after the star tree is enabled, data inconsistency occurs during query. | [I4QQUB](https://e.gitee.com/open_lookeng/issues/list?issue=I4QQUB) |
-|               | When the reload cube command is executed for 10 different cubes at the same time, some cubes fail to be reloaded. | [I4VSVJ](https://e.gitee.com/open_lookeng/issues/list?issue=I4VSVJ) |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.0/hetu-docs/en)
-

hetu-docs/en/releasenotes/releasenotes-1.6.1.md

@@ -1,15 +0,0 @@
-# Release 1.6.1 (27 Apr 2022)
-
-## Key Features
-
-This release is mainly about modification and enhancement of some SPIs, which are used in more scenarios.
-
-| Area                    | Feature                                                      | PR #s                                                        |
-| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
-| Data Source statistics                | The method of obtaining statistics is added so that statistics can be directly obtained from the Connector. Some operators can be pushed down to the Connector for calculation. You may need to obtain statistics from the Connector to display the amount of processed data.                                               | 1450                                                          |
-| Operator processing extension | Users can customize the physical execution plan of worker nodes. Users can use their own operator pipelines to replace the native implementation to accelerate operator processing. | 1436                                                           |
-| HIVE UDF extension | Adds the adaptation of HIVE UDF function namespace to support the execution of UDFs (including GenericUDF) written based on the HIVE UDF framework. | 1456                                                           |
-
-## Obtaining the Document 
-
-For details, see [https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en](https://gitee.com/openlookeng/hetu-core/tree/1.6.1/hetu-docs/en)

hetu-docs/en/security/built-in-system-access-control.md

@@ -1,3 +1,4 @@
+
 Built-in System Access Control
 ==============================
 
@@ -56,10 +57,7 @@ composed of the following fields:
 
 -   `user` (optional): regex to match against user name. Defaults to `.*`.
 -   `catalog` (optional): regex to match against catalog name. Defaults to `.*`.
--   ``allow`` (required): string indicating whether a user has access to the catalog.
+-   `allow` (required): boolean indicating whether a user has access to the catalog
-    This value can be ``all``, ``read-only`` or ``none``, and defaults to ``none``.
-    Setting this value to ``read-only`` has the same behavior as the ``read-only``
-    system access control plugin.
 
 
 **Note**
@@ -67,9 +65,7 @@ composed of the following fields:
 *By default, all users have access to the `system` catalog. You can override this behavior by adding a rule.*
 
 
-For example, if you want to allow only the user ``admin`` to access the``mysql`` and the ``system`` catalog, 
+For example, if you want to allow only the user `admin` to access the `mysql` and the `system` catalog, allow all users to access the `hive` catalog, and deny all other access, you can use the following rules:
-allow all users to access the ``hive`` catalog, allow the user ``alice`` read-only access to the ``postgresql`` 
-catalog, and deny all other access, you can use the following rules:
 
 ``` json
 {
@@ -77,20 +73,15 @@ catalog, and deny all other access, you can use the following rules:
     {
       "user": "admin",
       "catalog": "(mysql|system)",
-      "allow": all
+      "allow": true
     },
     {
       "catalog": "hive",
-      "allow": all
+      "allow": true
-    },
-    {
-      "user": "alice",
-      "catalog": "postgresql",
-      "allow": "read-only"
     },
     {
       "catalog": "system",
-      "allow": none
+      "allow": false
     }
   ]
 }
@@ -204,7 +195,6 @@ If you want to allow users to use the  extactly same name as their Kerberos prin
 ```
 
 ### Node State Rules
-
 These rules govern the node state info particular users can access. The user is granted access to update a node state based on the first matching rule read from top to bottom. If no rule matches, access is denied. Each rule is
 composed of the following fields:
 

hetu-docs/en/security/ranger-access-control.md

@@ -7,7 +7,7 @@ Overview
 
 Apache Ranger delivers a comprehensive approach to security for a Hadoop cluster. It provides a centralized platform to define, administer and manage security policies consistently across Hadoop components. Check [Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index) for detail introduction and user guide.
 
-[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is developed based on Ranger 2.1.0, which is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
+[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
 
 Build Process
 -------------------------

hetu-docs/en/sql/create-cube.md

@@ -0,0 +1,69 @@
+CREATE CUBE
+============
+
+Synopsis
+--------
+
+``` sql
+CREATE CUBE [ IF NOT EXISTS ]
+cube_name ON table_name WITH (
+   AGGREGATIONS = ( expression [, ...] ),
+   GROUP = ( column_name [, ...])
+   [, FILTER = (expression)]
+   [, ( property_name = expression [, ...] ) ] 
+)
+```
+
+Description
+-----------
+
+Create a new, empty star-tree cube with the specified group and aggregations. Use `insert-into-cube` to insert data.
+
+The optional `IF NOT EXISTS` clause causes the error to be suppressed if the table already exists.
+
+The optional `property_name` section can be used to set properties on the newly created cube. To list all available table properties, run the following query:
+
+    SELECT * FROM system.metadata.table_properties
+
+Examples
+--------
+
+Create a new cube `orders_cube` on `orders`:
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
+      GROUP = ( orderstatus, orderdate ),
+      format = 'ORC'
+    )
+
+Create a new partitioned cube `orders_cube`:
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
+      GROUP = ( orderstatus, orderdate ),
+      format = 'ORC',
+      partitioned_by = ARRAY['orderdate']
+    )
+
+Create a new cube 'orders_cube' with some source filter
+
+    CREATE CUBE orders_cube ON orders WITH (
+      AGGREGATIONS = ( SUM(totalprice), COUNT DISTINCT(orderid) ),
+      GROUP = ( orderstatus ),
+      FILTER = (orderdate BETWEEN 2512450 AND 2512460)
+    )
+
+Filter is additional predicate that applied on the source table when building a cube. The columns used in the filter predicate must not be part the Cube.
+
+Limitations
+-----------
+
+- Supported aggregate functions:
+      COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
+- Only one group supported per Cube.  
+- Different connector might support different data type, and different table/column properties.
+- Can currently only create cubes in Hive connector, but the cubes can be created on a table from another connector. 
+
+See Also
+--------
+[INSERT INTO CUBE](./insert-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)

hetu-docs/en/sql/drop-cube.md

@@ -0,0 +1,33 @@
+
+DROP CUBE
+==========
+
+Synopsis
+--------
+
+``` sql
+DROP CUBE  [ IF EXISTS ] cube_name
+```
+
+Description
+-----------
+
+Drop an existing cube.
+
+The optional `IF EXISTS` clause causes the error to be suppressed if the cube does not exist.
+
+Examples
+--------
+
+Drop the cube `orders_cube`:
+
+    DROP CUBE orders_cube
+
+Drop the cube `orders_cube` if it exists:
+
+    DROP CUBE IF EXISTS orders_cube
+
+See Also
+--------
+
+[CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [INSERT INTO CUBE](./insert-cube.md)

hetu-docs/en/sql/insert-cube.md

@@ -0,0 +1,44 @@
+INSERT INTO CUBE
+======
+
+Synopsis
+--------
+
+``` sql
+INSERT INTO CUBE cube_name [WHERE condition]
+```
+
+Description
+-----------
+
+Insert data into a star-tree cube. Predicate information is optional. If predicate provided, only data matching 
+the given predicate are processed from the source table and inserted into the cube. Otherwise, entire 
+data from the source table is processed and inserted into Cube.
+
+Examples
+--------
+
+Insert data based on condition into the `orders_cube` cube:
+
+    INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
+    INSERT INTO CUBE order_all_cube;
+
+See Also
+--------
+
+[INSERT OVERWRITE CUBE](./insert-overwrite-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)
+
+
+Limitations
+----------
+1. Insert statement does not allow different columns to be used in the where clause for successive inserts.
+
+```sql
+   CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (count(*)), GROUP = (orderdate));
+   
+   INSERT INTO CUBE orders_cube WHERE orderdate BETWEEN date '1999-01-01' AND date '1999-01-05';
+   
+   -- This statement would fail because its possible the Cube already contain rows matching the given predicate.
+   INSERT INTO CUBE orders_cube WHERE location = 'Canada';
+```
+Note: this means that columns used in the first insert must be used in every insert predicate following the first to avoid inserting duplicate data.

hetu-docs/en/sql/insert-overwrite-cube.md

@@ -0,0 +1,28 @@
+INSERT INTO CUBE
+======
+
+Synopsis
+--------
+
+``` sql
+INSERT OVERWRITE CUBE cube_name [WHERE condition]
+```
+
+Description
+-----------
+
+Similar to INSERT INTO CUBE statement but with this statement the existing data is overwritten. Predicates 
+are optional.
+
+Examples
+--------
+
+Insert data based on condition into the `orders_cube` cube:
+
+    INSERT OVERWRITE CUBE orders_cube WHERE orderdate > date '1999-01-01';
+    INSERT OVERWRITE CUBE orders_cube;
+
+See Also
+--------
+
+[INSERT INTO CUBE](./insert-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)

hetu-docs/en/sql/insert-overwrite.md

@@ -28,7 +28,7 @@ Assume `orders` is not a partitioned table, and have 100 rows, then execute belo
 
     INSERT OVERWRITE orders VALUES (1, 'SUCCESS', '10.25', DATA '2020-01-01');
 
-Then the `orders` table will only have 1 row that is the data specified in the `VALUE` clause.
+Then the `orders` table will only have 1 rows, that is the data specified in the `VALUE` clause.
 
 Assume `users` has 3 columns: (`id`, `name`, `state`) and partitioned by `state`, and the existing data has follow rows:
 

hetu-docs/en/sql/select.md

@@ -508,7 +508,7 @@ _col0
 **INTERSECT**
 
 `INTERSECT` returns only the rows that are in the result sets of both the first and the second queries. The following is an example of one of the simplest possible `INTERSECT` clauses. It selects the values `13`
-and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results:
+and `42` and combines this result set with a second query that selects the value `13`. Since `42` is only in the result set of the first query, it is not included in the final results.:
 
     SELECT * FROM (VALUES 13, 42)
     INTERSECT
@@ -524,7 +524,7 @@ _col0
 **EXCEPT**
 
 `EXCEPT` returns the rows that are in the result set of the first query, but not the second. The following is an example of one of the simplest possible `EXCEPT` clauses. It selects the values `13` and `42` and
-combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result:
+combines this result set with a second query that selects the value `13`. Since `13` is also in the result set of the second query, it is not included in the final result.:
 
     SELECT * FROM (VALUES 13, 42)
     EXCEPT
@@ -746,7 +746,7 @@ Joins allow you to combine data from multiple relations.
 
 ### CROSS JOIN
 
-A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explicit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
+A cross join returns the Cartesian product (all combinations) of two relations. Cross joins can either be specified using the explit `CROSS JOIN` syntax or by specifying multiple relations in the `FROM` clause.
 
 Both of the following queries are equivalent:
 

hetu-docs/en/sql/show-create-cube.md

@@ -1,32 +0,0 @@
-
-SHOW CREATE CUBE
-=================
-
-Synopsis
---------
-
-``` sql
-SHOW CREATE CUBE cube_name
-```
-
-Description
------------
-
-Show the SQL statement that creates the specified cube.
-
-Examples
---------
-
-Create a cube `orders_cube` on `orders` table as follows
-
-    CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
-    GROUP = (custKEY, ORDERkey), format= 'orc')
-
-Use `SHOW CREATE CUBE` command to show the SQL statement that was used to create the cube `orders_cube`:
-
-    SHOW CREATE CUBE orders_cube;
-
-``` sql
-CREATE CUBE orders_cube ON orders WITH (AGGREGATIONS = (avg(totalprice), sum(totalprice), count(*)), 
-GROUP = (custKEY, ORDERkey), format= 'orc')
-```

hetu-docs/en/sql/show-cubes.md

@@ -0,0 +1,35 @@
+
+SHOW CUBES
+==========
+
+Synopsis
+--------
+
+``` sql
+SHOW CUBES [ FOR table_name ];
+```
+
+Description
+-----------
+
+`SHOW CUBES` lists all cubes. Adding the optional `table_name` lists only the cubes for that table.
+
+Examples
+--------
+
+Show all cubes:
+
+```sql
+    SHOW CUBES;
+```
+
+Show cubes for `orders` table:
+
+```sql
+    SHOW CUBES FOR orders;
+```
+
+See Also
+--------
+
+[CREATE CUBE](./create-cube.md), [DROP CUBE](./drop-cube.md), [INSERT INTO CUBE](./insert-cube.md)

hetu-docs/zh/admin/audit-log.md

@@ -1,6 +1,6 @@
 # 审计日志
 
-openLooKeng审计日志记录功能是一个自定义事件监听器，监听openLooKeng集群启停与集群中节点的动态添加与删除事件；监听WebUi用户登录与退出事件；监听查询事件,在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
+openLooKeng审计日志记录功能是一个自定义事件监听器，在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
 
 1. 事件发生时间
 2. 用户ID
@@ -23,17 +23,15 @@ openLooKeng审计日志记录功能是一个自定义事件监听器，监听ope
 hetu.event.listener.type=AUDIT
 hetu.event.listener.listen.query.creation=true
 hetu.event.listener.listen.query.completion=true
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```
 
 其他审计日志记录属性包括：
 
-`hetu.event.listener.type`：用于定义审计日志的记录类型，允许的值为AUDIT和LOGGER。
+`hetu.event.listener.audit.file`：可选属性，用于定义审计文件的绝对文件路径。确保运行openLooKeng服务器的进程对该目录有写权限。
 
-`hetu.auditlog.logoutput`：用于定义审计文件的绝对目录路径。确保运行openLooKeng服务器的进程对该目录有写权限。
+`hetu.event.listener.audit.filecount`：可选属性，用于定义要使用的文件数。
 
-`hetu.auditlog.logconversionpattern`：用于定义审计日志的轮转模式。允许的值为yyyy-MM-dd.HH和yyyy-MM-dd。
+`hetu.event.listener.audit.limit`：可选属性，用于定义写入任一文件的最大字节数。
 
 配置文件示例：
 
@@ -45,6 +43,4 @@ hetu.event.listener.listen.query.completion=true
 hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
 hetu.event.listener.audit.filecount=1
 hetu.event.listener.audit.limit=100000
-hetu.auditlog.logoutput=/var/log/
-hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
 ```

hetu-docs/zh/admin/extension-execution-planner.md

@@ -1,24 +0,0 @@
-#扩展物理执行计划
-本节介绍openLooKeng如何添加扩展物理执行计划。通过物理执行计划的扩展，openLooKeng可以使用其他算子加速库来加速SQL语句的执行。
-
-##配置
-在配置文件`config.properties`增加如下配置：
-
-``` properties
-extension_execution_planner_enabled=true
-extension_execution_planner_jar_path=file:///xxPath/omni-openLooKeng-adapter-1.6.1-SNAPSHOT.jar
-extension_execution_planner_class_path=nova.hetu.olk.OmniLocalExecutionPlanner
-```
-
-上述属性说明如下：
-
-- `extension_execution_planner_enabled`：是否开启扩展物理执行计划特性。
-- `extension_execution_planner_jar_path`：指定扩展jar包的文件路径。
-- `extension_execution_planner_class_path`：指定扩展jar包中执行计划生成类的包路径。
-
-
-##使用
-当运行openLooKeng时，可在WebUI或Cli中通过如下命令控制扩展物理执行计划的开启:
-```
-set session extension_execution_planner_enabled=true/false
-```

hetu-docs/zh/admin/properties.md

@@ -115,20 +115,6 @@
 > 
 > 此属性是在JVM堆中为openLooKeng不跟踪的分配留作裕量/缓冲区的内存量。
 
-### `query.suspend-query-enabled`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 系统资源不足时，临时挂起运行中的查询。
-
-### `query.max-suspended-queries`
-
-> -   **类型：** `integer`
-> -   **默认值：** `10`
->
-> 终止查询之前，查询挂起尝试的最大次数。仅当`query.suspend-query-enabled`设置为`true`时，此属性才生效。
-
 ## 溢出属性
 
 ### `experimental.spill-enabled`
@@ -162,24 +148,6 @@
 > 
 > 此配置属性可由`spill_window_operator`会话属性重写。
 
-### `experimental.spill-build-for-outer-join-enabled`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 为右外连接和全外连接操作启用溢出功能。
->
-> 此config属性可被`spill_build_for_outer_join_enabled`会话属性覆盖。
-
-### `experimental.inner-join-spill-filter-enabled`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 启用基于布隆过滤器的构建侧溢出匹配，以进行探查侧溢出决策。
->
-> 此config属性可被`inner_join_spill_filter_enabled`会话属性覆盖。
-
 ### `experimental.spill-reuse-tablescan`
 
 > - **类型**：`boolean`
@@ -189,13 +157,13 @@
 >
 > 此配置属性可由`spill_reuse_tablescan`会话属性重写。
 
-### `experimental.spiller-spill-path`
+### experimental.spiller-spill-path`
 
 > - **类型：** `string`
 > - **无默认值。** 启用溢出时必须设置。
 > 
 > 溢出内容写入的目录。该属性可以是一个逗号分隔的列表，以同时溢出到多个目录，这有助于利用系统中安装的多个驱动器。
-> 当`experimental.spiller-spill-to-hdfs`为`true`时，`experimental.spiller-spill-path`必须只包含一个目录。
+> 
 > 不建议溢出到系统驱动器上。最重要的是，不要溢出到写入JVM日志的驱动器，因为磁盘过度使用可能导致JVM长时间暂停，从而导致查询失败。
 
 ### `experimental.spiller-max-used-space-threshold`
@@ -240,7 +208,7 @@
 >
 > 用于在Reuse Exchange中缓存页面的内存限制。
 
-### `experimental.spill-compression-enabled`
+### experimental.spill-compression-enabled`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -254,69 +222,6 @@
 > 
 > 允许使用随机生成的密钥（每个溢出文件）来加密和解密溢出到磁盘的数据。
 
-### `experimental.spill-direct-serde-enabled`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 允许将页面直接序列化/读取到流中或从流中序列化/读取页面。
-
-### `experimental.spill-prefetch-read-pages`
-
-> -   **类型：** `integer`
-> -   **默认值：** `1`
->
-> 设置从溢出文件读取时预取的页数。
-
-
-### `experimental.spill-use-kryo-serialization`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 启用基于Kryo的序列化以溢出到磁盘，而不使用默认的Java序列化器。
-
-
-### `experimental.revocable-memory-selection-threshold`
-
-> -   **类型：** `data size`
-> -   **默认值：** `512 MB`
->
-> 设置运算符可撤销内存的内存选择阈值，直接为准备撤销的剩余字节分配可撤销内存。
-
-### `experimental.prioritize-larger-spilts-memory-revoke`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `true`
->
-> 启用对具有较大可撤销内存的Split进行优先级排序。
-
-### `experimental.spill-non-blocking-orderby`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 开启按照运算符排序，使用异步机制溢出。即使在溢出正在进行时，也可以累积输入，并在次要数据累积超过阈值或主溢出完成时启动次溢出。阈值的默认值是20MB到可用内存的5%之间的最小值。此属性必须与`experimental.spill-enabled`属性结合使用。
->
-> 此config属性可被`spill_non_blocking_orderby`会话属性覆盖。
-
-### `experimental.spiller-spill-to-hdfs`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 启用溢出到HDFS。当此属性设置为`true`时，必须设置`experimental.spiller-spill-profile`属性，并且`experimental.spiller-spill-path`必须仅包含单个路径。
-
-### `experimental.spiller-spill-profile`
-
-> -   **类型：** `string`
-> -   **无默认值。** 启用溢出到HDFS时必须设置此属性。
->
->
-> 此属性定义用于溢出的[filesystem](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果此属性设置为`experimental.spiller-spill-profile=spill-hdfs`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`spill-hdfs.properties`，其中包含必要的信息，包括身份验证类型、config和keytab（如果适用，详情请参见[filesystem](../develop/filesystem.md）)。
->
-> 当`experimental.spiller-spill-to-hdfs`设置为`true`时，必须配置此属性。所有Coordinator和Worker的配置文件中必须包含此属性。指定的文件系统必须可由所有Worker访问，并且Worker必须能够读取和写入指定文件系统中`experimental.spiller-spill-path`文件夹中指明的路径。
-
 ## 交换属性
 
 在openLooKeng节点之间为查询的不同阶段交换数据。调整这些属性可有助于解决节点间通信问题或提高网络利用率。
@@ -354,124 +259,21 @@
 > 
 > 如果网络延迟较高，增大该值可以提高网络吞吐量。减小该值可以提高大型集群的查询性能，因为它减少了由于交换客户端缓冲区保存了较多任务（而不是保存较少任务中的较多数据）的响应而导致的倾斜。
 
+### `exchange.max-error-duration`
+
+> - **类型：** `duration`
+> - **最小值：** `1m`
+> - **默认值：** `7m`
+> 
+> 交换错误最大缓冲时间，超过该时限则查询失败。
+
 ### `sink.max-buffer-size`
 
 > - **类型：** `data size`
 > - **默认值：** `32MB`
 > 
->等待上游任务拉取的任务数据的输出缓冲区大小。如果任务输出是哈希分区的，则缓冲区将在所有分区的消费者之间共享。如果网络延迟高或集群中有许多节点，则增加此值可以提高阶段之间传输数据的网络吞吐量。
+> 上游任务等待拉取任务数据的输出缓冲区大小。如果任务输出是经过哈希分区的，那么缓冲区将在所有分区的使用者之间共享。如果网络延迟较高或集群中有多个节点，增加此值可以提高在阶段之间传输的数据的网络吞吐量。
 
- ## 故障恢复处理属性
- 
- ### 失败重试策略
- 
- ### `failure.recovery.retry.profile`
- 
- > -   **类型：** `string`
- > -   **默认值：** `default`
- >
- > 此属性定义用于确定HTTP客户端上是否发生故障的故障检测配置文件。此属性的值`<profile-name>`必须对应`etc/failure-retry-policy/`路径中的`<profile-name>.properties`文件。如果没有此类配置文件可用，并且未设置此属性，则使用“default”配置文件。
- > 例如，`failure.recovery.retry.profile="test"`要求`test.properties`文件存在于`etc/failure-retry-policy`路径中。
- > `test.properties`文件必须包含指定的`failure.recovery.retry.type`。
- 
- 
- ### `failure.recovery.retry.type`
- 
- > -   **类型：** `string`
- > -   **默认值：** `timeout`
- >
- > 此属性用来设置正在使用的故障检测机制。默认值是基于`timeout`的故障检测。
- 
- #### 基于`timeout`的故障检测
- 
- > 如果使用此机制，HTTP客户端故障将在指定时间段内重试，重试失败则被视为永久故障。
- >
- > 可以为此类故障检测定义`max.error.duration`属性。
- 
- #### 基于`max-retry`的故障检测
- 
- > 如果使用此机制，HTTP客户端故障将在被视为永久故障之前重试指定次数。
- > 可以为此类故障检测定义`max.retry.count`和`max.error.duration`属性。
- > 在这种类型的故障检测中，在查询故障检测模块之前，会执行`max.retry.count`次重试。当故障检测器模块检测到远程节点发生故障时，HTTP客户端将此故障视为永久故障。否则，例如，当远程工作节点处于活动状态但没有响应时，在`max.error.duration`指定的时间段内重试，重试失败则被视为永久故障。
- 
- ### `max.error.duration`
- 
- > -   **类型：** `duration`
- > -   **默认值：** `300s`
- >
- > 被视为永久故障前，协调器等待解决任务间相关错误的最长时间。
- 
- ### `max.retry.count`
- 
- > -   **类型：** `integer`
- > -   **默认值：** `100`
- >
- > 协调器在向故障检测器模块查询远程节点状态之前，对失败任务执行的最大重试次数。
- > 此属性指定查询失败检测模块之前的最小重试次数。因此，实际故障数量可能会因为集群大小和集群负载而略有不同。
- > 此属性仅用于基于`max-retry`的故障检测配置文件。
- > 最小值为100。
- 
- ### 故障检测Gossip协议配置
- 
- ### `failure-detection-protocol`
- 
- >- **类型：** `string`
- >- **默认值：** `heartbeat`
- >
- >此属性定义正在使用的故障检测器的类型。默认配置为`heartbeat`故障检测器。
- >在`config.properties`文件中，将此属性配置为`gossip`，可以启用Gossip协议。
- >集群中的所有节点（即协调器和工作节点）都应在其各自的`etc/config.properties`文件中指定此属性。
- 
- ### `failure-detector.heartbeat-interval`
- 
- >- **类型：** `duration`
- >- **默认值：** `500ms` （500毫秒）
- >
- >集群中两个节点之间的消息散播间隔。
- >在Gossip协议中，两个工作节点间的消息散播频率高于协调器和一个工作节点间。
- >在协调器的`config.properties`文件中，可以为此属性配置一个较大的值，例如`5s`（5秒）。
- >在工作节点中，可以使用默认值。
- 
- ### `failure-detector.worker-gossip-probe-interval`
- 
- >- **类型：** `duration`
- >- **默认值：** `5s`（5秒）
- >
- >Gossip协议使用监控任务（与`heartbeat`故障检测器相同）来监控其他节点。
- >此属性指定监控任务刷新间隔，以触发工作节点消息散播。
- >仅可以为工作节点指定默认值以外的任何其他值。
- >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
- 
- ### `failure-detector.coordinator-gossip-probe-interval`
- 
- >- **类型：** `duration`
- >- **默认值：** `5s`（5秒）
- >
- >Gossip协议使用监控任务（与heartbeat故障检测器相同）来监控其他节点。
- >此属性指定监控任务刷新间隔，以触发协调器参与工作节点消息散播。
- >仅可以为协调器指定默认值以外的任何其他值。
- >该属性的值必须大于`failure-detector.heartbeat-interval`和`failure-detector.worker-gossip-probe-interval`的值。
- 
- ### `failure-detector.coordinator-gossip-collate-interval`
- 
- >- **类型：** `duration`
- >- **默认值：** `2s`（2秒）
- >
- >此属性指定协调器整理从所有工作节点获得的所有散播消息的间隔。
- >此属性只支持为协调器配置。
- >该属性的值必须大于`failure-detector.heartbeat-interval`的值。
- 
- ### `failure-detector.gossip-group-size`
- 
- >- **类型：** `integer`
- >- **默认值：** `Integer.MAX_VALUE`
- >
- >此属性定义单个工作节点在集群中散播消息的工作节点数量。
- >任何大于集群大小（即工作节点数量）的值都意味着all-to-all消息散播。
- >要保持较低的网络开销，针对大型集群，请将此属性设置为一个较小的值（例如，100工作节点的集群设置为10）。
- >每次刷新协调器上的工作节点监视任务时，协调器都会定义工作节点URI列表，其大小由`failure-detector.gossip-group-size`指定，以触发worker-to-worker消息散播。
- >
->
 ## 任务属性
 
 ### `task.concurrency`
@@ -715,7 +517,7 @@
 
 ## 启发式索引属性
 
-启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持Hive连接器的ORC存储格式的表。
+启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持使用ORC存储格式的表支持蜂巢连接器。
 
 ### `hetu.heuristicindex.filter.enabled`
 
@@ -832,7 +634,7 @@
 
 > 自动清空使系统能够通过持续监测需要清空的表来自动管理清空作业，以保持最佳性能。引擎从符合清空条件的数据源获取表，并触发对这些表的清空操作。
 
-### `auto-vacuum.enabled`
+### `auto-vacuum.enabled:`
 
 > - **类型：** `boolean`
 > - **默认值：** `false`
@@ -859,7 +661,7 @@
 >
 > **注意：** 此属性只能在协调节点中配置。
 
-## CTE属性
+## **CTE属性**
 
 ### `cte.cte-max-queue-size`
 
@@ -898,25 +700,18 @@
 > 
 > 远程任务错误最大缓冲时间，超过该时限则查询失败。
 
-## 查询恢复
+## 分布式快照
-
-### `recovery_enabled`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 此会话属性用于启用或禁用恢复框架，该框架在发生故障时启用或禁用查询重启/恢复。
 
 ### `snapshot_enabled`
 
-> -   **类型：** `boolean`
+> - 类型：`boolean`
-> -   **默认值：** `false`
+> - **默认值**：`false`
 >
-> 启用恢复框架时，启用此会话属性可以在查询执行期间捕获快照。如果未启用恢复框架，则此属性不生效。
+> 此会话属性用于启用或禁用分布式快照功能。
 
 ### `hetu.experimental.snapshot.profile`
 
-> - **类型：**`string`
+> - 类型：`string`
 >
 > 此属性定义用于存储快照的[文件系统](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果将该属性设置为`hetu.experimental.snapshot.profile=snapshot-hdfs1`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`snapshot-hdfs1.properties`，其中包含的必要信息包括身份验证类型、配置和密钥表（如适用）。具体细节请参考[文件系统](../develop/filesystem.md)相关章节。
 >
@@ -924,58 +719,20 @@
 >
 > 作为实验性属性，或可以将快照存储在非文件系统位置，如连接器。
 
-### `hetu.recovery.maxRetries`
+### `hetu.snapshot.maxRetries`
 
-> - **类型：** `integer`
+> - 类型：`int`
-> - **默认值：** `10`
+> - **默认值**：`10`
 >
-> 此属性定义查询错误恢复尝试的最大次数。当达到限制时，查询失败。
+> 此属性定义查询的错误恢复尝试的最大次数。达到限制时，查询失败。
 >
-> 也可以使用`recovery_max_retries`会话属性为每个查询指定此属性。
+> 也可以使用`snapshot_max_retries`会话属性在每个查询基础上指定。
 
-### `hetu.recovery.retryTimeout`
+### `hetu.snapshot.retryTimeout`
 
-> - **类型：** `duration`
+> - 类型：`duration`
 > - **默认值：**`10m`（10分钟）
 >
-> 此属性定义系统等待所有任务成功恢复的最长时间。如果在此时间内有任何任务未就绪，则恢复尝试将被视为失败，查询将尝试从较早的快照恢复（如果可用）。
+> 此属性定义系统等待所有任务成功恢复的最大时长。如果在此超时时限内任何任务未就绪，则认为恢复失败，查询将尝试从较早快照恢复（如果可用）。
 >
-> 也可以使用`recovery_retry_timeout`会话属性为每个查询指定此属性。
+> 也可以使用`snapshot_retry_timeout`会话属性在每个查询基础上指定。
-
-### `hetu.snapshot.useKryoSerialization`
-
-> -   **类型：** `boolean`
-> -   **默认值：** `false`
->
-> 为快照启用基于Kryo的序列化，而不是默认的Java序列化。
-
-## HTTP客户端属性配置
-
-### `http.client.idle-timeout`
-
-> -   **类型:** `duration`
-> -   **默认值:** `30s` （30秒）
->
-> 此参数定义了当http客户端没有任何操作时，其保持连接的时间。
-> 当超过指定时间还没有任何操作的话，将关闭客户端并释放相关资源。
->
-> （注意：建议在高负载环境下，该参数配置大一点。）
-
-### `http.client.request-timeout`
-
-> -   **类型:** `duration`
-> -   **默认值:** `10s` （10秒）
->
-> 此参数定义了http客户端接收响应的时间阈值。
-> 当超过所配置时间，客户端没有接收到任何响应，则视为客户端的请求提交失败。
->
-> （注意： 建议在高负载环境下，该参数配置大一点。）
-
-## 连接器属性配置
-
-### `case-insensitive-name-matching`
-
-> -   **类型:** `boolean`
-> -   **默认值:** `false`
->
-> 不区分大小写匹配数据库和集合名称，默认区分大小写。

hetu-docs/zh/admin/reliable-execution.md

@@ -10,9 +10,9 @@
 
 自版本1.2.0起，openLooKeng支持恢复任务和工作节点故障。
 
-## 启用恢复框架
+## 启用分布式快照
 
-恢复框架对于长时间运行的查询最有用。默认禁用，可以使用会话属性[`recovery_enabled`](properties.md#recovery_enabled)启用和禁用恢复框架。建议仅对可靠性要求高的复杂查询启用该功能。
+分布式快照适用于长时间运行的查询任务。该功能默认为禁用状态，可以通过会话属性[`snapshot_enabled`](properties.md#snapshot_enabled)启用或禁用。建议仅在对可靠性要求高的复杂查询场景下启用该功能。
 
 ## 要求
 
@@ -35,7 +35,7 @@
 
 ## 检测
 
-当协调器与远程任务之间的通信长时间失败时，将触发错误恢复，由[`故障恢复处理属性`](properties.md#故障恢复处理属性)配置控制。
+协调节点与远程任务之间的通信长时间失败时，将触发错误恢复，由[`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration)配置控制。
 
 ## 存储注意事项
 
@@ -53,20 +53,8 @@
 
 从错误和快照中恢复需要成本。捕获快照需要时间，时间长短取决于复杂性。因此，需要在性能和可靠性之间进行权衡。
 
-建议在必要时打开快照捕获，例如对于长时间运行的查询。对于这些类型的工作负载，拍摄快照的开销可以忽略不计。
+建议仅在必要时启用分布式快照，如运行时间较长的查询任务。对于这些类型的工作负载，捕获快照的开销可以忽略不计。
-
-## 快照统计信息
-
-在调试模式下启动CLI时，快照捕获信息和恢复信息将与查询结果一起显示在CLI中。
-
-快照捕获统计信息包括捕获的快照数量、捕获的快照大小、捕获快照所需的CPU时间和在查询期间捕获快照所需的挂钟时间。所有快照和最后一个快照的统计信息会分别显示。
-
-快照恢复信息包括查询期间从快照恢复的次数、加载用于恢复的快照大小、从快照恢复所需的CPU时间和从快照恢复所需的挂钟时间。仅当查询期间发生恢复时，才会显示恢复信息。
-
-此外，在查询正在进行时，将显示捕获的快照数量和恢复的快照的ID。更多详细信息，见下图。
-
-![](../images/snapshot_statistics_cn.png)
 
 ## 配置
 
-与恢复框架功能相关的配置，请参见[属性参考](properties.md#查询恢复)。
+与分布式快照功能相关的配置可参见[属性参考](properties.md#分布式快照)。

hetu-docs/zh/admin/resource-groups.md

@@ -1,73 +1,52 @@
 
 # 资源组
 
-资源组限制资源使用，并可以对在其内部运行的查询执行排队策略，或在子组之间划分资源。一个查询属于单个资源组，并使用该组（及其祖先）的资源。除了对排队查询进行限制外，资源组耗尽资源时不会导致正在运行的查询失败；相反，新的查询将进入排队状态。资源组可以有子组，也可以接受查询，但不能同时执行两者。
+资源组限制资源使用，并且可以对在资源组内运行的查询执行队列策略，或者在子组之间划分资源。一个查询属于单个资源组，并消耗该组（及其祖先）的资源。除了对排队查询的限制之外，当资源组耗尽资源时不会导致正在运行的查询失败。相反，新的查询会进入排队状态。资源组可以有子组，也可以接受查询，但不能两者都进行。
 
-资源组和关联的选择规则由可插拔的管理器配置。添加包含以下内容的`etc/resource-groups.properties`文件，使内置管理器能够读取JSON配置文件：
+资源组和关联的选择规则由可插拔的管理器配置。添加具有如下内容的`etc/resource-groups.properties`文件使内置管理器可以读取JSON配置文件：
 
 ``` properties
 resource-groups.configuration-manager=file
 resource-groups.config-file=etc/resource-groups.json
 ```
 
-将`resource-groups.config-file`的值修改为指向一个JSON配置文件，可以是绝对路径，也可以是相对于openLooKeng数据目录的路径。
+将`resource-groups.config-file`的值修改为指向一个json配置文件，可以是绝对路径，也可以是相对于openLooKeng数据目录的相对路径。
-
-### 其他配置
-
-除了`etc/resource-groups.properties`中的上述属性外，还可以配置以下两个属性，这些属性与kill策略一起使用（详细信息与kill策略的部分相同）
-
-`resource-groups.memory-margin-percent`（可选）-这是两个被认为相同的查询之间所允许的内存变化百分比。在此情况下，查询不会根据内存使用情况进行排序，而是根据查询执行进度进行排序，前提是查询进度差异大于配置差异。默认值为10%。
-
-`resource-groups.query-progress-margin-percent`（可选）-这是两个被认为相同的查询之间所允许的查询执行进度百分比。在此情况下，查询不会根据执行进度进行排序。默认值为5%。
 
 ## 资源组属性
 
-
 - `name`（必填）：群组名称。可以是模板（见下文）。
-
 - `maxQueued`（必填）：最大排队查询数。一旦达到此限制，新的查询将被拒绝。
--   `hardConcuritiesLimit`（必填）：最大运行查询数。
+- `hardConcurrencyLimit`（必填）：最大运行查询数。
--   `softMemoryLimit`（必填）：新查询进入队列之前，该组可以使用的最大分布式内存量。可指定为集群内存的绝对值（如`1GB`）或百分比（如`10%`）。
+- `softMemoryLimit`（必填）：新查询进入队列之前，该组可以使用的最大分布式内存量。可指明为集群内存的绝对值（即`1GB`）或百分比（即`10%`）。
--   `softCpuLimit`（可选）：在对最大运行查询数进行惩罚之前的时间内（参见`cpuQuotaPeriod`），该组可能使用的最大CPU时间。同时必须指定`hardCpuLimit`。
+- `softCpuLimit`（可选）：在对最大运行查询数进行惩罚之前的时间内（参见`cpuQuotaPeriod`）该组可能使用的最大CPU时间。必须还指定`hardCpuLimit`。
 - `hardCpuLimit`（可选）：该组在一段时间内可能使用的最大CPU时间。
--   `schedulingPolicy`（可选）：指定如何选择运行排队的查询，以及子组如何成为符合条件的查询。可以配置为以下三种策略之一。当集群开启高可用模式（多个coordinator）时，仅支持`fair`调度策略：
+- `schedulingPolicy`（可选）：指定如何选择排队的查询来运行，以及子组如何成为符合条件的查询。可以配置以下的策略。注意：当集群开启高可用模式时（多个coordinator），当前只支持`fair`策略：
-    - `fair`（默认）：排队的查询按先进先出的顺序处理，子组必须轮流启动新查询（如果它们有排队的话）。
+  - `fair`（默认）：排队的查询将按先进先出的顺序处理，子组必须轮流启动新的查询（如果它们有排队的话）。
-    - `weighted_fair`：根据子组的`schedulingWeight`和子网并发的查询数量选择子组。子组正在运行的查询预期份额基于当前所有符合条件的子组权重计算。选择与其份额相比并发数最小的子组开始下一次查询。
+  - `weighted_fair`：根据子组的`schedulingWeight`和子组的并发数选择子组。子组正在运行的查询的预期份额基于当前所有符合条件的子组的权重计算。选择与其份额相比并发度最小的子组开始下一次查询。
-    - `weighted`：按照优先级（通过`query_priority`[会话属性](../sql/set-session.md)指定）的随机选择排队的查询。按照`schedulingWeight`的比例选择子组以启动新查询。
+  - `weighted`：按其优先级（通过`query_priority`[会话属性](../sql/set-session.md)指定）的比例随机选择排队的查询。选择子组以按其`schedulingWeight`的比例启动新查询。
   - `query_priority`：所有子组都必须配置`query_priority`。排队的查询将严格按照其优先级进行选择。
 - `schedulingWeight`（可选）：该子组的权重。参见上文。默认为`1`。
 - `jmxExport`（可选）：如果为true，则导出群组统计信息到JMX进行监控。默认为`false`。
--   `subGroups`（可选）：子组列表。
+- `subGroups`（可选）：子群组列表。
-- `killPolicy`（可选）：当查询提交给worker后，如果总内存使用量超过**softMemoryLimit**，选择其中一种策略终止正在运行的查询。
-
-  - `no_kill`（默认值）：不终止查询。
-  - `recent_queries`：根据执行顺序的倒序进行查询终止。
-  - `oldest_queries`：根据执行顺序进行查询终止。
-  - `high_memory_queries`：根据内存使用量进行查询终止。具有较高内存使用量的查询将首先被终止，以便在查询终止次数最少的情况下，释放更多内存。
-    作为此策略的一部分，我们尝试平衡内存使用量和完成百分比。因此，如果两个查询的内存使用量都在限制的10%以内（可通过resource-groups.memory-margin-percent配置），则进度慢（执行的百分比）的查询被终止。如果这两个查询在完成百分比方面的差异在5%以内（可通过resource-groups.query-progress-margin-percent配置），则内存使用量大的查询被终止。
-
-  - `finish_percentage_queries`：根据查询执行百分比进行查询终止。执行百分比最小的查询将首先被终止。
-
 
 ## 选择器规则
 
 - `user`（可选）：用于匹配用户名的正则表达式。
 - `source`（可选）：用于匹配源字符串的正则表达式。
--   `queryType`（可选）：用于匹配提交的查询类型的字符串。
+- `queryType`（可选）：用于匹配提交的查询类型的字符串：
--   `DATA_DEFINITION`：修改/创建/删除模式/表/视图的元数据，以及管理预备语句、权限、会话和事务的查询。
+- `DATA_DEFINITION`：更改/创建/删除模式/表/视图的元数据，以及管理预备语句、权限、会话和事务的查询。
   - `DELETE`：`DELETE`查询。
 - `DESCRIBE`：`DESCRIBE`、`DESCRIBE INPUT`、`DESCRIBE OUTPUT`以及`SHOW`查询。
   - `EXPLAIN`：`EXPLAIN`查询。
   - `INSERT`：`INSERT`和`CREATE TABLE AS`查询。
   - `SELECT`：`SELECT`查询。
--   `clientTags`（可选）：标签列表。为了能成功匹配，此列表中的每个标签都必须在客户端提供的与查询关联的标签列表中。
+- `clientTags`（可选）：标签列表。要匹配，此列表中的每个标记都必须在客户端提供的与查询关联的标记列表中。
--   `group`（必填）：这些查询将运行的组。
+- `group`（required）：这些查询将运行在哪个组中。
 
 选择器按顺序处理，并将使用第一个匹配的选择器。
 
 ## 全局属性
 
-
 - `cpuQuotaPeriod`（可选）：CPU配额的执行周期。
 
 ## 提供选择器属性
@@ -82,64 +61,28 @@ resource-groups.config-file=etc/resource-groups.json
 - CLI：使用`--client-tags`选项。
 - JDBC：在`Connection`实例上设置 `ClientTags`客户端信息属性。
 
-## 限制和终止查询
-
-查询提交给worker后，可能超出内存限制，需要使用以下机制处理正在运行的查询：
-
-- 限制查询
-- 终止查询
-
-### 限制查询
-
-对新的split schedule进行限制，避免worker内存占用进一步增加。如果当前查询资源组的内存使用量已超过**softReservedMemory**，将不会计划进行新的拆分，除非内存使用量低于**softReservedMemory***。
-
-建议配置**softReservedMemory**小于**softMemoryLimit**。
-
-用户还可以选择省略**softReservedMemory**配置 从而禁用限制查询。
-
-### 终止查询
-
-如果查询无法被限制，并且内存使用量超过softMemoryLimit，则将根据配置的终止策略终止查询（使查询失败）。只有子组运行的查询才会被终止。
-
 ## 示例
 
+- 在下面的配置示例中，有几个资源组，其中部分是模板。模板允许管理员动态构建资源组树。例如，在`pipeline_${USER}`组中，`${USER}`将展开为提交查询的用户的名称。同样支持`${SOURCE}`，其将展开为提交查询的源。你也可以在`source`和`user`正则表达式中使用自定义命名变量。
   
--   在以下示例配置中，部分资源组是模板。模板允许管理员动态构建资源组树。例如，在`pipeline_${USER}`组中，`${USER}`将被拓展为提交查询的用户的名称。同时支持`${SOURCE}`拓展为提交查询的源。还可以在`source`和`user`正则表达式中使用自定义命名变量。
+  有四个选择器定义哪些查询在哪个资源组中运行：
-    
-     
-    
-    以下四个选择器可以用来定义哪些查询在哪个资源组中运行：
-    
-     
-    
-    > - 第一个选择器匹配来自`bob`的查询，并将其放置在管理组中。
-    > - 第二个选择器匹配源名称中包括`pipeline`的所有数据定义（DDL）查询，并将其放置在`global.data_definition`组中，以减少此类查询的排队时间，因为它们会被快速执行。
-    > - 第三个选择器匹配源名称中包括`pipeline`的查询，并将其放置在`global.pipeline`组下动态创建的每个用户管道组中。
-    > - 第四个选择器匹配来自BI工具的查询，BI工具有一个源与正则表达式`jdbc#(?.*)`匹配，并且客户端提供的标签是`hi-pri`的超集。这些查询被放置在`global.pipeline.tools`组下动态创建的子组中。动态子组将基于命名变量`toolname`创建，该命名变量从源的正则表达式中提取。假设有一个源为`jdbc#powerfulbi`，用户为`kayla`，客户端标签为`hipri`和`fast`的查询。此查询将被路由到`global.pipeline.bi-powerfulbi.kayla`资源组。
-    > - 最后一个选择器是一个回收器，将所有尚未匹配的查询放入每个用户的临时组中。
-    
-     
-    
   
+  > - 第一个选择器将匹配来自`bob`的查询并将其置于管理组中。
+  > - 第二个选择器匹配来自包括`pipeline`的源名称的所有数据定义（DDL）查询并将其放入`global.data_definition`组中。这可以帮助减少此类查询的排队时间，因为它们本应很快。
+  > - 第三个选择器将匹配来自包含`pipeline`的源名称的查询，并将它们放在`global.pipeline`组下动态创建的每个用户管道组中。
+  > - 第四个选择器匹配来自BI工具的查询，BI工具有一个匹配正则表达式`jdbc#(?.*)`的源，并且客户端提供的标记是`hi-pri`的超集。这些查询被放置在`global.pipeline.tools`组下的动态创建的子组中。动态子组将基于命名变量`toolname`创建，该命名变量从源的正则表达式的中提取。考虑源为`jdbc#powerfulbi`、用户为`kayla`、客户端标记为`hipri`和`fast`的查询。此查询将被路由到`global.pipeline.bi-powerfulbi.kayla`资源组。
+  > - 最后一个选择器是一个回收器，将所有尚未匹配的查询放入到每个用户特定的组中。
   
   这些选择器共同执行以下策略：
   
-     
+  - 用户`bob`是管理员用户，可以运行最多50个并发查询。查询将根据用户提供的优先级运行。
-    
-    - 用户`bob`是管理员，最多可以运行50个并发查询。查询将根据用户提供的优先级运行。
-    
-     
   
   对于其余用户：
   
-     
+  - 同时运行的查询总数不能超过100个。
-    
-    - 并发运行的查询总数不得超过100个。
   - 最多可以运行5个源为`pipeline`的并发DDL查询。查询按先进先出顺序运行。
-    - 非DDL查询将在`global.Peline`组下运行，总并发量为45，每个用户并发量为5。查询按先进先出顺序运行。
+  - 非DDL查询将在`global.pipeline`组下运行，总并发量为45，每用户并发量为5。查询按先进先出顺序运行。
   - 对于BI工具，每个工具最多可以运行10个并发查询，每个用户最多可以运行3个并发查询。如果总需求超过10的限制，那么运行查询最少的用户将获得下一个并发槽位。这种策略在争用时保证公平。
-    - 其余的查询都放在`global.adhoc.other`下的每个用户组中，每个用户组的行为类似。
+  - 所有其余的查询都放在`global.adhoc.other`下行为类似的每个用户组中。
-
 
 [resource-groups-example.json](resource-groups-example.json)
-