From 684e72e62df2fed5176fc72c220da9683e751f33 Mon Sep 17 00:00:00 2001 From: sandeep Date: Tue, 17 Nov 2020 16:48:21 +0530 Subject: [PATCH] 388 [I25VGI] Auto Cleanup UT not stable --- .../TestCarbonAllDataType.java | 301 ------------- .../TestCarbondataAutoCleanup.java | 411 ++++++++++++++++++ 2 files changed, 411 insertions(+), 301 deletions(-) create mode 100644 hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java diff --git a/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java b/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java index 317f4e91a..ed955d393 100644 --- a/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java +++ b/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbonAllDataType.java @@ -1498,281 +1498,6 @@ public class TestCarbonAllDataType CarbonMetadata.getInstance().loadTableMetadata(tableInfo); } - - @Test - public void testAutoCleanupInUpdate() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup1"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup1 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup1 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup1/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - hetuServer.execute("UPDATE testdb.testtableautocleanup1 SET a=232 WHERE b=511"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false); - } - catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup1"); - } - - @Test - public void testAutoCleanupInVacuum() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup2"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup2 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (130, 411)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup2/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup2"); - } - - @Test - public void testAutoCleanupInDelete() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup3"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup3"); - } - - @Test - public void testAutoCleanupInDeleteWithPushdown() throws SQLException - { - try { - hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = true"); - hetuServer.execute("drop table if exists testdb.testtableautocleanup3"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - } - finally { - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup3"); - hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = false"); - } - } - - @Test - public void testAutoCleanupInInsert() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup4"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup4 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup4 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup4/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup4"); - } - - @Test - public void testFullAutoCleanupInUpdate() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup5"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup5 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup5 AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup5/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("UPDATE testdb.testtableautocleanup5 SET a=232 WHERE b=511"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false); - } - catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup5"); - } - - @Test - public void testFullAutoCleanupInVacuum() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup6"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup6 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (130, 411)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup6/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup6"); - } - - @Test - public void testFullAutoCleanupInDelete() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup7"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup7 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup7 FULL AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup7/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("DELETE FROM testdb.testtableautocleanup7 WHERE a=130"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup7"); - } - - @Test - public void testFullAutoCleanupInInsert() throws SQLException - { - hetuServer.execute("drop table if exists testdb.testtableautocleanup8"); - hetuServer.execute("CREATE TABLE testdb.testtableautocleanup8 (a int, b int)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (10, 11)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (110, 211)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (120, 311)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 411)"); - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)"); - hetuServer.execute("vacuum table testdb.testtableautocleanup8 FULL AND WAIT"); - - reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup8/Metadata"); - CarbondataMetadata.enableTracingCleanupTask(true); - - hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)"); - try { - CarbondataMetadata.waitForSubmittedTasksFinish(); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_0", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_1", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false); - assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false); - } catch (IOException exception) { - - } - - CarbondataMetadata.enableTracingCleanupTask(false); - hetuServer.execute("drop table testdb.testtableautocleanup8"); - } - @Test public void testCheckCreateTablePartitionByNotSupported() throws SQLException { @@ -1808,30 +1533,4 @@ public class TestCarbonAllDataType assertEquals("true", "false"); } - private void reduceModificationOrdeletionTimesStamp(String tableMetadatPath) - { - LoadMetadataDetails[] metadataDetails = - SegmentStatusManager.readLoadMetadata(tableMetadatPath); - for (LoadMetadataDetails oneLoad : metadataDetails) { - long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); - - String modificationOrdeletionTimesStamp = Long.toString(deletionTime); - deletionTime -= 70000L; - String replace = Long.toString(deletionTime); - System.out.println("replace" + replace); - - Path path = Paths.get(tableMetadatPath + "/tablestatus"); - Charset charset = StandardCharsets.UTF_8; - - String content = null; - try { - content = new String(Files.readAllBytes(path), charset); - - content = content.replaceFirst(modificationOrdeletionTimesStamp, replace); - Files.write(path, content.getBytes(charset)); - } catch (IOException e) { - e.printStackTrace(); - } - } - } } diff --git a/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java b/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java new file mode 100644 index 000000000..53f38e3c1 --- /dev/null +++ b/hetu-carbondata/src/test/java/io/hetu/core/plugin/carbondata/integrationtest/TestCarbondataAutoCleanup.java @@ -0,0 +1,411 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.hetu.core.plugin.carbondata.integrationtest; + +import io.hetu.core.plugin.carbondata.CarbondataMetadata; +import io.hetu.core.plugin.carbondata.server.HetuTestServer; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastore.impl.FileFactory; +import org.apache.carbondata.core.statusmanager.LoadMetadataDetails; +import org.apache.carbondata.core.statusmanager.SegmentStatusManager; +import org.apache.carbondata.core.util.CarbonProperties; +import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.log4j.Logger; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; + +import static org.testng.Assert.assertEquals; + +public class TestCarbondataAutoCleanup { + + private final Logger logger = LogServiceFactory.getLogService(TestCarbondataAutoCleanup.class.getCanonicalName()); + + private String rootPath = new File(this.getClass().getResource("/").getPath() + "../..") + .getCanonicalPath(); + + private String storePath = rootPath + "/target/store_autocleanup"; + private String systemPath = rootPath + "/target/system"; + private HetuTestServer hetuServer = new HetuTestServer(); + private String carbonStoreLocation = storePath + "/carbon.store"; + + public TestCarbondataAutoCleanup() throws Exception + { + } + + @BeforeClass + public void setup() throws Exception + { + logger.info("Setup begin: " + this.getClass().getSimpleName()); + String dataPath = rootPath + "/src/test/resources/alldatatype.csv"; + + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, "HetuTest"); + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME, "0"); + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_SEGMENT_LOCK_FILES_PRESERVE_HOURS, "0"); + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_INVISIBLE_SEGMENTS_PRESERVE_COUNT, "1"); + + Map map = new HashMap(); + map.put("hive.metastore", "file"); + map.put("hive.allow-drop-table", "true"); + map.put("hive.metastore.catalog.dir", "file://" + storePath + "/hive.store"); + map.put("carbondata.store-location", "file://" + carbonStoreLocation); + map.put("carbondata.minor-vacuum-seg-count", "4"); + map.put("carbondata.major-vacuum-seg-size", "1"); + + if (!FileFactory.isFileExist( storePath + "/carbon.store")) { + FileFactory.mkdirs( storePath + "/carbon.store"); + } + + hetuServer.startServer("testdb", map); + hetuServer.execute("drop table if exists testdb.testtableautocleanup1"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup2"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup3"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup4"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup5"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup6"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup7"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup8"); + hetuServer.execute("drop schema if exists testdb"); + hetuServer.execute("drop schema if exists default"); + hetuServer.execute("create schema testdb"); + hetuServer.execute("create schema default"); + + logger.info("CarbonStore created at location : " + storePath); + } + + @AfterClass + public void tearDown() throws SQLException, IOException, InterruptedException + { + logger.info("TearDown begin: " + this.getClass().getSimpleName()); + hetuServer.stopServer(); + CarbonUtil.deleteFoldersAndFiles(FileFactory.getCarbonFile(storePath)); + } + + + @Test + public void testAutoCleanupInUpdate() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup1"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup1 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup1 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup1 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup1/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + hetuServer.execute("UPDATE testdb.testtableautocleanup1 SET a=232 WHERE b=511"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup1/Fact/Part0/Segment_3", false), false); + } + catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup1"); + } + + @Test + public void testAutoCleanupInVacuum() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup2"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup2 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup2 VALUES (130, 411)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup2/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("vacuum table testdb.testtableautocleanup2 AND WAIT"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup2/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup2"); + } + + @Test + public void testAutoCleanupInDelete() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup3"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup3"); + } + + @Test + public void testAutoCleanupInDeleteWithPushdown() throws SQLException + { + try { + hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = true"); + hetuServer.execute("drop table if exists testdb.testtableautocleanup3"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup3 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup3 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup3 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup3/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("DELETE FROM testdb.testtableautocleanup3 WHERE a=130"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup3/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + } + finally { + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup3"); + hetuServer.execute("set session carbondata.orc_predicate_pushdown_enabled = false"); + } + } + + @Test + public void testAutoCleanupInInsert() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup4"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup4 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup4 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup4/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("INSERT INTO testdb.testtableautocleanup4 VALUES (130, 511)"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup4/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup4"); + } + + @Test + public void testFullAutoCleanupInUpdate() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup5"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup5 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup5 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup5 AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup5/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("UPDATE testdb.testtableautocleanup5 SET a=232 WHERE b=511"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup5/Fact/Part0/Segment_3", false), false); + } + catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup5"); + } + + @Test + public void testFullAutoCleanupInVacuum() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup6"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup6 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup6 VALUES (130, 411)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup6/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("vacuum table testdb.testtableautocleanup6 FULL AND WAIT"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup6/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup6"); + } + + @Test + public void testFullAutoCleanupInDelete() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup7"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup7 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup7 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup7 FULL AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup7/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("DELETE FROM testdb.testtableautocleanup7 WHERE a=130"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup7/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup7"); + } + + @Test + public void testFullAutoCleanupInInsert() throws SQLException + { + hetuServer.execute("drop table if exists testdb.testtableautocleanup8"); + hetuServer.execute("CREATE TABLE testdb.testtableautocleanup8 (a int, b int)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (10, 11)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (110, 211)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (120, 311)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 411)"); + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)"); + hetuServer.execute("vacuum table testdb.testtableautocleanup8 FULL AND WAIT"); + + reduceModificationOrdeletionTimesStamp(storePath + "/carbon.store/testdb/testtableautocleanup8/Metadata"); + CarbondataMetadata.enableTracingCleanupTask(true); + + hetuServer.execute("INSERT INTO testdb.testtableautocleanup8 VALUES (130, 511)"); + try { + CarbondataMetadata.waitForSubmittedTasksFinish(); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_0", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_1", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_2", false), false); + assertEquals(FileFactory.isFileExist(storePath + "/carbon.store/testdb/testtableautocleanup8/Fact/Part0/Segment_3", false), false); + } catch (IOException exception) { + + } + + CarbondataMetadata.enableTracingCleanupTask(false); + hetuServer.execute("drop table testdb.testtableautocleanup8"); + } + + private void reduceModificationOrdeletionTimesStamp(String tableMetadatPath) + { + LoadMetadataDetails[] metadataDetails = + SegmentStatusManager.readLoadMetadata(tableMetadatPath); + for (LoadMetadataDetails oneLoad : metadataDetails) { + long deletionTime = oneLoad.getModificationOrdeletionTimesStamp(); + + String modificationOrdeletionTimesStamp = Long.toString(deletionTime); + deletionTime -= 70000L; + String replace = Long.toString(deletionTime); + System.out.println("replace" + replace); + + Path path = Paths.get(tableMetadatPath + "/tablestatus"); + Charset charset = StandardCharsets.UTF_8; + + String content = null; + try { + content = new String(Files.readAllBytes(path), charset); + + content = content.replaceFirst(modificationOrdeletionTimesStamp, replace); + Files.write(path, content.getBytes(charset)); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + +}