diff --git a/hetu-docs/en/indexer/bitmap.md b/hetu-docs/en/indexer/bitmap.md index 2b111603d..29c28e829 100644 --- a/hetu-docs/en/indexer/bitmap.md +++ b/hetu-docs/en/indexer/bitmap.md @@ -24,6 +24,12 @@ such as a Gender column. ## Supported operators = Equality + > Greater than + >= Greater than or equal + < Less than + <= Less than or equal + BETWEEN Between range + IN IN set ## Supported column types "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" @@ -42,4 +48,10 @@ create index idx using bitmap on hive.hindex.users (gender) where regionkey in ( Using index: ```sql select name from hive.hindex.users where gender="female" +select * from hive.hindex.users where id>123 +select * from hive.hindex.users where id<123 +select * from hive.hindex.users where id>=123 +select * from hive.hindex.users where id<=123 +select * from hive.hindex.users where id between (100, 200) +select * from hive.hindex.users where id in (123, 199) ``` \ No newline at end of file diff --git a/hetu-docs/zh/indexer/bitmap.md b/hetu-docs/zh/indexer/bitmap.md index 5ce24d704..dd20cb483 100644 --- a/hetu-docs/zh/indexer/bitmap.md +++ b/hetu-docs/zh/indexer/bitmap.md @@ -18,7 +18,13 @@ BitMap索引在拥有较少不同值数量的列上比较适用，例如：性 ## 支持的运算符 - = Equality + = Equality + > Greater than + >= Greater than or equal + < Less than + <= Less than or equal + BETWEEN Between range + IN IN set ## 支持的列类型 "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" @@ -37,4 +43,10 @@ create index idx using bitmap on hive.hindex.users (gender) where regionkey in ( 使用: ```sql select name from hive.hindex.users where gender="female" +select * from hive.hindex.users where id>123 +select * from hive.hindex.users where id<123 +select * from hive.hindex.users where id>=123 +select * from hive.hindex.users where id<=123 +select * from hive.hindex.users where id between (100, 200) +select * from hive.hindex.users where id in (123, 199) ``` \ No newline at end of file diff --git a/hetu-heuristic-index/pom.xml b/hetu-heuristic-index/pom.xml index eb4828c83..c2873316b 100644 --- a/hetu-heuristic-index/pom.xml +++ b/hetu-heuristic-index/pom.xml @@ -18,6 +18,25 @@ 1.7.30 + + + + org.apache.maven.plugins + maven-surefire-plugin + + + 2 + false + + + + + org.jetbrains.kotlin diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/bitmap/BitmapIndex.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/bitmap/BitmapIndex.java index de2949453..116c44ff3 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/bitmap/BitmapIndex.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/bitmap/BitmapIndex.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableSet; import io.prestosql.spi.heuristicindex.Index; import io.prestosql.spi.heuristicindex.Pair; import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.Marker; import io.prestosql.spi.predicate.Range; import io.prestosql.spi.predicate.SortedRangeSet; import org.apache.commons.io.IOUtils; @@ -28,7 +29,6 @@ import org.mapdb.DBMaker; import org.mapdb.Serializer; import org.mapdb.serializer.GroupSerializer; import org.mapdb.serializer.SerializerCompressionWrapper; -import org.roaringbitmap.PeekableIntIterator; import org.roaringbitmap.RoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import org.xerial.snappy.SnappyInputStream; @@ -53,6 +53,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.atomic.AtomicBoolean; import static io.hetu.core.heuristicindex.util.TypeUtils.getNativeValue; @@ -184,6 +185,26 @@ public class BitmapIndex return lookUp(expression).hasNext(); } + private RoaringBitmap lookUpSingle(Object lookupValue) + { + try { + Object objValue = btree.get(lookupValue); + + if (objValue == null) { + return null; + } + + byte[] value = (byte[]) objValue; + ByteBuffer bb = ByteBuffer.wrap(value); + ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(bb); + RoaringBitmap roaringBitmap = new RoaringBitmap(bitmap); + return roaringBitmap; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + @Override public Iterator lookUp(Object expression) { @@ -194,56 +215,74 @@ public class BitmapIndex List ranges = ((SortedRangeSet) (predicate.getValues())).getOrderedRanges(); Class javaType = predicate.getValues().getType().getJavaType(); - if (ranges.size() != 1) { - throw new UnsupportedOperationException("Bitmap only supports single equality expressions"); - } - - Object lookupValue = null; - for (Range range : ranges) { - // unique value(for example: id=1, id in (1,2)), bound: EXACTLY - if (range.isSingleValue()) { - lookupValue = getNativeValue(range.getSingleValue()); - break; - } - else { - throw new UnsupportedOperationException("Bitmap only supports single equality expressions"); - } - } - try { - Object objValue = getBtreeReadOptimized().get(lookupValue); + btree = getBtreeReadOptimized(); + ArrayList allMatches = new ArrayList<>(); + for (Range range : ranges) { + if (range.isSingleValue()) { + // unique value(for example: id=1, id in (1,2) (IN operator gives single exact values one by one)), bound: EXACTLY + RoaringBitmap bitmap = lookUpSingle(getNativeValue(range.getSingleValue())); + if (bitmap != null) { + allMatches.add(bitmap); + } + } + else { + // <, <=, >=, >, BETWEEN + boolean highBoundless = range.getHigh().isUpperUnbounded(); + boolean lowBoundless = range.getLow().isLowerUnbounded(); + ConcurrentNavigableMap concurrentNavigableMap = null; - if (objValue == null) { - return Collections.emptyIterator(); + if (highBoundless && !lowBoundless) { + // >= or > + Object low = range.getLow().getValue(); + Object high = btree.lastKey(); + boolean fromInclusive = range.getLow().getBound().equals(Marker.Bound.EXACTLY); + if (btree.comparator().compare(low, high) > 0) { + Object temp = low; + low = high; + high = temp; + } + concurrentNavigableMap = btree.subMap(low, fromInclusive, high, true); + } + else if (!highBoundless && lowBoundless) { + // <= or < + Object low = btree.firstKey(); + Object high = range.getHigh().getValue(); + boolean toInclusive = range.getHigh().getBound().equals(Marker.Bound.EXACTLY); + if (btree.comparator().compare(low, high) > 0) { + Object temp = low; + low = high; + high = temp; + } + concurrentNavigableMap = btree.subMap(low, true, high, toInclusive); + } + else if (!highBoundless && !lowBoundless) { + // BETWEEN + Object low = range.getHigh().getValue(); + Object high = range.getLow().getValue(); + if (btree.comparator().compare(low, high) > 0) { + Object temp = low; + low = high; + high = temp; + } + concurrentNavigableMap = btree.subMap(low, true, high, true); + } + else { + // This case, combined gives a range of boundless for both high and low end + throw new UnsupportedOperationException("No use for bitmap index as all values are matched due to no bounds."); + } + + for (Object i : concurrentNavigableMap.keySet()) { + RoaringBitmap bitmap = lookUpSingle(getNativeValue(i)); + allMatches.add(bitmap); + } + } } - byte[] value = (byte[]) objValue; - ByteBuffer bb = ByteBuffer.wrap(value); - ImmutableRoaringBitmap bitmap = new ImmutableRoaringBitmap(bb); - - PeekableIntIterator iterator = bitmap.getIntIterator(); - - if (iterator == null) { - return Collections.emptyIterator(); - } - - return new Iterator() - { - @Override - public boolean hasNext() - { - return iterator.hasNext(); - } - - @Override - public Integer next() - { - return iterator.next(); - } - }; + return RoaringBitmap.or(allMatches.iterator()).iterator(); } catch (Exception e) { - throw new RuntimeException(e); + throw new UnsupportedOperationException("Unsupported expression type.", e); } } else { diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java index e1b46844d..d1b7e6818 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/btree/BTreeIndex.java @@ -250,6 +250,8 @@ public class BTreeIndex concurrentNavigableMap = dataMap.subMap(key, true, dataMap.lastKey(), true); result.addAll(concurrentNavigableMap.values().stream().map(this::translateSymbols).flatMap(Collection::stream).collect(Collectors.toList())); break; + default: + throw new UnsupportedOperationException("Expression not supported"); } } } @@ -270,6 +272,8 @@ public class BTreeIndex } } break; + default: + throw new UnsupportedOperationException("Expression not supported"); } } else { diff --git a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/minmax/MinMaxIndex.java b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/minmax/MinMaxIndex.java index afc8e7c57..0c6136121 100644 --- a/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/minmax/MinMaxIndex.java +++ b/hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/minmax/MinMaxIndex.java @@ -129,7 +129,7 @@ public class MinMaxIndex case GREATER_THAN_OR_EQUAL: return value.compareTo(max) < 0 || value.compareTo(max) == 0; default: - throw new IllegalArgumentException("Unsupported operator " + operator); + throw new UnsupportedOperationException("Unsupported operator " + operator); } } } diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HeuristicIndexUtConstants.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HeuristicIndexUtConstants.java deleted file mode 100644 index 864d52f79..000000000 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HeuristicIndexUtConstants.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.heuristicindex; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class HeuristicIndexUtConstants -{ - /** - * list of cvs columns data types - */ - public static final List CVS_COLUMNS_DATA_TYPES = Collections.unmodifiableList(new ArrayList() { - { - this.add(Long.class.getName()); - this.add(String.class.getName()); - } - }); - - private HeuristicIndexUtConstants() - { - } -} diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHeuristicIndexWriter.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHeuristicIndexWriter.java deleted file mode 100644 index 3e29c592f..000000000 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHeuristicIndexWriter.java +++ /dev/null @@ -1,354 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -//package io.hetu.core.heuristicindex; -// -//import io.hetu.core.common.filesystem.TempFolder; -//import io.hetu.core.filesystem.HetuLocalFileSystemClient; -//import io.hetu.core.filesystem.LocalConfig; -//import io.hetu.core.plugin.heuristicindex.index.bloom.BloomIndex; -//import io.hetu.core.plugin.heuristicindex.index.minmax.MinMaxIndex; -//import io.prestosql.spi.filesystem.HetuFileSystemClient; -//import io.prestosql.spi.heuristicindex.DataSource; -//import io.prestosql.spi.heuristicindex.Index; -//import org.apache.commons.compress.archivers.ArchiveEntry; -//import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -//import org.testng.annotations.Test; -// -//import java.io.File; -//import java.io.IOException; -//import java.nio.file.Files; -//import java.nio.file.Path; -//import java.nio.file.Paths; -//import java.util.HashSet; -//import java.util.Properties; -//import java.util.Set; -//import java.util.stream.Collectors; -// -//import static org.mockito.Mockito.mock; -//import static org.testng.Assert.assertEquals; -//import static org.testng.Assert.assertFalse; -//import static org.testng.Assert.assertThrows; -//import static org.testng.Assert.assertTrue; -// -//public class TestHeuristicIndexWriter -//{ -// private static final Logger LOG = LoggerFactory.getLogger(TestHeuristicIndexWriter.class); -// -// @Test -// public void testIndexWriterSimple() -// throws IOException -// { -// // Simple workflow without calling readSplit's callback -// DataSource ds = mock(DataSource.class); -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), Paths.get("/")); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, null); -// -// // Runtime exception will be thrown because the table was empty -// assertThrows(RuntimeException.class, () -> writer.createIndex("catalog.schema.table", new String[] {"test"}, new String[] {}, "bloom")); -// -// // Null check -// assertThrows(RuntimeException.class, () -> new FileIndexWriter(null, null, null, null)); -// -// // Invalid table format -// assertThrows(RuntimeException.class, -// () -> writer.createIndex("invalid", new String[] {"test"}, new String[] {}, "bloom")); -// } -// -// @Test -// public void testUnsupportedIndexType() -// throws IOException -// { -// DataSource ds = new TestDataSource(); -// try (TempFolder folder = new TempFolder()) { -// folder.create(); -// -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath()); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, folder.getRoot().toPath()); -// String tableName = "catalog.schema.table"; -// -// assertThrows(RuntimeException.class, -// () -> writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "Random")); -// assertIndexWriterCleanUp(folder.getRoot().toPath(), tableName); -// } -// } -// -// @Test -// public void testIndexWriterCallback() -// throws IOException -// { -// DataSource ds = new TestDataSource(); -// try (TempFolder folder = new TempFolder()) { -// folder.create(); -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath()); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, folder.getRoot().toPath()); -// String tableName = "catalog.schema.table"; -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "bloom"); -// -// // Assert index files are created -// File indexFolder = new File(folder.getRoot().getAbsolutePath() + "/" + tableName); -// assertTrue(indexFolder.listFiles().length > 0); -// } -// } -// -// /** -// * create a bloom index followed by a minmax index -// *

-// * the lastModifiedTime of the datasource split will change, this means -// * only the latter minmax index should remain -// * -// * @throws IOException -// */ -// @Test -// public void testIndexWriterMultipleWritesExpired() -// throws IOException -// { -// DataSource ds = new DataSource() -// { -// @Override -// public String getId() -// { -// return "test"; -// } -// -// @Override -// public void readSplits(String schema, String table, String[] columns, String[] partitions, DataSource.Callback callback) -// { -// Object[] values = new Object[] {"test", "dsfdfs", "random"}; -// callback.call("UT_test_column", values, "UT_test", 100, System.currentTimeMillis(), 0); -// } -// }; -// try (TempFolder folder = new TempFolder()) { -// folder.create(); -// -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath()); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// indices.add(new MinMaxIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, folder.getRoot().toPath()); -// String tableName = "catalog.schema.table"; -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "bloom"); -// -// File indexFolder = new File(folder.getRoot().getAbsolutePath() + "/" + tableName); -// LOG.info("Previous files:"); -// Set previousFiles = Files.walk(Paths.get(indexFolder.getAbsolutePath())) -// .filter(Files::isRegularFile).collect(Collectors.toSet()); -// previousFiles.forEach(f -> LOG.info(f.toString())); -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "bloom"); -// LOG.info("New files:"); -// Set newFiles = Files.walk(Paths.get(indexFolder.getAbsolutePath())) -// .filter(Files::isRegularFile).collect(Collectors.toSet()); -// newFiles.forEach(f -> LOG.info(f.toString())); -// -// // catalog.schema.table/UT_test_column/minmax/UT_test/lastModified=123.tar -// assertEquals(newFiles.size(), 1); -// -// // all files should be different -// for (Path previousFile : previousFiles) { -// assertFalse(newFiles.contains(previousFile), "should not have found " + previousFile); -// } -// } -// } -// -// /** -// * create a bloom index followed by a minmax index -// *

-// * the lastModifiedTime of the datasource split will remain the same -// * this means both bloom and minmax index should remain with -// * the same lastModifiedFile -// * -// * @throws IOException -// */ -// @Test -// public void testIndexWriterMultipleWrites() -// throws IOException -// { -// DataSource ds = new DataSource() -// { -// @Override -// public String getId() -// { -// return "test"; -// } -// -// @Override -// public void readSplits(String schema, String table, String[] columns, String[] partitions, DataSource.Callback -// callback) -// { -// Object[] values = new Object[] {"test", "dsfdfs", "random"}; -// callback.call("UT_test_column", values, "UT_test", 100, 123, 0); -// } -// }; -// -// try (TempFolder folder = new TempFolder()) { -// folder.create(); -// -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath()); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// indices.add(new MinMaxIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, folder.getRoot().toPath()); -// String tableName = "catalog.schema.table"; -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "bloom"); -// -// File indexFolder = new File(folder.getRoot().getAbsolutePath() + "/" + tableName); -// LOG.info("Previous files:"); -// Set previousFiles = Files.walk(Paths.get(indexFolder.getAbsolutePath())) -// .filter(Files::isRegularFile).collect(Collectors.toSet()); -// previousFiles.forEach(f -> LOG.info(f.toString())); -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "minmax"); -// LOG.info("New files:"); -// Set newFiles = Files.walk(Paths.get(indexFolder.getAbsolutePath())) -// .filter(Files::isRegularFile).collect(Collectors.toSet()); -// newFiles.forEach(f -> LOG.info(f.toString())); -// -// // two files: -// // catalog.schema.table/UT_test_column/minmax/UT_test/lastModified=123.tar -// // catalog.schema.table/UT_test_column/bloom/UT_test/lastModified=123.tar -// assertEquals(newFiles.size(), 2); -// assertTarEntry(newFiles.iterator().next(), 1); -// -// // previous files should still be there -// for (Path previousFile : previousFiles) { -// assertTrue(newFiles.contains(previousFile), "did not find " + previousFile); -// } -// } -// } -// -// /** -// * create a bloom index with debug on, this will also write -// * the data of each split into a file alongside the index file -// * -// * @throws IOException -// */ -// @Test -// public void testDebugMode() -// throws IOException -// { -// DataSource ds = new DataSource() -// { -// @Override -// public String getId() -// { -// return "test"; -// } -// -// @Override -// public void readSplits(String schema, String table, String[] columns, String[] partitions, DataSource.Callback -// callback) -// { -// Object[] values = new Object[] {"test", "dsfdfs", "random"}; -// callback.call("UT_test_column", values, "UT_test", 100, System.currentTimeMillis(), 0); -// } -// }; -// -// try (TempFolder folder = new TempFolder()) { -// folder.create(); -// -// HetuFileSystemClient fs = new HetuLocalFileSystemClient(new LocalConfig(new Properties()), folder.getRoot().toPath()); -// -// Set indices = new HashSet<>(); -// indices.add(new BloomIndex()); -// -// FileIndexWriter writer = new FileIndexWriter(ds, indices, fs, folder.getRoot().toPath()); -// String tableName = "catalog.schema.table"; -// -// writer.createIndex(tableName, new String[] {"test"}, new String[] {}, "bloom", true); -// -// File indexFolder = new File(folder.getRoot().getAbsolutePath() + "/" + tableName); -// LOG.info("Previous files:"); -// Set files = Files.walk(Paths.get(indexFolder.getAbsolutePath())) -// .filter(Files::isRegularFile).collect(Collectors.toSet()); -// files.forEach(f -> LOG.info(f.toString())); -// -// assertEquals(files.size(), 1); -// assertTarEntry(files.iterator().next(), 1); -// } -// } -// -// private void assertIndexWriterCleanUp(Path root, String tableName) -// throws IOException -// { -// // TODO: note empty directories may be left behind -// assertTrue(Files.walk(Paths.get(root.toAbsolutePath().toString())).noneMatch(Files::isRegularFile), -// "all part files and lock files has to be deleted upon error"); -// } -// -// private static class TestDataSource -// implements DataSource -// { -// @Override -// public String getId() -// { -// return "test"; -// } -// -// @Override -// public void readSplits(String schema, String table, String[] columns, String[] partitions, Callback callback) -// throws IOException -// { -// Object[] values = new Object[] {"test", "dsfdfs", "random"}; -// callback.call("UT_test_column", values, "UT_test", 100, System.currentTimeMillis(), 0); -// } -// } -// -// private void assertTarEntry(Path pathToTarFile, int expectedEntryCount) -// throws IOException -// { -// int entryCount = 0; -// try (TarArchiveInputStream i = new TarArchiveInputStream(Files.newInputStream(pathToTarFile))) { -// ArchiveEntry entry; -// while ((entry = i.getNextEntry()) != null) { -// entryCount++; -// } -// } -// -// assertEquals(entryCount, expectedEntryCount); -// } -// -// class FilesystemAndRoot -// { -// HetuFileSystemClient fileSystemClient; -// Path root; -// -// FilesystemAndRoot(HetuFileSystemClient fs, Path root) -// { -// this.fileSystemClient = fs; -// this.root = root; -// } -// } -//} diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindex.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindex.java index 1f28e2aef..f7209df35 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindex.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindex.java @@ -35,9 +35,7 @@ public class TestHindex { String tableName = getNewTableName(); String indexName = getNewIndexName(); - String[] testerData = createTableDataTypeWithQuery(tableName, dataType); - tableName = testerData[0]; - String testerQuery = testerData[1]; + String testerQuery = createTableDataTypeWithQuery(tableName, dataType); // Get splits and result Pair resultPairBeforeIndex = getSplitAndMaterializedResult(testerQuery); @@ -101,7 +99,7 @@ public class TestHindex throws InterruptedException { String tableName = getNewTableName(); - tableName = createTable1(tableName); + createTable1(tableName); String testerQuery = "SELECT * FROM " + tableName + " WHERE " + queryVariable + "=" + queryValue; String indexName = getNewIndexName(); @@ -166,7 +164,7 @@ public class TestHindex throws InterruptedException { String tableName = getNewTableName(); - tableName = createTable2(tableName); + createTable2(tableName); String testerQuery = "SELECT * FROM " + tableName + " WHERE " + queryVariable + "=" + queryValue; String indexName = getNewIndexName(); @@ -240,7 +238,7 @@ public class TestHindex throws InterruptedException { String tableName = getNewTableName(); - tableName = createTableNullData(tableName); + createTableNullData(tableName); String testerQuery = "SELECT * FROM " + tableName + " WHERE " + queryVariable + "=" + queryValue; String indexName = getNewIndexName(); @@ -297,7 +295,7 @@ public class TestHindex { String tableName = getNewTableName(); String testerQuery = "SELECT * FROM " + tableName + " WHERE id = 2"; - tableName = createTable1(tableName); + createTable1(tableName); // Get splits and result Pair resultPairBeforeIndex = getSplitAndMaterializedResult(testerQuery); @@ -345,7 +343,7 @@ public class TestHindex String tableName = getNewTableName(); String indexName = getNewIndexName(); String testerQuery = "SELECT * FROM " + tableName + " WHERE id = 2"; - tableName = createTable1(tableName); + createTable1(tableName); // Get splits and result Pair resultPairBeforeIndex = getSplitAndMaterializedResult(testerQuery); @@ -389,7 +387,7 @@ public class TestHindex { String tableName = getNewTableName(); String indexName = getNewIndexName(); - tableName = createTable1(tableName); + createTable1(tableName); // Create index if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -480,53 +478,32 @@ public class TestHindex } } - @Test - public void testBtreeIndexMultiPartitionedColumn() + @Test(dataProvider = "queryOperatorTest") + public void testQueryOperator(String testerQuery, String indexType) throws InterruptedException { String tableName = getNewTableName(); - tableName = createBtreeTableMultiPart1(tableName); + createTable1(tableName); + testerQuery = "SELECT * FROM " + tableName + " WHERE " + testerQuery; + String indexName = getNewIndexName(); - String indexName1 = getNewIndexName(); - assertQuerySucceeds("CREATE INDEX " + indexName1 + " USING btree ON " + tableName + - " (key1) WITH (level=partition) WHERE key3 = 222"); + if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { + assertQuerySucceeds("CREATE INDEX " + indexName + " USING " + + indexType + " ON " + tableName + " (id) WITH (level='table')"); + } + else { + assertQuerySucceeds("CREATE INDEX " + indexName + " USING " + + indexType + " ON " + tableName + " (id)"); + } - String testerQuery1 = "SELECT * FROM " + tableName + " WHERE key1 = 2"; - - Pair resultPairLoadingIndex1 = getSplitAndMaterializedResult(testerQuery1); - int splitsLoadingIndex1 = resultPairLoadingIndex1.getFirst(); - MaterializedResult resultLoadingIndex1 = resultPairLoadingIndex1.getSecond(); + MaterializedResult resultLoadingIndex = computeActual(testerQuery); // Wait before continuing Thread.sleep(1000); - Pair resultPairIndexLoaded1 = getSplitAndMaterializedResult(testerQuery1); - int splitsIndexLoaded1 = resultPairIndexLoaded1.getFirst(); - MaterializedResult resultIndexLoaded1 = resultPairIndexLoaded1.getSecond(); + MaterializedResult resultIndexLoaded = computeActual(testerQuery); - assertEquals(splitsLoadingIndex1, splitsIndexLoaded1); - assertTrue(verifyEqualResults(resultLoadingIndex1, resultIndexLoaded1), "The results should be equal."); - - // Create second index and do query again on different keys - - String indexName2 = getNewIndexName(); - assertQuerySucceeds("CREATE INDEX " + indexName2 + " USING btree ON " + tableName + - " (key2) WITH (level=partition) WHERE key5 = 22222"); - - String testerQuery2 = "SELECT * FROM " + tableName + " WHERE key2 = 22"; - - Pair resultPairLoadingIndex2 = getSplitAndMaterializedResult(testerQuery2); - int splitsLoadingIndex2 = resultPairLoadingIndex2.getFirst(); - MaterializedResult resultLoadingIndex2 = resultPairLoadingIndex2.getSecond(); - - // Wait before continuing - Thread.sleep(1000); - - Pair resultPairIndexLoaded2 = getSplitAndMaterializedResult(testerQuery2); - int splitsIndexLoaded2 = resultPairIndexLoaded2.getFirst(); - MaterializedResult resultIndexLoaded2 = resultPairIndexLoaded2.getSecond(); - - assertEquals(splitsLoadingIndex2, splitsIndexLoaded2); - assertTrue(verifyEqualResults(resultLoadingIndex2, resultIndexLoaded2), "The results should be equal."); + assertTrue(verifyEqualResults(resultLoadingIndex, resultIndexLoaded), + "The results should be equal for " + testerQuery + " " + indexType); } } diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestBTree.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBTreeIndex.java similarity index 72% rename from hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestBTree.java rename to hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBTreeIndex.java index 1fca3211d..957214c23 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestBTree.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBTreeIndex.java @@ -22,14 +22,14 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; -public class TestBTree +public class TestHindexBTreeIndex extends TestIndexResources { @Test public void testBtreeIndexOnPartitionedColumnCreateAndDelete() { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -41,7 +41,7 @@ public class TestBTree public void testBtreeIndexOnNonePartitionedColumnCreateAndDelete() { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -53,7 +53,7 @@ public class TestBTree public void testBtreeIndexHasKeyWhereDelete() { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -65,7 +65,7 @@ public class TestBTree public void testBtreeIndexInvalidKeyWhereDelete() { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -83,7 +83,7 @@ public class TestBTree throws InterruptedException { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -110,7 +110,7 @@ public class TestBTree public void testBtreeIndexCreationWhereNonPartitionedColumn() { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); assertQueryFails("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -123,7 +123,7 @@ public class TestBTree throws InterruptedException { String tableName = getNewTableName(); - tableName = createBtreeTableTransact1(tableName); + createBtreeTableTransact1(tableName); String indexName = getNewIndexName(); assertQuerySucceeds("CREATE INDEX " + indexName + " USING btree ON " + tableName + @@ -165,7 +165,7 @@ public class TestBTree public void testBtreeIndexOperators(String condition) { String tableName = getNewTableName(); - tableName = createBtreeTable1(tableName); + createBtreeTable1(tableName); String indexName = getNewIndexName(); try { @@ -178,4 +178,54 @@ public class TestBTree " Only in-predicate/equality-expressions are supported e.g. partition=1 or partition=2/partition in (1,2)")); } } + + @Test + public void testBtreeIndexMultiPartitionedColumn() + throws InterruptedException + { + String tableName = getNewTableName(); + createBtreeTableMultiPart1(tableName); + + String indexName1 = getNewIndexName(); + assertQuerySucceeds("CREATE INDEX " + indexName1 + " USING btree ON " + tableName + + " (key1) WITH (level=partition) WHERE key3 = 222"); + + String testerQuery1 = "SELECT * FROM " + tableName + " WHERE key1 = 2"; + + Pair resultPairLoadingIndex1 = getSplitAndMaterializedResult(testerQuery1); + int splitsLoadingIndex1 = resultPairLoadingIndex1.getFirst(); + MaterializedResult resultLoadingIndex1 = resultPairLoadingIndex1.getSecond(); + + // Wait before continuing + Thread.sleep(1000); + + Pair resultPairIndexLoaded1 = getSplitAndMaterializedResult(testerQuery1); + int splitsIndexLoaded1 = resultPairIndexLoaded1.getFirst(); + MaterializedResult resultIndexLoaded1 = resultPairIndexLoaded1.getSecond(); + + assertEquals(splitsLoadingIndex1, splitsIndexLoaded1); + assertTrue(verifyEqualResults(resultLoadingIndex1, resultIndexLoaded1), "The results should be equal."); + + // Create second index and do query again on different keys + + String indexName2 = getNewIndexName(); + assertQuerySucceeds("CREATE INDEX " + indexName2 + " USING btree ON " + tableName + + " (key2) WITH (level=partition) WHERE key5 = 22222"); + + String testerQuery2 = "SELECT * FROM " + tableName + " WHERE key2 = 22"; + + Pair resultPairLoadingIndex2 = getSplitAndMaterializedResult(testerQuery2); + int splitsLoadingIndex2 = resultPairLoadingIndex2.getFirst(); + MaterializedResult resultLoadingIndex2 = resultPairLoadingIndex2.getSecond(); + + // Wait before continuing + Thread.sleep(1000); + + Pair resultPairIndexLoaded2 = getSplitAndMaterializedResult(testerQuery2); + int splitsIndexLoaded2 = resultPairIndexLoaded2.getFirst(); + MaterializedResult resultIndexLoaded2 = resultPairIndexLoaded2.getSecond(); + + assertEquals(splitsLoadingIndex2, splitsIndexLoaded2); + assertTrue(verifyEqualResults(resultLoadingIndex2, resultIndexLoaded2), "The results should be equal."); + } } diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBitmapIndex.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBitmapIndex.java new file mode 100644 index 000000000..fb6ed0ecc --- /dev/null +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexBitmapIndex.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.heuristicindex; + +import io.prestosql.testing.MaterializedResult; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertTrue; + +public class TestHindexBitmapIndex + extends TestIndexResources +{ + @Test(dataProvider = "bitmapOperatorInputRowsTest") + public void testBitmapOperatorInputRows(String predicateQuery, String baseQuery) + throws InterruptedException + { + String tableName = getNewTableName(); + createTable2(tableName); + String indexName = getNewIndexName(); + + // baseQuery is the query at which all, most or at least more selected than predicateQuery + // baseQuery does not use indexing, predicateQuery does + // predicateQuery should have less input rows than baseQuery + baseQuery = "SELECT * FROM " + tableName + baseQuery; + predicateQuery = "SELECT * FROM " + tableName + " WHERE " + predicateQuery; + + MaterializedResult baseQueryResult = computeActual(baseQuery); + long baseQueryInputRows = getInputRowsOfLastQueryExecution(baseQuery); + + assertQuerySucceeds("CREATE INDEX " + indexName + " USING bitmap ON " + tableName + " (key)"); + + MaterializedResult predicateQueryResultLoadingIndex = computeActual(predicateQuery); + + // Wait before continuing + Thread.sleep(1000); + + MaterializedResult predicateQueryResultIndexLoaded = computeActual(predicateQuery); + long predicateQueryInputRowsIndexLoaded = getInputRowsOfLastQueryExecution(predicateQuery); + + assertTrue(verifyEqualResults(predicateQueryResultLoadingIndex, predicateQueryResultIndexLoaded), + "The results should be equal."); + assertTrue(baseQueryResult.getRowCount() > predicateQueryResultIndexLoaded.getRowCount(), + "Predicate query with index loaded should have less results than base query. " + + "baseQueryResult row count: " + baseQueryResult.getRowCount() + + " predicateQueryResultIndexLoaded row count: " + predicateQueryResultIndexLoaded.getRowCount()); + assertTrue(baseQueryInputRows > predicateQueryInputRowsIndexLoaded, + "Predicate query with index loaded should have less input rows than base query. " + + "baseQueryInputRows: " + baseQueryInputRows + + " predicateQueryInputRowsIndexLoaded: " + predicateQueryInputRowsIndexLoaded); + } +} diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java index 0f792e668..f86ca2371 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java @@ -34,7 +34,7 @@ public class TestHindexFailure { String tableName = getNewTableName(); String indexName = getNewIndexName(); - tableName = createTable1(tableName); + createTable1(tableName); // Create index if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -47,7 +47,7 @@ public class TestHindexFailure } assertQueryFails("DROP INDEX hive", "line 1:1: Index 'hive' does not exist"); assertQueryFails("DROP INDEX HDFS", "line 1:1: Index 'hdfs' does not exist"); - assertQueryFails("DROP INDEX test", "line 1:1: Index 'test' does not exist"); + assertQueryFails("DROP INDEX TEST", "line 1:1: Index 'test' does not exist"); assertQueryFails("DROP INDEX wrongtest", "line 1:1: Index 'wrongtest' does not exist"); String[] table = tableName.split("\\."); assertQueryFails("DROP INDEX " + table[2], "line 1:1: Index '" + table[2] + "' does not exist"); @@ -63,7 +63,7 @@ public class TestHindexFailure { String tableName = getNewTableName(); String indexName = getNewIndexName(); - tableName = createTable1(tableName); + createTable1(tableName); // Create index if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -85,7 +85,7 @@ public class TestHindexFailure public void testMultipleSameIndexCreation(String indexType, String queryVariable) { String tableName = getNewTableName(); - tableName = createTable1(tableName); + createTable1(tableName); String indexName1 = getNewIndexName(); if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -120,7 +120,7 @@ public class TestHindexFailure throws IllegalStateException { String tableName = getNewTableName(); - tableName = createEmptyTable(tableName); + createEmptyTable(tableName); String indexName = getNewIndexName(); if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -292,7 +292,7 @@ public class TestHindexFailure public void testIndexWithoutColumnCreation(String indexType) { String tableName = getNewTableName(); - tableName = createTable1(tableName); + createTable1(tableName); // Error of "line 1:115: mismatched input ')'. Expecting: " exists // But code style does not allow ) to exist inside a string without having ( before it. @@ -318,7 +318,7 @@ public class TestHindexFailure throws SemanticException { String tableName = getNewTableName(); - tableName = createTable1(tableName); + createTable1(tableName); String indexName = getNewIndexName(); if (indexType.toLowerCase(Locale.ROOT).equals("btree")) { @@ -339,10 +339,10 @@ public class TestHindexFailure throws ParsingException { String tableName = getNewTableName(); - tableName = createTable1(tableName); + createTable1(tableName); String indexName = getNewIndexName(); assertQueryFails("CREATE INDEX " + indexName + " USING wrong_filter ON " + tableName + " (id)", - "line 1:59: mismatched input 'wrong_filter'. Expecting: '.', 'USING'"); + "line 1:26: mismatched input 'wrong_filter'. Expecting: '.', 'USING'"); } } diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java index b67ec33ae..229299aee 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java @@ -23,8 +23,9 @@ import org.testng.annotations.DataProvider; import java.util.ArrayList; import java.util.Collections; import java.util.Locale; +import java.util.concurrent.atomic.AtomicInteger; -import static org.testng.Assert.assertNotEquals; +import static org.testng.Assert.assertEquals; public class TestIndexResources extends AbstractTestQueryFramework @@ -86,46 +87,67 @@ public class TestIndexResources return new Object[][] {{"BITMAP"}, {"BLOOM"}, {"MINMAX"}, {"BTREE"}}; } - String safeCreateTable(String query, String tableName) + @DataProvider(name = "bitmapOperatorInputRowsTest") + public Object[][] bitmapOperatorInputRowsTest() { - String newName = tableName; - int max = 1; - for (int i = 0; i < 10; i++) { - try { - assertQuerySucceeds(query); - return newName; - } - catch (AssertionError e) { - max *= 10; - newName = tableName + ((int) (Math.random() * max)); - if (newName.length() >= 128) { - throw new RuntimeException("Cannot Create Table: " + tableName + " due to " + e.getCause().toString()); - } - query = query.replaceFirst(tableName, newName); - } - } - assertQuerySucceeds(query); - return newName; + return new Object[][] { + {"key = 5", ""}, {"key IN (1, 2)", ""}, {"key BETWEEN 3 AND 7", " WHERE key BETWEEN 2 AND 8"}, + {"key < 5", " WHERE key <= 5"}, {"key <= 5", " WHERE key <= 8"}, + {"key >= 5", " WHERE key >= 3"}, {"key > 5", " WHERE key >= 5"}, + {"key < 0", " WHERE key > -1"}, {"key <= -1", " WHERE key >= 0"}, + {"key >= 100", " WHERE key >= 3"}, {"key > 99", " WHERE key >= 5"}, + {"key <> 5", ""}, {"key NOT IN (1, 2)", ""}, {"key BETWEEN 7 AND 3", " WHERE key BETWEEN 2 AND 8"}, + {"key > 5 AND key < 5", ""}, {"key > 5 OR key < 5", ""}, {"key < 5 AND key > 5", ""}, {"key < 5 OR key > 5", ""}, + {"key = 5 AND key IN (5)", ""}, {"key <> 5 AND key IN (5)", ""}, {"key <> 5 OR key NOT IN (5)", ""}, + {"key <> 5 AND key NOT IN (5)", ""}, {"key = 5 AND key NOT IN (5)", ""}, {"key = 5 OR key IN (5)", ""}, + {"key BETWEEN 3 AND 5 OR key = 6", ""}, {"key BETWEEN 3 AND 5 AND key = 4", ""}}; } - String createEmptyTable(String tableName) + @DataProvider(name = "queryOperatorTest") + public Object[][] queryOperatorTest() { - return safeCreateTable("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR(10))", tableName); + return new Object[][] { + {"id = 3", "bitmap"}, {"id = 3", "bloom"}, {"id = 3", "btree"}, {"id = 3", "minmax"}, + {"id <> 3", "bitmap"}, {"id <> 3", "bloom"}, {"id <> 3", "btree"}, {"id <> 3", "minmax"}, + {"id < 3", "bitmap"}, {"id < 3", "bloom"}, {"id < 3", "btree"}, {"id < 3", "minmax"}, + {"id > 3", "bitmap"}, {"id > 3", "bloom"}, {"id > 3", "btree"}, {"id > 3", "minmax"}, + {"id <= 3", "bitmap"}, {"id <= 3", "bloom"}, {"id <= 3", "btree"}, {"id <= 3", "minmax"}, + {"id >= 3", "bitmap"}, {"id >= 3", "bloom"}, {"id >= 3", "btree"}, {"id >= 3", "minmax"}, + {"id IN (1, 2)", "bitmap"}, {"id IN (1, 2)", "bloom"}, {"id IN (1, 2)", "btree"}, {"id IN (1, 2)", "minmax"}, + {"id IN (3, 3)", "bitmap"}, {"id IN (3, 3)", "bloom"}, {"id IN (3, 3)", "btree"}, {"id IN (3, 3)", "minmax"}, + {"id NOT IN (1, 2)", "bitmap"}, {"id NOT IN (1, 2)", "bloom"}, {"id NOT IN (1, 2)", "btree"}, {"id NOT IN (1, 2)", "minmax"}, + {"id BETWEEN 3 AND 5", "bitmap"}, {"id BETWEEN 3 AND 5", "bloom"}, {"id BETWEEN 3 AND 5", "btree"}, {"id BETWEEN 3 AND 5", "minmax"}, + {"id BETWEEN 3 AND 3", "bitmap"}, {"id BETWEEN 3 AND 3", "bloom"}, {"id BETWEEN 3 AND 3", "btree"}, {"id BETWEEN 3 AND 3", "minmax"}, + {"id > 3 AND id < 3", "bitmap"}, {"id > 3 AND id < 3", "bloom"}, {"id > 3 AND id < 3", "btree"}, {"id > 3 AND id < 3", "minmax"}, + {"id > 3 OR id < 3", "bitmap"}, {"id > 3 OR id < 3", "bloom"}, {"id > 3 OR id < 3", "btree"}, {"id > 3 OR id < 3", "minmax"}, + {"id < 3 AND id > 3", "bitmap"}, {"id < 3 AND id > 3", "bloom"}, {"id < 3 AND id > 3", "btree"}, {"id < 3 AND id > 3", "minmax"}, + {"id < 3 OR id > 3", "bitmap"}, {"id < 3 OR id > 3", "bloom"}, {"id < 3 OR id > 3", "btree"}, {"id < 3 OR id > 3", "minmax"}, + {"id <> 3 OR id NOT IN (3)", "bitmap"}, {"id <> 3 OR id NOT IN (3)", "bloom"}, {"id <> 3 OR id NOT IN (3)", "btree"}, {"id <> 3 OR id NOT IN (3)", "minmax"}, + {"id <> 3 AND id NOT IN (3)", "bitmap"}, {"id <> 3 AND id NOT IN (3)", "bloom"}, {"id <> 3 AND id NOT IN (3)", "btree"}, {"id <> 3 AND id NOT IN (3)", "minmax"}, + {"id = 3 OR id IN (3)", "bitmap"}, {"id = 3 OR id IN (3)", "bloom"}, {"id = 3 OR id IN (3)", "btree"}, {"id = 3 OR id IN (3)", "minmax"}, + {"id = 3 AND id NOT IN (3)", "bitmap"}, {"id = 3 AND id NOT IN (3)", "bloom"}, {"id = 3 AND id NOT IN (3)", "btree"}, {"id = 3 AND id NOT IN (3)", "minmax"}, + {"id = 3 AND id IN (3)", "bitmap"}, {"id = 3 AND id IN (3)", "bloom"}, {"id = 3 AND id IN (3)", "btree"}, {"id = 3 AND id IN (3)", "minmax"}, + {"id <> 3 AND id IN (3)", "bitmap"}, {"id <> 3 AND id IN (3)", "bloom"}, {"id <> 3 AND id IN (3)", "btree"}, {"id <> 3 AND id IN (3)", "minmax"}, + {"id BETWEEN 1 AND 3 OR id = 5", "bitmap"}, {"id BETWEEN 1 AND 3 OR id = 5", "bloom"}, {"id BETWEEN 1 AND 3 OR id = 5", "btree"}, {"id BETWEEN 1 AND 3 OR id = 5", "minmax"}, + {"id BETWEEN 1 AND 3 AND id = 2", "bitmap"}, {"id BETWEEN 1 AND 3 AND id = 2", "bloom"}, {"id BETWEEN 1 AND 3 AND id = 2", "btree"}, {"id BETWEEN 1 AND 3 AND id = 2", "minmax"}}; } - String createTable1(String tableName) + void createEmptyTable(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR(10))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR(10))"); + } + + void createTable1(String tableName) + { + assertQuerySucceeds("CREATE TABLE " + tableName + " (id INTEGER, name VARCHAR(10))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(1, 'test')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(2, '123'), (3, 'temp')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(3, 'data'), (9, 'ttt'), (5, 'num')"); - return tableName; } - String createTable2(String tableName) + void createTable2(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (key BIGINT, status VARCHAR(7), price DECIMAL(5,2), date DATE)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (key BIGINT, status VARCHAR(7), price DECIMAL(5,2), date DATE)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(0, 'SUCCESS', 101.12, DATE '2021-01-01')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(1, 'FAILURE', 101.12, DATE '2021-01-01')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(2, 'SUCCESS', 252.36, DATE '2021-01-01')"); @@ -142,123 +164,111 @@ public class TestIndexResources assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(6, 'FAILURE', 252.36, DATE '2021-01-03')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(7, 'SUCCESS', 101.12, DATE '2021-01-03')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(8, 'SUCCESS', 101.12, DATE '2021-01-03')"); - return tableName; } - String createTableNullData(String tableName) + void createTableNullData(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + " (col1 INTEGER, col2 BIGINT, col3 TINYINT)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (col1 INTEGER, col2 BIGINT, col3 TINYINT)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(111, NULL, NULL)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(NULL, BIGINT '222', NULL)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(NULL, NULL, TINYINT '33')"); - return tableName; } - String createBtreeTable1(String tableName) + void createBtreeTable1(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (key1 INT, key2 INT)" + - " WITH (partitioned_by = ARRAY['key2'])", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (key1 INT, key2 INT)" + " WITH (partitioned_by = ARRAY['key2'])"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(0, 10), (1, 11), (2, 12), (1, 11), (3, 13)," + " (1, 11), (2, 12), (4, 14), (3, 13), (5, 15), (6, 16)"); - return tableName; } - String createBtreeTableTransact1(String tableName) + void createBtreeTableTransact1(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (key1 INT, key2 INT)" + - " WITH (transactional = true, partitioned_by = ARRAY['key2'])", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (key1 INT, key2 INT)" + " WITH (transactional = true, partitioned_by = ARRAY['key2'])"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(0, 10), (1, 11), (2, 12), (1, 11), (3, 13)," + " (1, 11), (2, 12), (4, 14), (3, 13), (5, 15), (6, 16)"); - return tableName; } - String createBtreeTableMultiPart1(String tableName) + void createBtreeTableMultiPart1(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (key1 INT, key2 INT, key3 INT, key4 INT, key5 INT)" + - " WITH (partitioned_by = ARRAY['key3', 'key4', 'key5'])", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (key1 INT, key2 INT, key3 INT, key4 INT, key5 INT)" + + " WITH (partitioned_by = ARRAY['key3', 'key4', 'key5'])"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(1, 11, 111, 1111, 11111), (2, 22, 222, 2222, 22222), (3, 33, 333, 3333, 33333)," + " (2, 22, 222, 2222, 22222), (5, 55, 555, 5555, 55555), (1, 11, 111, 1111, 11111)," + " (6, 66, 666, 6666, 66666), (7, 77, 777, 7777, 77777), (2, 22, 222, 2222, 22222), (2, 22, 222, 2222, 22222)"); - return tableName; } - String[] createTableDataTypeWithQuery(String tableName, String dataType) + String createTableDataTypeWithQuery(String tableName, String dataType) throws InterruptedException { String query = "SELECT * FROM " + tableName + " WHERE data_col2="; switch (dataType.toLowerCase(Locale.ROOT)) { case "bigint": - tableName = createTableBigInt(tableName); + createTableBigInt(tableName); query += "5675354"; break; case "boolean": - tableName = createTableBoolean(tableName); + createTableBoolean(tableName); query = query.substring(0, query.length() - 1); break; case "char": - tableName = createTableChar(tableName); + createTableChar(tableName); query += "'z'"; break; case "date": - tableName = createTableDate(tableName); + createTableDate(tableName); query += "'2021-12-21'"; break; case "decimal": - tableName = createTableDecimal(tableName); + createTableDecimal(tableName); query += "DECIMAL '21.21'"; break; case "double": - tableName = createTableDouble(tableName); + createTableDouble(tableName); query += "DOUBLE '21.21'"; break; case "int": - tableName = createTableInt(tableName); + createTableInt(tableName); query += "606"; break; case "real": - tableName = createTableReal(tableName); + createTableReal(tableName); query += "21.21"; break; case "smallint": - tableName = createTableSmallInt(tableName); + createTableSmallInt(tableName); query += "32767"; break; case "string": - tableName = createTableString(tableName); + createTableString(tableName); query += ""; break; case "timestamp": - tableName = createTableTimestamp(tableName); + createTableTimestamp(tableName); query += ""; break; case "tinyint": - tableName = createTableTinyInt(tableName); + createTableTinyInt(tableName); query += "0"; break; case "varbinary": - tableName = createTableVarBinary(tableName); + createTableVarBinary(tableName); query += "AAA"; break; case "varchar": - tableName = createTableVarChar(tableName); + createTableVarChar(tableName); query += "'tester'"; break; default: throw new InterruptedException("Not supported data type."); } - String[] result = {tableName, query}; - return result; + return query; } - String createTableBigInt(String tableName) + void createTableBigInt(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 BIGINT, data_col2 BIGINT, data_col3 BIGINT)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 BIGINT, data_col2 BIGINT, data_col3 BIGINT)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(1, 1, 1)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(2, 2, 2)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(3, 3, 3)"); @@ -267,13 +277,11 @@ public class TestIndexResources assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(2818475983711351641, 2818475983711351641, 2818475983711351641)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(9214215753243532641, 9214215753243532641, 9214215753243532641)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(5675354, 5675354, 5675354)"); - return tableName; } - String createTableBoolean(String tableName) + void createTableBoolean(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 BOOLEAN, data_col2 BOOLEAN, data_col3 BOOLEAN)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 BOOLEAN, data_col2 BOOLEAN, data_col3 BOOLEAN)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '0', BOOLEAN '0', BOOLEAN '0')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '0', BOOLEAN '0', BOOLEAN '1')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '0', BOOLEAN '1', BOOLEAN '0')"); @@ -282,59 +290,49 @@ public class TestIndexResources assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '1', BOOLEAN '0', BOOLEAN '1')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '1', BOOLEAN '1', BOOLEAN '0')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(BOOLEAN '1', BOOLEAN '1', BOOLEAN '1')"); - return tableName; } - String createTableChar(String tableName) + void createTableChar(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 CHAR(1), data_col2 CHAR(1), data_col3 CHAR(1))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 CHAR(1), data_col2 CHAR(1), data_col3 CHAR(1))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('a', 'a', 'a')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('b', 'b', 'b')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('c', 'c', 'c')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('d', 'd', 'd')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('e', 'e', 'e')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('z', 'z', 'z')"); - return tableName; } - String createTableDate(String tableName) + void createTableDate(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 DATE, data_col2 DATE, data_col3 DATE)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 DATE, data_col2 DATE, data_col3 DATE)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'1997-01-01', DATE'1997-01-01', DATE'1997-01-01')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'2021-02-09', DATE'2021-02-09', DATE'2021-02-09')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'2020-03-08', DATE'2020-03-08', DATE'2020-03-08')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'2022-04-07', DATE'2022-04-07', DATE'2022-04-07')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'1899-05-06', DATE'1997-05-06', DATE'1899-05-06')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DATE'2021-12-21', DATE'2021-12-21', DATE'2021-12-21')"); - return tableName; } - String createTableDecimal(String tableName) + void createTableDecimal(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 DECIMAL(4,2), data_col2 DECIMAL(4,2), data_col3 DECIMAL(4,2))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 DECIMAL(4,2), data_col2 DECIMAL(4,2), data_col3 DECIMAL(4,2))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DECIMAL '11.11', DECIMAL '11.11', DECIMAL '11.11')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DECIMAL '21.21', DECIMAL '21.21', DECIMAL '21.21')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DECIMAL '12.34', DECIMAL '43.21', DECIMAL '88.34')"); - return tableName; } - String createTableDouble(String tableName) + void createTableDouble(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 DOUBLE(4,2), data_col2 DOUBLE(4,2), data_col3 DOUBLE(4,2))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 DOUBLE(4,2), data_col2 DOUBLE(4,2), data_col3 DOUBLE(4,2))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DOUBLE '11.11', DOUBLE '11.11', DOUBLE '11.11')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DOUBLE '21.21', DOUBLE '21.21', DOUBLE '21.21')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(DOUBLE '12.34', DOUBLE '43.21', DOUBLE '88.34')"); - return tableName; } - String createTableInt(String tableName) + void createTableInt(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 INT, data_col2 INT, data_col3 INT)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 INT, data_col2 INT, data_col3 INT)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(1, 1, 1)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(2, 2, 2)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(3, 3, 3)"); @@ -342,76 +340,62 @@ public class TestIndexResources assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(5, 5, 5)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(606, 606, 606)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(123, 123, 123)"); - return tableName; } - String createTableReal(String tableName) + void createTableReal(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 REAL(4,2), data_col2 REAL(4,2), data_col3 REAL(4,2))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 REAL(4,2), data_col2 REAL(4,2), data_col3 REAL(4,2))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(REAL '11.11', REAL '11.11', REAL '11.11')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(REAL '21.21', REAL '21.21', REAL '21.21')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(REAL '12.34', REAL '43.21', REAL '88.34')"); - return tableName; } - String createTableSmallInt(String tableName) + void createTableSmallInt(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 SMALLINT, data_col2 SMALLINT, data_col3 SMALLINT)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 SMALLINT, data_col2 SMALLINT, data_col3 SMALLINT)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '1', SMALLINT '1', SMALLINT '1')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '2', SMALLINT '2', SMALLINT '2')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '3', SMALLINT '3', SMALLINT '3')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '4', SMALLINT '4', SMALLINT '4')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '5', SMALLINT '5', SMALLINT '5')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(SMALLINT '-32768', SMALLINT '32767', SMALLINT '0')"); - return tableName; } - String createTableString(String tableName) + void createTableString(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 STRING, data_col2 STRING, data_col3 STRING)", tableName); - return tableName; + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 STRING, data_col2 STRING, data_col3 STRING)"); } - String createTableTimestamp(String tableName) + void createTableTimestamp(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 TIMESTAMP, data_col2 TIMESTAMP, data_col3 TIMESTAMP)", tableName); - return tableName; + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 TIMESTAMP, data_col2 TIMESTAMP, data_col3 TIMESTAMP)"); } - String createTableTinyInt(String tableName) + void createTableTinyInt(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 TINYINT, data_col2 TINYINT, data_col3 TINYINT)", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 TINYINT, data_col2 TINYINT, data_col3 TINYINT)"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '1', TINYINT '1', TINYINT '1')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '2', TINYINT '2', TINYINT '2')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '3', TINYINT '3', TINYINT '3')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '4', TINYINT '4', TINYINT '4')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '5', TINYINT '5', TINYINT '5')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES(TINYINT '-127', TINYINT '0', TINYINT '127')"); - return tableName; } - String createTableVarBinary(String tableName) + void createTableVarBinary(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 VARBINARY(5), data_col2 VARBINARY(5), data_col3 VARBINARY(5))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 VARBINARY(5), data_col2 VARBINARY(5), data_col3 VARBINARY(5))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('11111', '11111', '11111')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('22222', '22222', '22222')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('33333', '33333', '33333')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('44444', '44444', '44444')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('55555', '55555', '55555')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('12351', '12341', '15311')"); - return tableName; } - String createTableVarChar(String tableName) + void createTableVarChar(String tableName) { - tableName = safeCreateTable("CREATE TABLE " + tableName + - " (data_col1 VARCHAR(10), data_col2 VARCHAR(10), data_col3 VARCHAR(10))", tableName); + assertQuerySucceeds("CREATE TABLE " + tableName + " (data_col1 VARCHAR(10), data_col2 VARCHAR(10), data_col3 VARCHAR(10))"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('aaa', 'aaa', 'aaa')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('bbb', 'bbb', 'bbb')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('ccc', 'ccc', 'ccc')"); @@ -420,82 +404,42 @@ public class TestIndexResources assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('tester', 'tester', 'tester')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('blah', 'blah', 'blah')"); assertQuerySucceeds("INSERT INTO " + tableName + " VALUES('nothing', 'nothing', 'nothing')"); - return tableName; } + static AtomicInteger count = new AtomicInteger(0); + String getNewTableName() { - MaterializedResult result = computeActual("SHOW TABLES FROM hive.test"); - String tableName = "table" + ((Thread.currentThread().getStackTrace())[2]).getMethodName().toLowerCase(Locale.ROOT); - int newIndex = 1; - ArrayList results = new ArrayList<>(); - for (MaterializedRow item : result.getMaterializedRows()) { - results.add(item.getField(0).toString()); - } - int size = result.getRowCount(); - for (int i = 0; i < size; i++) { - if (results.contains(tableName + newIndex)) { - newIndex++; - } - else { - break; - } - } - return "hive.test." + tableName + newIndex; + return "hive.test.t" + Integer.valueOf(count.getAndIncrement()).toString(); } String getNewIndexName() { - MaterializedResult result = computeActual("SHOW INDEX"); - String indexName = "index" + ((Thread.currentThread().getStackTrace())[2]).getMethodName().toLowerCase(Locale.ROOT); - int newIndex = 1; - ArrayList results = new ArrayList<>(); - for (MaterializedRow item : result.getMaterializedRows()) { - results.add(item.getField(0).toString()); - } - int size = result.getRowCount(); - for (int i = 0; i < size; i++) { - if (results.contains(indexName + newIndex)) { - newIndex++; - } - else { - break; - } - } - return indexName + newIndex; + return "idx" + Integer.valueOf(count.getAndIncrement()).toString(); } // Get the split count and MaterializedResult in one pair to return. Pair getSplitAndMaterializedResult(String testerQuery) { - String testerQueryID = ""; - int testerSplits = 0; - // Select the entry with specifics MaterializedResult queryResult = computeActual(testerQuery); - // Get queries executed and query ID - MaterializedResult systemQueriesResult = computeActual("SELECT * FROM system.runtime.queries ORDER BY query_id DESC"); - for (MaterializedRow item : systemQueriesResult.getMaterializedRows()) { - // Find query to match and get the query_id of that query for getting sum of splits later - if (testerQueryID.equals("") && item.getField(4).toString().equals(testerQuery)) { - testerQueryID = item.getField(0).toString(); - } - } + // Get queries executed and query ID to find the task with sum of splits + String splits = "select sum(splits) from system.runtime.tasks where query_id in " + + "(select query_id from system.runtime.queries " + + "where query='" + testerQuery + "' order by created desc limit 1)"; - assertNotEquals(testerQueryID, ""); + MaterializedResult rows = computeActual(splits); - // Select entries for tasks done using the previously retrieved query ID amd get sum of splits - MaterializedResult splitsResult = computeActual("SELECT splits " + - "FROM system.runtime.tasks AS t1, system.runtime.queries AS t2 " + - "WHERE t1.query_id = t2.query_id " + - "AND t2.query_id = '" + testerQueryID + "'AND t2.query = '" + testerQuery + "'"); - for (MaterializedRow item : splitsResult.getMaterializedRows()) { - // Sum up the splits - testerSplits += Integer.parseInt(item.getField(0).toString()); - } + assertEquals(rows.getRowCount(), 1); - return new Pair<>(testerSplits, queryResult); + MaterializedRow materializedRow = rows.getMaterializedRows().get(0); + int fieldCount = materializedRow.getFieldCount(); + assertEquals(fieldCount, 1, + "Expected only one column, but got '%d', fiedlCount: " + fieldCount); + Object value = materializedRow.getField(0); + + return new Pair<>((int) (long) value, queryResult); } // Compare the two results are consistent. @@ -513,4 +457,22 @@ public class TestIndexResources Collections.sort(data2); return data1.equals(data2); } + + long getInputRowsOfLastQueryExecution(String sql) + { + String inputRowsSql = "select sum(raw_input_rows) from system.runtime.tasks where query_id in " + + "(select query_id from system.runtime.queries where query='" + sql + "' order by created desc limit 1)"; + + MaterializedResult rows = computeActual(inputRowsSql); + + assertEquals(rows.getRowCount(), 1); + + MaterializedRow materializedRow = rows.getMaterializedRows().get(0); + int fieldCount = materializedRow.getFieldCount(); + assertEquals(fieldCount, 1, + "Expected only one column, but got '%d', fiedlCount: " + fieldCount); + Object value = materializedRow.getField(0); + + return (long) value; + } }