diff --git a/hetu-docs/en/indexer/bitmap.md b/hetu-docs/en/indexer/bitmap.md index 18133794c..862ae3b39 100644 --- a/hetu-docs/en/indexer/bitmap.md +++ b/hetu-docs/en/indexer/bitmap.md @@ -48,7 +48,9 @@ data is being filtered on both columns and they both have low cardinality. IN IN set ## Supported column types - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**Note:** Index cannot be created on unsupported data types. ## Examples @@ -68,7 +70,7 @@ select * from hive.hindex.users where age>20 select * from hive.hindex.users where age<25 select * from hive.hindex.users where age>=21 select * from hive.hindex.users where age<=24 -select * from hive.hindex.users where age between (20, 25) +select * from hive.hindex.users where age between 20 AND 25 select * from hive.hindex.users where age in (22, 23) ``` diff --git a/hetu-docs/en/indexer/bloom.md b/hetu-docs/en/indexer/bloom.md index 97e73a92b..e635c85c9 100644 --- a/hetu-docs/en/indexer/bloom.md +++ b/hetu-docs/en/indexer/bloom.md @@ -31,7 +31,9 @@ data is being filtered on the column and `phone` column has a high cardinality. = Equality ## Supported column types - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**Note:** Index cannot be created on unsupported data types. ## Configurations diff --git a/hetu-docs/en/indexer/btree.md b/hetu-docs/en/indexer/btree.md index 1d7829476..63955c2cb 100644 --- a/hetu-docs/en/indexer/btree.md +++ b/hetu-docs/en/indexer/btree.md @@ -44,7 +44,9 @@ When selecting between BTreeIndex and BloomIndex, the following should be consid IN IN set ## Supported column types - "integer", "smallint", "bigint", "tinyint", "varchar", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "double", "real", "date", "decimal" + +**Note:** Index cannot be created on unsupported data types. ## Examples @@ -65,7 +67,7 @@ select * from hive.hindex.orders where orderid>12345 select * from hive.hindex.orders where orderid<12345 select * from hive.hindex.orders where orderid>=12345 select * from hive.hindex.orders where orderid<=12345 -select * from hive.hindex.orders where orderid between (10000, 20000) +select * from hive.hindex.orders where orderid between 10000 AND 20000 select * from hive.hindex.orders where orderid in (12345, 7890) ``` diff --git a/hetu-docs/en/indexer/hindex-statements.md b/hetu-docs/en/indexer/hindex-statements.md index c83f4dcb0..1233cbf86 100644 --- a/hetu-docs/en/indexer/hindex-statements.md +++ b/hetu-docs/en/indexer/hindex-statements.md @@ -11,7 +11,7 @@ To create an index you can run sql statements of the form: CREATE INDEX [ IF NOT EXISTS ] index_name USING [ BITMAP | BLOOM | BTREE | MINMAX ] ON tbl_name (col_name) -WITH ( "level" = ['STRIPE', 'PARTITION'], "bloom.fpp" = '0.001', [, …] ) +WITH ( 'level' = ['STRIPE', 'PARTITION'], "bloom.fpp" = '0.001', [, …] ) WHERE predicate; ``` diff --git a/hetu-docs/en/indexer/minmax.md b/hetu-docs/en/indexer/minmax.md index 040ff18d6..690598c5a 100644 --- a/hetu-docs/en/indexer/minmax.md +++ b/hetu-docs/en/indexer/minmax.md @@ -31,7 +31,9 @@ the data is sorted on `age` column. <= Less than or equal ## Supported column types - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**Note:** Index cannot be created on unsupported data types. ## Examples diff --git a/hetu-docs/en/indexer/overview.md b/hetu-docs/en/indexer/overview.md index ea4459a3a..21481b710 100644 --- a/hetu-docs/en/indexer/overview.md +++ b/hetu-docs/en/indexer/overview.md @@ -62,6 +62,11 @@ For a complete list of configuration properties see [Properties](../admin/proper ### 1. Configure indexer settings +Heuristic indexer uses Hetu Metastore to manage its metadata. Hetu Metastore is a shared metadata management +utility used by multiple openLooKeng features. For more information about how to configure it, +please check [Hetu Metastore](../admin/meta-store.md). +Note: Indexer won't work if Hetu Metastore is not properly configured! + In `etc/config.properties`, add these lines: hetu.heuristicindex.filter.enabled=true @@ -72,7 +77,7 @@ In `etc/config.properties`, add these lines: Path whitelist：`["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]` **Note**： -- `LOCAL` filesystem type is NOT supported. +- `LOCAL` filesystem type is NOT supported. Local filesystem should NOT be used in Hetu metastore as well. - `HDFS` filesystem type should be used in production in order for the index to be accessible by all nodes in the cluster. - All nodes should be configured to use the same filesystem profile. - Heuristic Index can be disabled while the engine is running by setting: `set session heuristicindex_filter_enabled=false;` @@ -127,8 +132,6 @@ Subsequent queries will utilize the index to reduce the amount of data read | hetu.heuristicindex.indexstore.filesystem.profile | local-config-default| No | This property defines the filesystem profile used to read and write index| | hetu.heuristicindex.filter.cache.preload-indices | | No | Preload the specified indices (comma-separated) when the server starts. Put `ALL` to load all indices| -Heuristic indexer now uses Hetu Metastore to manage its metadata. Please check [Hetu Metastore](../admin/meta-store.md) for more information. - ## Index Statements See [Heuristic Index Statements](./hindex-statements.md). @@ -144,8 +147,9 @@ See [Heuristic Index Statements](./hindex-statements.md). | [MinMax](./minmax.md) | Split
Stripe | Column which table is sorted on | `=` `>` `>=` `<` `<=` | `create index idx using bloom on hive.hindex.users (age);`
(assuming users is sorted by age)
`select name from hive.hindex.users where age>25` | | [Bitmap](./bitmap.md) | Row | Low cardinality
(such as Gender column) | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` | `create index idx using bitmap on hive.hindex.users (gender);`
`select name from hive.hindex.users where gender='female'` | -**Note:** unsupported operators will still function correctly but will not benefit from the index. - +**Notes:** +· Unsupported operators will still function correctly but will not benefit from the index. +· Additional data types are not supported if not listed by the individual index types. ## Choosing Index Type @@ -159,6 +163,10 @@ because IDs are unique. Whereas `employeeType` will have low cardinality because there are likely only a few different types (e.g. Manager, Developer, Tester). +Disk usage and creation speed might also be the factors to be taken into consideration when choosing the best index. +BTree index uses significantly more space and more time when creating, compared to other split filtering indices, such +as Bloom and Minmax. + ![index-decision](../images/index-decision.png) Example queries: @@ -197,3 +205,11 @@ See [Adding your own Index Type](./new-index.md). ## Access Control See [Built-in System Access Control](../security/built-in-system-access-control.md). + +## Troubleshooting + +### When index creation gets stuck and lasts too long (Usually happens on low-performance machines, e.g. dev laptop, k8s env with few cores) + +The machine might not have enough threads to complete the task. Try reducing the task concurrency by setting +the session property: `set session task_concurrency=X;`, where X is recommended to be less than or equal to the CPU +cores/threads. For example, if the CPU has 8 threads, 8 can be set. \ No newline at end of file diff --git a/hetu-docs/zh/indexer/bitmap.md b/hetu-docs/zh/indexer/bitmap.md index 5414fe458..113393540 100644 --- a/hetu-docs/zh/indexer/bitmap.md +++ b/hetu-docs/zh/indexer/bitmap.md @@ -1,15 +1,15 @@ # BitmapIndex（位图索引） -BitmapIndex使用位图来进行早期行过滤，这可以帮助减少CPU和内存使用量。 +BitmapIndex使用Bitmap来进行早期行过滤，这可以帮助减少CPU和内存使用量。 这在高并发queries中是有益的。 -BitmapIndex对于低基数（即独特数据不多的）的列效果很好， -因为index的大小随着独特数量的增加而增加。 +BitmapIndex对于低基数（low cardinality, 即不同数据值的个数不多）的列效果很好， +因为index的大小随着不同值数量的增加而增加。 例如，`gender`之类的列将具有较小的尺寸。 -而像`id`这样的列将具有一个极高的大小（不推荐）。 +而像`id`这样的列有很多不同的值，因此不推荐使用位图索引。 -Bitmap是为每个独特列值而构造一个位图，可以用来记录并且在其中找到该值的行号。 +Bitmap是为每个不同的值构造一个位图，并记录包含该值的行号。 然后，B+Tree会被用来存储值与其位图之间的映射。 通过使用B+Tree，BitmapIndex可以支持使用运算符之类的范围query，例如 大于（`>`），小于（`<`），`BETWEEN`等。 @@ -27,8 +27,7 @@ BitmapIndex用于过滤从ORC文件中读取的数据，且仅供worker节点使 ## 选择适用的列 -以高并发率运行的queries，并且在具有低基数（独特值不多的）条件的列上具有过滤predicates -可以从BitmapIndex中得到好的效果。 +以高并发数运行的queries，并且在低基数的列上过滤predicates可以从BitmapIndex中得到好的效果。 例如，类似`SELECT * FROM Employees WHERE gender='M' AND type='FULLTIME' AND salary>10000`的query 可以在`gender`和`type`列上用BitmapIndex并且得到好的效果，因为数据在两列上都被过滤，并且两者的基数都很低。 @@ -44,7 +43,9 @@ BitmapIndex用于过滤从ORC文件中读取的数据，且仅供worker节点使 IN IN set ## 支持的列类型 - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**注意:** 不支持采用其它数据类型来创建index。 ## 用例 @@ -64,7 +65,7 @@ select * from hive.hindex.users where age>20 select * from hive.hindex.users where age<25 select * from hive.hindex.users where age>=21 select * from hive.hindex.users where age<=24 -select * from hive.hindex.users where age between (20, 25) +select * from hive.hindex.users where age between 20 AND 25 select * from hive.hindex.users where age in (22, 23) ``` @@ -74,8 +75,8 @@ select * from hive.hindex.users where age in (22, 23) 2. 数据作为有序列表插入，数据顺序是根据在Stripe中的出现顺序。 对于以下示例，`/hive/database.db/animals/000.orc stripe 1`的数据将如下插入： `["Ant", "Crab", "Bat", "Whale", "Ant", "Monkey"]` - 诸如上次修改时间之类的其他信息将作为元数据存储，以确保不使用陈旧索引。 -3. 数据插入完成后，将为每个独特值创建一个Bitmap。这是一种跟踪值存在的行的紧凑方式。（请参见表） + 诸如上次修改时间之类的其他信息将作为元数据存储并在调用时检查，以确保不使用陈旧索引。 +3. 数据插入完成后，将为每个独特值创建一个Bitmap。这是一种紧凑的跟踪值是否存在的方式。（请参见表） 4. 一旦为独特值创建了Bitmap。该值和相应的Bitmap被压缩并存储在B+Tree中，以允许在`O(log(n))`之内的运行速度来快速查找。 ![bitmap_animal_table](../images/bitmap_animal_table.png) diff --git a/hetu-docs/zh/indexer/bloom.md b/hetu-docs/zh/indexer/bloom.md index 59f43a4e7..9f94bef78 100644 --- a/hetu-docs/zh/indexer/bloom.md +++ b/hetu-docs/zh/indexer/bloom.md @@ -3,7 +3,7 @@ BloomIndex使用Bloom Filters(布隆过滤器)来在计划期间和读取数据时进行过滤。 -BloomIndex对于具有高基数的列以及索引大小很小的列都适用。 +BloomIndex对于具有高基数（有许多不同值）的列以及索引大小很小的列都适用。 布隆过滤器是使用列值构造的。然后在查找过程中，布隆过滤器会告诉我们布隆过滤器中是否有给定值。 @@ -28,7 +28,9 @@ BloomIndex仅支持相等表达式，例如`name='monkey'`。 = Equality ## 支持的列类型 - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**注意:** 不支持采用其它数据类型来创建index。 ## 配置参数 diff --git a/hetu-docs/zh/indexer/btree.md b/hetu-docs/zh/indexer/btree.md index b92f48831..dc369bb00 100644 --- a/hetu-docs/zh/indexer/btree.md +++ b/hetu-docs/zh/indexer/btree.md @@ -44,7 +44,9 @@ BTreeIndex用于调度时的分片(Split)过滤，被coordinator节点使用。 IN IN set ## 支持的列类型 - "integer", "smallint", "bigint", "tinyint", "varchar", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "double", "real", "date", "decimal" + +**注意:** 不支持采用其它数据类型来创建index。 ## 用例 @@ -65,7 +67,7 @@ select * from hive.hindex.orders where orderid>12345 select * from hive.hindex.orders where orderid<12345 select * from hive.hindex.orders where orderid>=12345 select * from hive.hindex.orders where orderid<=12345 -select * from hive.hindex.orders where orderid between (10000, 20000) +select * from hive.hindex.orders where orderid between 10000 AND 20000 select * from hive.hindex.orders where orderid in (12345, 7890) ``` diff --git a/hetu-docs/zh/indexer/hindex-statements.md b/hetu-docs/zh/indexer/hindex-statements.md index 4aba9c368..457529cfc 100644 --- a/hetu-docs/zh/indexer/hindex-statements.md +++ b/hetu-docs/zh/indexer/hindex-statements.md @@ -10,7 +10,7 @@ CREATE INDEX [ IF NOT EXISTS ] index_name USING [ BITMAP | BLOOM | BTREE | MINMAX ] ON tbl_name (col_name) -WITH ( "level" = ['STRIPE', 'PARTITION'], "bloom.fpp" = '0.001', [, …] ) +WITH ( 'level' = ['STRIPE', 'PARTITION'], "bloom.fpp" = '0.001', [, …] ) WHERE predicate; ``` diff --git a/hetu-docs/zh/indexer/minmax.md b/hetu-docs/zh/indexer/minmax.md index c05ec8070..b64a9b0fb 100644 --- a/hetu-docs/zh/indexer/minmax.md +++ b/hetu-docs/zh/indexer/minmax.md @@ -26,7 +26,9 @@ MinMaxIndex用于调度时的分片过滤，被coordinator节点使用。 <= Less than or equal ## 支持的列类型 - "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date" + "integer", "smallint", "bigint", "tinyint", "varchar", "char", "boolean", "double", "real", "date", "decimal" + +**注意:** 不支持采用其它数据类型来创建index。 ## 用例 diff --git a/hetu-docs/zh/indexer/overview.md b/hetu-docs/zh/indexer/overview.md index 4f7e555d7..260063757 100644 --- a/hetu-docs/zh/indexer/overview.md +++ b/hetu-docs/zh/indexer/overview.md @@ -17,8 +17,8 @@ **注意：当前，启发式索引仅支持ORC存储格式的Hive数据源。** 1. BloomIndex，MinMaxIndex和BtreeIndex可以在Coordinator上使用，以在调度期间过滤Splits -2. 在读取ORC文件时，为了过滤Splits，可以在workers上使用MinMaxIndex或者BloomIndex -3. 在读取ORC文件时，BitmapIndex可以在workers上用于过滤数据行 +2. 在读取ORC文件时，可以在worker上使用MinMaxIndex或者BloomIndex过滤stripe +3. 在读取ORC文件时，可以在worker上使用BitmapIndex过滤数据行 ### 1.查询过程中过滤预定分片 @@ -52,6 +52,10 @@ ### 1. 配置索引 +索引功能现使用Hetu Metastore管理元数据。Hetu Metastore是一个被整个OLK集群多个功能共用的元数据管理器， +请参阅 [Hetu Metastore](../admin/meta-store.md) 获取关于如何配置的更多信息。 +注意：必须先配置好Hetu Metastore，启发式索引才能正常运行！ + 在 `etc/config.properties` 中加入这些行： hetu.heuristicindex.filter.enabled=true @@ -64,7 +68,7 @@ 避免选择根目录；路径不能包含../；如果配置了node.data_dir,那么当前工作目录为node.data_dir的父目录；如果没有配置，那么当前工作目录为openlookeng server的目录 **注意**： -- `LOCAL` 本地文件系统是*不*被支持的。 +- `LOCAL` 本地文件系统是*不*被支持的。同时在Hetu Metastore中也不应该选择本地文件系统。 - `HDFS` 应用于生产环境来在集群中共享数据。 - 所有节点必须有相同的文件系统配置。 - 在服务器运行中可以通过`set session heuristicindex_filter_enabled=false;`关闭启发式索引。 @@ -112,8 +116,6 @@ | hetu.heuristicindex.indexstore.filesystem.profile | local-config-default| 否 | 用于存储索引文件的文件系统属性描述文件名称| | hetu.heuristicindex.filter.cache.preload-indices | | 否 | 在服务器启动时预加载指定名称的索引(用逗号分隔), 当值为`ALL`时将预载入全部索引| -索引功能现使用Hetu Metastore管理元数据。请参阅 [Hetu Metastore](../admin/meta-store.md) 获取关于如何配置的更多信息。 - ## 索引语句 参见 [Heuristic Index Statements](./hindex-statements.md). @@ -129,8 +131,9 @@ | [MinMax](./minmax.md) | Split
Stripe | 列数据被排序 | `=` `>` `>=` `<` `<=` | `create index idx using bloom on hive.hindex.users (age);`
(假设数据根据年龄已排序)
`select name from hive.hindex.users where age>25` | | [Bitmap](./bitmap.md) | Row | 少量不同数据值
(如性别) | `=` `>` `>=` `<` `<=` `IN` `BETWEEN` | `create index idx using bitmap on hive.hindex.users (gender);`
`select name from hive.hindex.users where gender='female'` | -**注意:** 包含不支持的运算符的语句依然会正常运行，但是不会从启发式索引中获得性能提升。 - +**注意:** +· 包含不支持的运算符的语句依然会正常运行，但是不会从启发式索引中获得性能提升。 +· 对应每个index type，不支持其它除了列出以外的数据类。 ## 选择索引类型 @@ -139,6 +142,9 @@ Cardinality 是指数据集中值域的大小。例如，`ID`列通常有很大的cardinality， 而`employeeType`列通常cardinality很小(如 Manager, Developer, Tester)。 +在有些使用场景中，创建索引的速度和索引占用的磁盘空间也可能是需要考虑的因素。与其他主要用于分片过滤的索引，如 +Bloom和Minmax索引相比，创建BTree索引需要的时间和索引使用的空间明显更大。 + ![index-decision](../images/index-decision.png) 用例: @@ -171,3 +177,10 @@ Cardinality 是指数据集中值域的大小。例如，`ID`列通常有很大 ## 权限控制 参见 [Built-in System Access Control](../security/built-in-system-access-control.md). + +## 常见问题 + +### 如果索引创建卡住，并且长时间都没有变化（常发生在性能较差的环境中，例如开发人员的笔记本，核分配不多的k8s环境） + +当前机器可能没有足够的线程资源来完成任务。请尝试通过设置会话属性降低任务并发度：`set session task_concurrency=X;`， +其中X建议小于等于机器CPU的线程数。例如，如果处理器可用线程数为8，则X可设置为8。 \ No newline at end of file diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HindexQueryRunner.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HindexQueryRunner.java index 62e539bf1..14b828e40 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HindexQueryRunner.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/HindexQueryRunner.java @@ -60,7 +60,7 @@ public final class HindexQueryRunner configs.put("hetu.heuristicindex.filter.cache.max-memory", "1GB"); configs.put("hetu.heuristicindex.filter.cache.loading-delay", "100ms"); configs.put("hetu.heuristicindex.indexstore.uri", folder.getRoot().getAbsolutePath()); - configs.put("hetu.heuristicindex.indexstore.filesystem.profile", "default"); + configs.put("hetu.heuristicindex.indexstore.filesystem.profile", "__test__hdfs__"); File subFolder = folder.newFolder(); Map metastoreConfig = new HashMap<>(); diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java index 3277e5298..3be912967 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestHindexFailure.java @@ -150,13 +150,13 @@ public class TestHindexFailure throws SemanticException { String tableName = getNewTableName(); - String wrongTableName = "hive." + tableName.substring(10); + String wrongTableName = "hive.nonexistingschema." + tableName.substring(10); createEmptyTable(tableName); String indexName = getNewIndexName(); assertContains("CREATE INDEX " + indexName + " USING " + indexType + " ON " + wrongTableName + " (" + queryVariable + ")", - "Schema hive does not exist"); + "Table '" + wrongTableName + "' is invalid"); } // Tests the case where index is trying to be created with a wrong schema name (Error case). @@ -171,7 +171,7 @@ public class TestHindexFailure String indexName = getNewIndexName(); assertContains("CREATE INDEX " + indexName + " USING " + indexType + " ON " + wrongTableName + " (" + queryVariable + ")", - "Schema nonexisting does not exist"); + "Table '" + wrongTableName + "' is invalid"); } // Tests the case where index is trying to be created without table name (Error case). @@ -184,7 +184,7 @@ public class TestHindexFailure String indexName = getNewIndexName(); assertContains("CREATE INDEX " + indexName + " USING " + indexType + " ON " + wrongTableName + " (" + queryVariable + ")", - "Schema hive does not exist"); + "Table 'hive." + wrongTableName + "' is invalid"); } // Tests the case where index is trying to be created with a wrong table name (Error case). @@ -199,7 +199,7 @@ public class TestHindexFailure String indexName = getNewIndexName(); assertContains("CREATE INDEX " + indexName + " USING " + indexType + " ON " + wrongTableName + " (" + queryVariable + ")", - "Table " + wrongTableName + " does not exist"); + "Table '" + wrongTableName + "' is invalid"); } // Tests the case where index is trying to be created without column name (Error case). diff --git a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java index a57b14b39..b4a760cd8 100644 --- a/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java +++ b/hetu-heuristic-index/src/test/java/io/hetu/core/heuristicindex/TestIndexResources.java @@ -61,7 +61,6 @@ public class TestIndexResources } catch (AssertionError e) { assertTrue(e.getCause().toString().contains(contained)); - return; } } @@ -225,7 +224,7 @@ public class TestIndexResources {"bitmap", "char"}, {"bloom", "char"}, {"minmax", "char"}, {"bitmap", "date"}, {"bloom", "date"}, {"minmax", "date"}, {"btree", "date"}, {"bitmap", "decimal"}, {"bloom", "decimal"}, {"minmax", "decimal"}, {"btree", "decimal"}, - {"bitmap", "double"}, {"bloom", "double"}, {"minmax", "double"}, + {"bitmap", "double"}, {"bloom", "double"}, {"minmax", "double"}, {"btree", "double"}, {"bitmap", "int"}, {"bloom", "int"}, {"minmax", "int"}, {"btree", "int"}, {"bitmap", "real"}, {"bloom", "real"}, {"minmax", "real"}, {"btree", "real"}, {"bitmap", "smallint"}, {"bloom", "smallint"}, {"minmax", "smallint"}, {"btree", "smallint"}, diff --git a/presto-main/src/main/java/io/prestosql/filesystem/FileSystemClientManager.java b/presto-main/src/main/java/io/prestosql/filesystem/FileSystemClientManager.java index a13cd66fb..06bd2e4c5 100644 --- a/presto-main/src/main/java/io/prestosql/filesystem/FileSystemClientManager.java +++ b/presto-main/src/main/java/io/prestosql/filesystem/FileSystemClientManager.java @@ -37,17 +37,24 @@ public class FileSystemClientManager private static final String FS_CLIENT_TYPE = "fs.client.type"; private static final String FS_CONFIG_DIR = "etc/filesystem/"; private static final String DEFAULT_CONFIG_NAME = "default"; + private static final String TEST_HDFS = "__test__hdfs__"; private static final Map fileSystemFactories = new ConcurrentHashMap<>(); private static final Map availableFileSystemConfigs = new ConcurrentHashMap<>(); - private Properties defaultProfile; - public FileSystemClientManager() { // Default filesystem to be a local filesystem client - defaultProfile = new Properties(); + Properties defaultProfile = new Properties(); defaultProfile.setProperty(FS_CLIENT_TYPE, "local"); + + Properties testHdfsProfile = new Properties(); + testHdfsProfile.setProperty("fs.client.type", "hdfs"); + testHdfsProfile.setProperty("hdfs.config.resources", ""); + testHdfsProfile.setProperty("hdfs.authentication.type", "NONE"); + + availableFileSystemConfigs.put(DEFAULT_CONFIG_NAME, defaultProfile); + availableFileSystemConfigs.put(TEST_HDFS, testHdfsProfile); } public void addFileSystemClientFactories(HetuFileSystemClientFactory factory) @@ -92,16 +99,9 @@ public class FileSystemClientManager checkState(fileSystemFactories.containsKey(configType), "Factory for file system type %s not found", configType); - // If a file defines default properties, overwrite default then continue to next file - if (DEFAULT_CONFIG_NAME.equals(configName)) { - defaultProfile = properties; - LOG.info("default profile has been overridden by default.properties"); - } - // otherwise register config file into the map - else { - availableFileSystemConfigs.put(configName, properties); - LOG.info(String.format("Loaded '%s' file system config '%s'", configType, configName)); - } + // register profile into the map. will overwrite existing profile + availableFileSystemConfigs.put(configName, properties); + LOG.info(String.format("Loaded '%s' file system config '%s'", configType, configName)); } LOG.info(String.format("-- Loaded file system profiles: %s --", @@ -132,10 +132,10 @@ public class FileSystemClientManager public HetuFileSystemClient getFileSystemClient(String name, Path root) throws IOException { - if (!DEFAULT_CONFIG_NAME.equals(name) && !availableFileSystemConfigs.containsKey(name)) { + if (!availableFileSystemConfigs.containsKey(name)) { throw new IllegalArgumentException(String.format("Profile %s is not available. Please check the name provided.", name)); } - Properties fsConfig = DEFAULT_CONFIG_NAME.equals(name) ? defaultProfile : availableFileSystemConfigs.get(name); + Properties fsConfig = availableFileSystemConfigs.get(name); String type = fsConfig.getProperty(FS_CLIENT_TYPE); HetuFileSystemClientFactory factory = fileSystemFactories.get(type); try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factory.getClass().getClassLoader())) { @@ -193,10 +193,10 @@ public class FileSystemClientManager */ public boolean isFileSystemLocal(String name) { - if (!DEFAULT_CONFIG_NAME.equals(name) && !availableFileSystemConfigs.containsKey(name)) { + if (!availableFileSystemConfigs.containsKey(name)) { throw new IllegalArgumentException(String.format("Profile %s is not available. Please check the name provided.", name)); } - Properties fsConfig = DEFAULT_CONFIG_NAME.equals(name) ? defaultProfile : availableFileSystemConfigs.get(name); + Properties fsConfig = availableFileSystemConfigs.get(name); return fsConfig.getProperty(FS_CLIENT_TYPE).equals("local"); } } diff --git a/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java b/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java index ea2bf9955..d95187895 100644 --- a/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java +++ b/presto-main/src/main/java/io/prestosql/heuristicindex/HeuristicIndexerManager.java @@ -14,7 +14,6 @@ */ package io.prestosql.heuristicindex; -import com.google.common.annotations.VisibleForTesting; import com.google.inject.Inject; import io.airlift.log.Logger; import io.prestosql.execution.QueryInfo; @@ -35,8 +34,6 @@ import io.prestosql.testing.NoOpIndexClient; import io.prestosql.testing.NoOpIndexWriter; import java.io.IOException; -import java.nio.file.FileSystemException; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -75,17 +72,6 @@ public class HeuristicIndexerManager HeuristicIndexerManager.factory = indexFactory; } - @VisibleForTesting - protected static void checkFilesystemTimePrecision(String timeStamp) - throws FileSystemException - { - if (!timeStamp.matches(".*?:\\d+:\\d+\\.\\d{3,}Z")) { - throw new FileSystemException(String.format("The filesystem specified by hetu.heuristicindex.indexstore.filesystem.profile is not " + - "supported by Heuristic Index because the precision for Files.getLastModifiedTime(): %s is too low. " + - "Precision must at least be milliseconds.", timeStamp)); - } - } - public IndexClient getIndexClient() { return indexClient; @@ -117,10 +103,9 @@ public class HeuristicIndexerManager root = Paths.get(indexStoreRoot); fs = fileSystemClientManager.getFileSystemClient(fsProfile, root); if (fileSystemClientManager.isFileSystemLocal(fsProfile)) { - LOG.warn("Profile %s is not a shared filesystem. It may not work properly if the cluster has more than 1 nodes."); - String fileLastModifiedTimeSample = Files.getLastModifiedTime(Paths.get(System.getProperty("user.dir"))).toString(); - checkFilesystemTimePrecision(fileLastModifiedTimeSample); + throw new IllegalArgumentException("Indexer does not support local filesystem: " + fsProfile); } + metastore = hetuMetaStoreManager.getHetuMetastore(); if (metastore == null) { throw new IllegalStateException("Hetu metastore is not properly configured. Heuristic indexer needs it to manage index metadata. " + diff --git a/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java b/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java index c33ba2952..ae092c74d 100644 --- a/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java +++ b/presto-main/src/main/java/io/prestosql/sql/analyzer/StatementAnalyzer.java @@ -2812,10 +2812,13 @@ class StatementAnalyzer throw new SemanticException(MISSING_ATTRIBUTE, table, "Column '%s' cannot be resolved", column.getValue()); } } + + if (partitionColumn != null && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) { + throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns"); + } } - if (tableHandle.isPresent() && partitionColumn != null - && !tableHandle.get().getConnectorHandle().isPartitionColumn(partitionColumn)) { - throw new SemanticException(NOT_SUPPORTED, table, "Heuristic index creation is only supported for predicates on partition columns"); + else { + throw new SemanticException(MISSING_ATTRIBUTE, table, "Table '%s' is invalid", tableFullName); } List> indexColumns = new LinkedList<>(); diff --git a/presto-main/src/test/java/io/prestosql/heuristicindex/TestHeuristicIndexerManager.java b/presto-main/src/test/java/io/prestosql/heuristicindex/TestHeuristicIndexerManager.java deleted file mode 100644 index 64c49fc36..000000000 --- a/presto-main/src/test/java/io/prestosql/heuristicindex/TestHeuristicIndexerManager.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.prestosql.heuristicindex; - -import org.testng.annotations.Test; - -import java.nio.file.FileSystemException; - -import static io.prestosql.heuristicindex.HeuristicIndexerManager.checkFilesystemTimePrecision; - -public class TestHeuristicIndexerManager -{ - @Test - public void testFileSystemTimePrecisionMicro() - throws FileSystemException - { - String timeStamp = "2021-01-12T11:41:51.036465Z"; - checkFilesystemTimePrecision(timeStamp); - } - - @Test - public void testFileSystemTimePrecisionMilli() - throws FileSystemException - { - String timeStamp = "2021-01-12T11:41:51.036Z"; - checkFilesystemTimePrecision(timeStamp); - } - - @Test(expectedExceptions = FileSystemException.class) - public void testFileSystemTimePrecisionSec() - throws FileSystemException - { - String timeStamp = "2021-01-12T11:41:51Z"; - checkFilesystemTimePrecision(timeStamp); - } -}