diff --git a/hetu-docs/en/connector/mongodb.md b/hetu-docs/en/connector/mongodb.md new file mode 100644 index 000000000..27a997fdd --- /dev/null +++ b/hetu-docs/en/connector/mongodb.md @@ -0,0 +1,248 @@ +# MongoDB Connector + +The MongoDB connector allows MongoDB collections to be used as tables in the openLooKeng. + +**Note:** + + MongoDB 2.6 and later versions are supported, you are advised to use version 3.0 or later. + +## Configuration + +To configure the MongoDB connector, create a catalog property file `etc/catalog/mongodb.properties` by referring to the following content and replace the properties as required: + +```properties +connector.name=mongodb +mongodb.seeds=host1,host:port +``` + +### Multiple MongoDB Clusters + +Multiple catalogs can be created as required. Therefore, if there is an additional MongoDB cluster, you only need to add another property file with a different name to `etc/catalog` (ensure that it ends with `.properties`). For example, if you name the property file as `sales.properties`, the openLooKeng will create a catalog named `sales` using the configured connector. + +## Configuring Properties + +The following properties are available: + +| Property Name| Description| +|----------|----------| +| `mongodb.seeds`| List of all mongod servers| +| `mongodb.schema-collection`| A collection of schema information| +| `mongodb.case-insensitive-name-matching`| Case-insensitive matching between database and collection names| +| `mongodb.credentials`| List of credentials| +| `mongodb.min-connections-per-host`| Minimum number of the connection pools per host| +| `mongodb.connections-per-host`| Maximum number of the connection pools per host| +| `mongodb.max-wait-time`| Maximum waiting time| +| `mongodb.max-connection-idle-time`| Maximum idle time of connection pooling| +| `mongodb.connection-timeout`| Communication connection timeout| +| `mongodb.socket-timeout`| Communication timeout| +| `mongodb.socket-keep-alive`| Whether to enable the keep-alive function for each communication channel| +| `mongodb.ssl.enabled`| Using TLS/SSL to connect to mongod/mongos| +| `mongodb.read-preference`| Read preference| +| `mongodb.write-concern`| Write policy| +| `mongodb.required-replica-set`| Name of the required replica set| +| `mongodb.cursor-batch-size`| Number of elements returned in a batch| + +### `mongodb.seeds` + +List of all mongod servers in the same replica set, which are separated using commas (,). The format is in `hostname[:port]`. Or list of mongos servers in the same sharded cluster. If port is not specified, port 27017 is used. + +This property is mandatory. There is no default value, and at least one seed must be defined. + +### `mongodb.schema-collection` + +MongoDB is a document-oriented database, and there is no fixed schema information in the system. Therefore, a special collection in each MongoDB database should define the structure for all tables. For more information, see the [Table Definition](./mongodb.md#table-definition) section. + +At startup, this connector attempts to guess the type of the field, but the type may not match the collection you created. In this case, you need to manually modify it. `CREATE TABLE` and `CREATE TABLE AS SELECT` are used to create an entry for you. + +This property is optional. The default value is `_schema`. + +### `mongodb.case-insensitive-name-matching` + +Case-insensitive matching between database and collection names. + +This property is optional. The default value is `false`. + +### `mongodb.credentials` + +List of `username:password@collection` credentials separated by commas (,). + +This property is optional. There is no default value. + +### `mongodb.min-connections-per-host` + +Minimum number of connections per host in the MongoClient instance. These connections are retained in the connection pool when they are idle. Over time, the connection pool contains at least this minimum number of connections. + +This property is optional. The default value is `0`. + +### `mongodb.connections-per-host` + +Maximum number of connections per host in the MongoClient instance. These connections are retained in the connection pool when they are idle. Once the connection pool resources are exhausted, any operation that requires a connection is blocked and waits for an available connection. + +This property is optional. The default value is `100`. + +### `mongodb.max-wait-time` + +Maximum time (in milliseconds) that a thread can wait for a connection to become available. The value `0` indicates that the thread will not wait. A negative value indicates that the thread will wait will indefinitely for a connection to become available. + +This property is optional. The default value is `120000`. + +### `mongodb.connections-timeout` + +Connection timeout interval (in milliseconds). The value `0` indicates that no timeout occurs. This property is used only when a new connection is set up. + +This property is optional. The default value is `10000`. + +### `mongodb.socket-timeout` + +Socket timeout interval (in milliseconds). It is used for I/O socket read and write operations. + +This property is optional. The default value is `0`, indicating that no timeout occurs. + +### `mongodb.socket-keep-alive` + +This property controls the socket keep-alive function, which keeps the connection alive through the firewall. + +This property is optional. The default value is `false`. + +### `mongodb.ssl.enabled` + +This property is used to enable the SSL connection with the MongoDB server. + +This property is optional. The default value is `false`. + +### `mongodb.read-preference` + +The read preference are used for query, mapping restoration, aggregation, and counting. The value can be `PRIMARY`, `PRIMARY_PREFERRED`, `SECONDARY`, `SECONDARY_PREFERRED`, or `NEAREST`. + +This property is optional. The default value is `PRIMARY`. + +### `mongodb.write-concern` + +Write policy. The value can be `ACKNOWLEDGED`, `FSYNC_SAFE`, `FSYNCED`, `JOURNAL_SAFEY`, `JOURNALED`, `MAJORITY`, `NORMAL`, `REPLICA_ACKNOWLEDGED`, `REPLICAS_SAFE`, or `UNACKNOWLEDGED`. + +This property is optional. The default value is `ACKNOWLEDGED`. + +### `mongodb.required-replica-set` + +Name of the required replica set. After this property is set, the MongoClient instance performs the following operations: + +> - Connect in replica set mode and discover all members in the collection based on the specified server. +> - Ensure that the collection name reported by all members matches the required collection name. +> - If any member of the seed list is not part of a replica set with the required name, any requests are rejected. + +This property is optional. There is no default value. + +### `mongodb.cursor-batch-size` + +Limits the number of elements returned in a batch. A cursor typically fetches a batch of result objects and stores them locally. If **batchSize** is set to **0**, the default value of the driver is used. If the value of **batchSize** is positive, the value indicates the size of each batch of objects that are retrieved. The value can be adjusted to optimize performance and limit data transfer. If the value of **batchSize** is negative, it will limit the number of returned objects to the maximum batch size (typically 4 MB), and the cursor will be closed. For example, if **batchSize** is **-10**, the server will return up to 10 documents, return as many documents as possible in 4 MB, and then close the cursor. + +**Note** + + Do not set the batch size to 1. + +This property is optional. The default value is `0`. + +## Table Definition + +MongoDB maintains the table definition on the configuration special collection specified by `mongodb.schema-collection`. + +**Note** + + The plugin cannot detect a collection deletion. + You need to run db.getCollection("_schema").remove({table: delete_table_name}) in the Mongo shell to delete the collection. + Or you can delete the collection by running DROP TABLE table_name using openLooKeng. + +A collection in a schema consists of MongoDB documents of a table. + + { + "table": ..., + "fields": [ + { "name" : ..., + "type" : "varchar|bigint|boolean|double|date|array(bigint)|...", + "hidden" : false }, + ... + ] + } + } + +| Field| Mandatory or Optional| Type| Description| +|:----------|:----------|:----------|:----------| +| `table`| Mandatory| string| Name of the openLooKeng table| +| `fields`| Mandatory| array| Field definition list. For each column definition, a new column is created in the openLooKeng table.| + +The definition of each field is as follows: + + { + "name": ..., + "type": ..., + "hidden": ... + } + +| Field| Mandatory or Optional| Type| Description| +|:----------|:----------|:----------|:----------| +| `name`| Mandatory| string| Name of a column in the openLooKeng table| +| `type`| Mandatory| string| Type of a column| +| `hidden`| Optional| boolean| Hides the column from the `DESCRIBE ` and `SELECT *` results. The default value is `false`.| + +There is no restriction on the field description of the key or message. + +## ObjectId + +The MongoDB collection has a special field `_id`. The connector attempts to follow the same rules for this special field, so there will be a hidden field `_id`. + +```sql + CREATE TABLE IF NOT EXISTS orders ( + orderkey bigint, + orderstatus varchar, + totalprice double, + orderdate date + ); + + INSERT INTO orders VALUES(1, 'bad', 50.0, current_date); + INSERT INTO orders VALUES(2, 'good', 100.0, current_date); + SELECT _id, * FROM orders; +``` + +```sql + _id | orderkey | orderstatus | totalprice | orderdate + -------------------------------------+----------+-------------+------------+------------ + 55 b1 51 63 38 64 d6 43 8c 61 a9 ce | 1 | bad | 50.0 | 2015-07-23 + 55 b1 51 67 38 64 d6 43 8c 61 a9 cf | 2 | good | 100.0 | 2015-07-23 + (2 rows) +``` + +```sql + SELECT _id, * FROM orders WHERE _id = ObjectId('55b151633864d6438c61a9ce'); +``` + +```sql + _id | orderkey | orderstatus | totalprice | orderdate + -------------------------------------+----------+-------------+------------+------------ + 55 b1 51 63 38 64 d6 43 8c 61 a9 ce | 1 | bad | 50.0 | 2015-07-23 + (1 row) +``` + +The `_id` field can be rendered as a readable value and converted to `VARCHAR`: + +```sql + SELECT CAST(_id AS VARCHAR), * FROM orders WHERE _id = ObjectId('55b151633864d6438c61a9ce'); +``` + +```sql + + _id | orderkey | orderstatus | totalprice | orderdate + ---------------------------+----------+-------------+------------+------------ + 55b151633864d6438c61a9ce | 1 | bad | 50.0 | 2015-07-23 + (1 row) +``` + +## Restrictions + +\- [Row deletion](../sql/delete.md) is not supported. + +\- View creation is not supported. + +\- The empty tables and fields created by the openLooKeng cannot be queried in the MongoDB. + +\- After a table or field is deleted from the MongoDB, the table or field still exists in the openLooKeng while the values are NULL. + diff --git a/hetu-docs/zh/connector/mongodb.md b/hetu-docs/zh/connector/mongodb.md new file mode 100644 index 000000000..33fcbefe4 --- /dev/null +++ b/hetu-docs/zh/connector/mongodb.md @@ -0,0 +1,249 @@ + +# MongoDB连接器 + +MongoDB连接器允许将MongoDB集合作为openLooKeng中的表使用。 + +**注意** + + 支持MongoDB 2.6+，但建议使用3.0或更高版本。 + +## 配置 + +要配置MongoDB连接器，参考以下内容创建目录属性文件`etc/catalog/mongodb.properties`，并根据需要替换这些属性： + +``` properties +connector.name=mongodb +mongodb.seeds=host1,host:port +``` + +### 多个MongoDB集群 + +可以根据需要创建多个目录，因此，如果有额外的MongoDB集群，只需添加另一个不同的名称的属性文件到`etc/catalog`中（确保它以`.properties`结尾）。例如，如果将属性文件命名为`sales.properties`，openLooKeng将使用配置的连接器创建一个名为`sales`的目录。 + +## 配置属性 + +支持的配置属性： + +| 属性名称| 说明| +|----------|----------| +| `mongodb.seeds`| 所有mongod服务器的列表| +| `mongodb.schema-collection`| 一个包含schema信息的集合| +| `mongodb.case-insensitive-name-matching`| 不区分大小写匹配数据库和集合名称| +| `mongodb.credentials`| 认证列表| +| `mongodb.min-connections-per-host`| 每个主机的最小连接池大小| +| `mongodb.connections-per-host`| 每个主机的连接池的最大大小| +| `mongodb.max-wait-time`| 最大等待时间| +| `mongodb.max-connection-idle-time`| 池化连接的最大空闲时间| +| `mongodb.connection-timeout`| 通信连接超时| +| `mongodb.socket-timeout`| 通信超时| +| `mongodb.socket-keep-alive`| 是否在每个通信上使能keep-alive| +| `mongodb.ssl.enabled`| 使用TLS/SSL连接到mongod/mongos| +| `mongodb.read-preference`| 读偏好| +| `mongodb.write-concern`| 写入策略| +| `mongodb.required-replica-set`| 所需的副本集名称| +| `mongodb.cursor-batch-size`| 批量返回的元素数| + +### `mongodb.seeds` + +同一副本集中所有mongod服务器列表，以逗号分隔 ，格式如``hostname[:port]``，或者同一个分片集群的mongos服务器列表。如果不指定port，则使用27017端口。 + +此属性是必需的；没有默认值，并且至少必须定义一个seed。 + +### `mongodb.schema-collection` + +由于MongoDB是文档数据库，因此系统中没有固定的schema信息。因此，每个MongoDB数据库中的一个特殊集合应定义所有表的架构。有关详细信息，请参阅[表格定义](./mongodb.md#表格定义)部分。 + +在启动时，此连接器尝试猜测字段的类型，但可能与用户创建的集合可不匹配。在这种情况下，需要手动修改它。`CREATE TABLE`和`CREATE TABLE AS SELECT`会为用户创建一个条目。 + +该属性是可选的；默认值为`_schema`。 + +### `mongodb.case-insensitive-name-matching` + +不区分大小写匹配数据库和集合名称。 + +该属性是可选的；默认值为`false`。 + +### `mongodb.credentials` + +以逗号分隔的`username:password@collection`认证列表。 + +该属性是可选的；没有默认值。 + +### `mongodb.min-connectenions-per-host` + +此MongoClient实例每个主机的最小连接数。这些连接在空闲时将保留在连接池中。当这些连接空闲时，它们将保存在连接池中，并且随着时间的推移，连接池将确保至少包含这个最小数目的连接。 + +该属性是可选的；默认值为`0`。 + +### `mongodb.connections-per-host` + +此MongoClient实例每个主机允许的最大连接数。这些连接在空闲时将保留在连接池中。一旦连接池资源耗尽，任何需要连接的操作都将阻塞等待可用连接。 + +该属性是可选的；默认值为`100`。 + +### `mongodb.max-wait-time` + +线程可以等待连接变为可用状态的最大等待时间（以毫秒为单位）。值`0`表示它不会等待。负值表示无限期地等待连接变为可用。 + +该属性是可选的；默认值为`120000`。 + +### `mongodb.connections-timeout` + +连接超时（以毫秒为单位）。值`0`表示没有超时。仅在建立新连接时使用。 + +该属性是可选的；默认值为`10000`。 + +### `mongodb.socket-timeout` + +套接字超时（以毫秒为单位）。它用于I / O套接字读取和写入操作。 + +该属性是可选的；默认值为`0`并且表示没有超时。 + +### `mongodb.socket-keep-alive` + +此标志控制套接字保持活动功能，该功能通过防火墙保持连接活动。 + +该属性是可选的；默认值为`false`。 + +### `mongodb.ssl.enabled` + +此标志用于启用与MongoDB服务器的SSL连接。 + +该属性是可选的；默认值为`false`。 + +### `mongodb.read-preference` + +读偏好用于查询、映射还原、聚合和计数。可配置的值有`PRIMARY`，`PRIMARY_PREFERRED`，`SECONDARY`，`SECONDARY_PREFERRED`和`NEAREST`。 + +该属性是可选的；默认值为`PRIMARY`。 + +### `mongodb.write-concern` + +写入策略。可配置的值有`ACKNOWLEDGED`，`FSYNC_SAFE`，`FSYNCED`，`JOURNAL_SAFEY`，`JOURNALED`，`MAJORITY`，`NORMAL`，`REPLICA_ACKNOWLEDGED`，`REPLICAS_SAFE`和`UNACKNOWLEDGED`。 + +该属性是可选的；默认值为`ACKNOWLEDGED`。 + +### `mongodb.required-replica-set` + +所需的副本集名称。设置此选项后，MongoClient实例将执行以下操作: + +>- 以副本集模式连接，并根据给定的服务器发现集合中的所有成员 +>- 确保所有成员报告的集合名称与所需的集合名称匹配。 +>- 如果seed列表的任何成员不是具有必需名称的副本集的一部分，则拒绝处理任何请求。 + +该属性是可选的。没有默认值。 + +### `mongodb.cursor-batch-size` + +限制一批中返回的元素数。游标通常会获取一批结果对象并将其存储在本地。 +如果batchSize为0，将使用驱动程序的默认值。 +如果batchSize为正，则其值表示检索到的每批对象的大小。可以对其进行调整以优化性能并限制数据传输。 +如果batchSize为负，它将限制返回的对象数量，这些对象的数量在最大批处理大小限制内（通常为4MB），并且游标将被关闭。例如，如果batchSize为-10，则服务器将最多返回10个文档，并尽可能多地返回4MB中可以容纳的文档，然后关闭游标。 + +**注意** + + 请勿将批量大小设置为1。 + +该属性是可选的；默认值为`0`。 + +## 表格定义 + +MongoDB在`mongodb.schema-collection`指定的配置特殊集合上维护表格定义。 + +**注意** + + 插件无法检测到集合被删除。 + 需要在Mongo shell中通过db.getCollection("_schema").remove({ table: delete_table_name })删除集合。 + 或者通过使用openLooKeng运行DROP TABLE table_name来删除集合。 + +schema中的集合由表的MongoDB文档组成。 + + { + "table": ..., + "fields": [ + { "name" : ..., + "type" : "varchar|bigint|boolean|double|date|array(bigint)|...", + "hidden" : false }, + ... + ] + } + } + +| 字段| 是否必填| 类型| 说明| +|:----------|:----------|:----------|:----------| +| `table`| 必填| string| openLooKeng表名称。| +| `fields`| 必填| array| 字段定义列表。每个字段定义在openLooKeng表中创建一个新列。| + +每个字段定义： + + { + "name": ..., + "type": ..., + "hidden": ... + } + +| 字段| 是否必填| 类型| 说明| +|:----------|:----------|:----------|:----------| +| `name`| 必填| string| openLooKeng表中的列名。| +| `type`| 必填| string| 列的类型。| +| `hidden`| 可选| boolean| 从`DESCRIBE
`和`SELECT *`中隐藏该列。默认为`false`。| + +密钥或消息的字段描述没有限制。 + +## ObjectId + +MongoDB集合具有特殊字段`_id`。连接器尝试对此特殊字段遵循相同的规则，因此将存在隐藏字段`_id`。 +```sql + CREATE TABLE IF NOT EXISTS orders ( + orderkey bigint, + orderstatus varchar, + totalprice double, + orderdate date + ); + + INSERT INTO orders VALUES(1, 'bad', 50.0, current_date); + INSERT INTO orders VALUES(2, 'good', 100.0, current_date); + SELECT _id, * FROM orders; +``` +```sql + _id | orderkey | orderstatus | totalprice | orderdate + -------------------------------------+----------+-------------+------------+------------ + 55 b1 51 63 38 64 d6 43 8c 61 a9 ce | 1 | bad | 50.0 | 2015-07-23 + 55 b1 51 67 38 64 d6 43 8c 61 a9 cf | 2 | good | 100.0 | 2015-07-23 + (2 rows) +``` +```sql + SELECT _id, * FROM orders WHERE _id = ObjectId('55b151633864d6438c61a9ce'); +``` +```sql + _id | orderkey | orderstatus | totalprice | orderdate + -------------------------------------+----------+-------------+------------+------------ + 55 b1 51 63 38 64 d6 43 8c 61 a9 ce | 1 | bad | 50.0 | 2015-07-23 + (1 row) +``` + +可以将`_id`字段呈现为可读值，并转换为`VARCHAR`： + +```sql + SELECT CAST(_id AS VARCHAR), * FROM orders WHERE _id = ObjectId('55b151633864d6438c61a9ce'); +``` +```sql + + _id | orderkey | orderstatus | totalprice | orderdate + ---------------------------+----------+-------------+------------+------------ + 55b151633864d6438c61a9ce | 1 | bad | 50.0 | 2015-07-23 + (1 row) +``` + +限制 +----------- +>- 不支持使用[删除行](../sql/delete.md) +>- 不支持视图创建 +>- MongoDB查询不到openLooKeng新建的空表和字段。 +>- MongoDB删除表、字段后，openLooKeng中表格或者字段仍存在，但是值为NULL。 + + + + + + diff --git a/hetu-mongodb/pom.xml b/hetu-mongodb/pom.xml new file mode 100644 index 000000000..81d21a73a --- /dev/null +++ b/hetu-mongodb/pom.xml @@ -0,0 +1,200 @@ + + + + presto-root + io.hetu.core + 1.0.0-SNAPSHOT + + 4.0.0 + + hetu-mongodb + Hetu - mongodb Connector + hetu-plugin + + + ${project.parent.basedir} + 3.6.0 + 4.0.32.Final + + + + + org.mongodb + mongo-java-driver + ${mongo-java.version} + + + + joda-time + joda-time + + + + javax.validation + validation-api + + + + io.airlift + bootstrap + + + + io.airlift + json + + + + io.airlift + log + + + + io.airlift + configuration + + + + com.google.guava + guava + + + + com.google.inject + guice + + + + javax.inject + javax.inject + + + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + + + + io.airlift + log-manager + runtime + + + + + io.hetu.core + presto-spi + provided + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + org.openjdk.jol + jol-core + provided + + + + + io.hetu.core + presto-tests + test + + + + io.hetu.core + presto-main + test + + + + io.hetu.core + presto-main + test-jar + test + + + + io.hetu.core + presto-tpch + test + + + + io.airlift.tpch + tpch + test + + + + io.airlift + testing + test + + + + org.testcontainers + mongodb + 1.15.0 + + + org.slf4j + slf4j-api + + + com.fasterxml.jackson.core + jackson-annotations + + + net.java.dev.jna + jna + + + + + + net.java.dev.jna + jna + 5.5.0 + test + + + + org.testng + testng + test + + + + org.assertj + assertj-core + test + + + + io.netty + netty-transport + ${netty.version} + test + + + \ No newline at end of file diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientConfig.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientConfig.java new file mode 100644 index 000000000..48f418916 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientConfig.java @@ -0,0 +1,324 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; +import io.airlift.configuration.Config; +import io.airlift.configuration.DefunctConfig; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; + +import java.util.Arrays; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.mongodb.MongoCredential.createCredential; + +@DefunctConfig("mongodb.connection-per-host") +public class MongoClientConfig +{ + private static final Splitter SPLITTER = Splitter.on(',').trimResults().omitEmptyStrings(); + private static final Splitter PORT_SPLITTER = Splitter.on(':').trimResults().omitEmptyStrings(); + + private String schemaCollection = "_schema"; + private boolean caseInsensitiveNameMatching; + private List seeds = ImmutableList.of(); + private List credentials = ImmutableList.of(); + + private int minConnectionsPerHost; + private int connectionsPerHost = 100; + private int maxWaitTime = 120_000; + private int connectionTimeout = 10_000; + private int socketTimeout; + private int maxConnectionIdleTime; + private boolean socketKeepAlive; + private boolean sslEnabled; + + // query configurations + private int cursorBatchSize; // use driver default + + private ReadPreferenceType readPreference = ReadPreferenceType.PRIMARY; + private WriteConcernType writeConcern = WriteConcernType.ACKNOWLEDGED; + private String requiredReplicaSetName; + private String implicitRowFieldPrefix = "_pos"; + + @NotNull + public String getSchemaCollection() + { + return schemaCollection; + } + + @Config("mongodb.schema-collection") + public MongoClientConfig setSchemaCollection(String schemaCollection) + { + this.schemaCollection = schemaCollection; + return this; + } + + public boolean isCaseInsensitiveNameMatching() + { + return caseInsensitiveNameMatching; + } + + @Config("mongodb.case-insensitive-name-matching") + public MongoClientConfig setCaseInsensitiveNameMatching(boolean caseInsensitiveNameMatching) + { + this.caseInsensitiveNameMatching = caseInsensitiveNameMatching; + return this; + } + + @NotNull + @Size(min = 1) + public List getSeeds() + { + return seeds; + } + + @Config("mongodb.seeds") + public MongoClientConfig setSeeds(String commaSeparatedList) + { + this.seeds = buildSeeds(SPLITTER.split(commaSeparatedList)); + return this; + } + + public MongoClientConfig setSeeds(String... seeds) + { + this.seeds = buildSeeds(Arrays.asList(seeds)); + return this; + } + + @NotNull + public List getCredentials() + { + return credentials; + } + + @Config("mongodb.credentials") + public MongoClientConfig setCredentials(String credentials) + { + this.credentials = buildCredentials(SPLITTER.split(credentials)); + return this; + } + + public MongoClientConfig setCredentials(String... credentials) + { + this.credentials = buildCredentials(Arrays.asList(credentials)); + return this; + } + + private List buildSeeds(Iterable hostPorts) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String hostPort : hostPorts) { + List values = PORT_SPLITTER.splitToList(hostPort); + checkArgument(values.size() == 1 || values.size() == 2, "Invalid ServerAddress format. Requires host[:port]"); + if (values.size() == 1) { + builder.add(new ServerAddress(values.get(0))); + } + else { + builder.add(new ServerAddress(values.get(0), Integer.parseInt(values.get(1)))); + } + } + return builder.build(); + } + + private List buildCredentials(Iterable userPasses) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (String userPassDatabase : userPasses) { + int lastIndex = userPassDatabase.lastIndexOf('@'); + checkArgument(lastIndex > 0, "Invalid Credential format. Requires user:password@database"); + String userPass = userPassDatabase.substring(0, lastIndex); + String database = userPassDatabase.substring(lastIndex + 1); + + int firstIndex = userPass.indexOf(':'); + checkArgument(firstIndex > 0, "Invalid Credential format. Requires user:password@database"); + String user = userPass.substring(0, firstIndex); + String password = userPass.substring(firstIndex + 1); + + builder.add(createCredential(user, database, password.toCharArray())); + } + return builder.build(); + } + + @Min(0) + public int getMinConnectionsPerHost() + { + return minConnectionsPerHost; + } + + @Config("mongodb.min-connections-per-host") + public MongoClientConfig setMinConnectionsPerHost(int minConnectionsPerHost) + { + this.minConnectionsPerHost = minConnectionsPerHost; + return this; + } + + @Min(1) + public int getConnectionsPerHost() + { + return connectionsPerHost; + } + + @Config("mongodb.connections-per-host") + public MongoClientConfig setConnectionsPerHost(int connectionsPerHost) + { + this.connectionsPerHost = connectionsPerHost; + return this; + } + + @Min(0) + public int getMaxWaitTime() + { + return maxWaitTime; + } + + @Config("mongodb.max-wait-time") + public MongoClientConfig setMaxWaitTime(int maxWaitTime) + { + this.maxWaitTime = maxWaitTime; + return this; + } + + @Min(0) + public int getConnectionTimeout() + { + return connectionTimeout; + } + + @Config("mongodb.connection-timeout") + public MongoClientConfig setConnectionTimeout(int connectionTimeout) + { + this.connectionTimeout = connectionTimeout; + return this; + } + + @Min(0) + public int getSocketTimeout() + { + return socketTimeout; + } + + @Config("mongodb.socket-timeout") + public MongoClientConfig setSocketTimeout(int socketTimeout) + { + this.socketTimeout = socketTimeout; + return this; + } + + public boolean getSocketKeepAlive() + { + return socketKeepAlive; + } + + @Config("mongodb.socket-keep-alive") + public MongoClientConfig setSocketKeepAlive(boolean socketKeepAlive) + { + this.socketKeepAlive = socketKeepAlive; + return this; + } + + @NotNull + public ReadPreferenceType getReadPreference() + { + return readPreference; + } + + @Config("mongodb.read-preference") + public MongoClientConfig setReadPreference(ReadPreferenceType readPreference) + { + this.readPreference = readPreference; + return this; + } + + @NotNull + public WriteConcernType getWriteConcern() + { + return writeConcern; + } + + @Config("mongodb.write-concern") + public MongoClientConfig setWriteConcern(WriteConcernType writeConcern) + { + this.writeConcern = writeConcern; + return this; + } + + public String getRequiredReplicaSetName() + { + return requiredReplicaSetName; + } + + @Config("mongodb.required-replica-set") + public MongoClientConfig setRequiredReplicaSetName(String requiredReplicaSetName) + { + this.requiredReplicaSetName = requiredReplicaSetName; + return this; + } + + public int getCursorBatchSize() + { + return cursorBatchSize; + } + + @Config("mongodb.cursor-batch-size") + public MongoClientConfig setCursorBatchSize(int cursorBatchSize) + { + this.cursorBatchSize = cursorBatchSize; + return this; + } + + @NotNull + public String getImplicitRowFieldPrefix() + { + return implicitRowFieldPrefix; + } + + @Config("mongodb.implicit-row-field-prefix") + public MongoClientConfig setImplicitRowFieldPrefix(String implicitRowFieldPrefix) + { + this.implicitRowFieldPrefix = implicitRowFieldPrefix; + return this; + } + + public boolean getSslEnabled() + { + return this.sslEnabled; + } + + @Config("mongodb.ssl.enabled") + public MongoClientConfig setSslEnabled(boolean sslEnabled) + { + this.sslEnabled = sslEnabled; + return this; + } + + @Min(0) + public int getMaxConnectionIdleTime() + { + return maxConnectionIdleTime; + } + + @Config("mongodb.max-connection-idle-time") + public MongoClientConfig setMaxConnectionIdleTime(int maxConnectionIdleTime) + { + this.maxConnectionIdleTime = maxConnectionIdleTime; + return this; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientModule.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientModule.java new file mode 100644 index 000000000..5835835d6 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoClientModule.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.mongodb.MongoClient; +import com.mongodb.MongoClientOptions; +import io.prestosql.spi.type.TypeManager; + +import javax.inject.Singleton; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static java.util.Objects.requireNonNull; + +public class MongoClientModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(MongoConnector.class).in(Scopes.SINGLETON); + binder.bind(MongoSplitManager.class).in(Scopes.SINGLETON); + binder.bind(MongoPageSourceProvider.class).in(Scopes.SINGLETON); + binder.bind(MongoPageSinkProvider.class).in(Scopes.SINGLETON); + + configBinder(binder).bindConfig(MongoClientConfig.class); + } + + @Singleton + @Provides + public static MongoSession createMongoSession(TypeManager typeManager, MongoClientConfig config) + { + requireNonNull(config, "config is null"); + + MongoClientOptions.Builder options = MongoClientOptions.builder(); + + options.connectionsPerHost(config.getConnectionsPerHost()) + .connectTimeout(config.getConnectionTimeout()) + .socketTimeout(config.getSocketTimeout()) + .socketKeepAlive(config.getSocketKeepAlive()) + .sslEnabled(config.getSslEnabled()) + .maxWaitTime(config.getMaxWaitTime()) + .maxConnectionIdleTime(config.getMaxConnectionIdleTime()) + .minConnectionsPerHost(config.getMinConnectionsPerHost()) + .readPreference(config.getReadPreference().getReadPreference()) + .writeConcern(config.getWriteConcern().getWriteConcern()); + + if (config.getRequiredReplicaSetName() != null) { + options.requiredReplicaSetName(config.getRequiredReplicaSetName()); + } + + MongoClient client = new MongoClient(config.getSeeds(), config.getCredentials(), options.build()); + + return new MongoSession( + typeManager, + client, + config); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoColumnHandle.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoColumnHandle.java new file mode 100644 index 000000000..364688099 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoColumnHandle.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.type.Type; +import org.bson.Document; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class MongoColumnHandle + implements ColumnHandle +{ + private final String name; + private final Type type; + private final boolean hidden; + + @JsonCreator + public MongoColumnHandle( + @JsonProperty("name") String name, + @JsonProperty("columnType") Type type, + @JsonProperty("hidden") boolean hidden) + { + this.name = requireNonNull(name, "name is null"); + this.type = requireNonNull(type, "columnType is null"); + this.hidden = hidden; + } + + @JsonProperty + public String getName() + { + return name; + } + + @JsonProperty("columnType") + public Type getType() + { + return type; + } + + @JsonProperty + public boolean isHidden() + { + return hidden; + } + + public ColumnMetadata toColumnMetadata() + { + return new ColumnMetadata(name, type, null, hidden); + } + + public Document getDocument() + { + return new Document().append("name", name) + .append("type", type.getTypeSignature().toString()) + .append("hidden", hidden); + } + + @Override + public int hashCode() + { + return Objects.hash(name, type, hidden); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + MongoColumnHandle other = (MongoColumnHandle) obj; + return Objects.equals(name, other.name) && + Objects.equals(type, other.type) && + Objects.equals(hidden, other.hidden); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("name", name) + .add("type", type) + .add("hidden", hidden) + .toString(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnector.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnector.java new file mode 100644 index 000000000..2c717e31e --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnector.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import io.prestosql.spi.connector.Connector; +import io.prestosql.spi.connector.ConnectorMetadata; +import io.prestosql.spi.connector.ConnectorPageSinkProvider; +import io.prestosql.spi.connector.ConnectorPageSourceProvider; +import io.prestosql.spi.connector.ConnectorSplitManager; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.transaction.IsolationLevel; + +import javax.inject.Inject; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import static com.google.common.base.Preconditions.checkArgument; +import static io.prestosql.spi.transaction.IsolationLevel.READ_UNCOMMITTED; +import static io.prestosql.spi.transaction.IsolationLevel.checkConnectorSupports; +import static java.util.Objects.requireNonNull; + +public class MongoConnector + implements Connector +{ + private final MongoSession mongoSession; + private final MongoSplitManager splitManager; + private final MongoPageSourceProvider pageSourceProvider; + private final MongoPageSinkProvider pageSinkProvider; + + private final ConcurrentMap transactions = new ConcurrentHashMap<>(); + + @Inject + public MongoConnector( + MongoSession mongoSession, + MongoSplitManager splitManager, + MongoPageSourceProvider pageSourceProvider, + MongoPageSinkProvider pageSinkProvider) + { + this.mongoSession = mongoSession; + this.splitManager = requireNonNull(splitManager, "splitManager is null"); + this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null"); + this.pageSinkProvider = requireNonNull(pageSinkProvider, "pageSinkProvider is null"); + } + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + checkConnectorSupports(READ_UNCOMMITTED, isolationLevel); + MongoTransactionHandle transaction = new MongoTransactionHandle(); + transactions.put(transaction, new MongoMetadata(mongoSession)); + return transaction; + } + + @Override + public boolean isSingleStatementWritesOnly() + { + return true; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) + { + MongoMetadata metadata = transactions.get(transaction); + checkArgument(metadata != null, "no such transaction: %s", transaction); + return metadata; + } + + @Override + public void commit(ConnectorTransactionHandle transaction) + { + checkArgument(transactions.remove(transaction) != null, "no such transaction: %s", transaction); + } + + @Override + public void rollback(ConnectorTransactionHandle transaction) + { + MongoMetadata metadata = transactions.remove(transaction); + checkArgument(metadata != null, "no such transaction: %s", transaction); + metadata.rollback(); + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return splitManager; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return pageSourceProvider; + } + + @Override + public ConnectorPageSinkProvider getPageSinkProvider() + { + return pageSinkProvider; + } + + @Override + public void shutdown() + { + mongoSession.shutdown(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnectorFactory.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnectorFactory.java new file mode 100644 index 000000000..5374c20c4 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoConnectorFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.json.JsonModule; +import io.prestosql.spi.connector.Connector; +import io.prestosql.spi.connector.ConnectorContext; +import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.connector.ConnectorHandleResolver; +import io.prestosql.spi.type.TypeManager; + +import java.util.Map; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.requireNonNull; + +public class MongoConnectorFactory + implements ConnectorFactory +{ + private final String name; + + public MongoConnectorFactory(String name) + { + checkArgument(!isNullOrEmpty(name), "name is null or empty"); + this.name = name; + } + + @Override + public String getName() + { + return name; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new MongoHandleResolver(); + } + + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) + { + requireNonNull(config, "config is null"); + + Bootstrap app = new Bootstrap( + new JsonModule(), + new MongoClientModule(), + binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager())); + + Injector injector = app.strictConfig().doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + + return injector.getInstance(MongoConnector.class); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoHandleResolver.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoHandleResolver.java new file mode 100644 index 000000000..28a6579f0 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoHandleResolver.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorHandleResolver; +import io.prestosql.spi.connector.ConnectorInsertTableHandle; +import io.prestosql.spi.connector.ConnectorOutputTableHandle; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +public class MongoHandleResolver + implements ConnectorHandleResolver +{ + public MongoHandleResolver() + { + } + + @Override + public Class getTransactionHandleClass() + { + return MongoTransactionHandle.class; + } + + @Override + public Class getTableHandleClass() + { + return MongoTableHandle.class; + } + + @Override + public Class getColumnHandleClass() + { + return MongoColumnHandle.class; + } + + @Override + public Class getSplitClass() + { + return MongoSplit.class; + } + + @Override + public Class getOutputTableHandleClass() + { + return MongoOutputTableHandle.class; + } + + @Override + public Class getInsertTableHandleClass() + { + return MongoInsertTableHandle.class; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoIndex.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoIndex.java new file mode 100644 index 000000000..164c05422 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoIndex.java @@ -0,0 +1,133 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.mongodb.client.ListIndexesIterable; +import io.prestosql.spi.block.SortOrder; +import org.bson.Document; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + +public class MongoIndex +{ + private final String name; + private final List keys; + private final boolean unique; + + public static List parse(ListIndexesIterable indexes) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (Document index : indexes) { + // TODO: v, ns, sparse fields + Document key = (Document) index.get("key"); + String name = index.getString("name"); + boolean unique = index.getBoolean("unique", false); + + if (key.containsKey("_fts")) { // Full Text Search + continue; + } + builder.add(new MongoIndex(name, parseKey(key), unique)); + } + + return builder.build(); + } + + private static List parseKey(Document key) + { + ImmutableList.Builder builder = ImmutableList.builder(); + + for (String name : key.keySet()) { + Object value = key.get(name); + if (value instanceof Number) { + int order = ((Number) value).intValue(); + checkState(order == 1 || order == -1, "Unknown index sort order"); + builder.add(new MongodbIndexKey(name, order == 1 ? SortOrder.ASC_NULLS_LAST : SortOrder.DESC_NULLS_LAST)); + } + else if (value instanceof String) { + builder.add(new MongodbIndexKey(name, (String) value)); + } + else { + throw new UnsupportedOperationException("Unknown index type: " + value.toString()); + } + } + + return builder.build(); + } + + public MongoIndex(String name, List keys, boolean unique) + { + this.name = name; + this.keys = keys; + this.unique = unique; + } + + public String getName() + { + return name; + } + + public List getKeys() + { + return keys; + } + + public boolean isUnique() + { + return unique; + } + + public static class MongodbIndexKey + { + private final String name; + private final Optional sortOrder; + private final Optional type; + + public MongodbIndexKey(String name, SortOrder sortOrder) + { + this(name, Optional.of(sortOrder), Optional.empty()); + } + + public MongodbIndexKey(String name, String type) + { + this(name, Optional.empty(), Optional.of(type)); + } + + public MongodbIndexKey(String name, Optional sortOrder, Optional type) + { + this.name = requireNonNull(name, "name is null"); + this.sortOrder = sortOrder; + this.type = type; + } + + public String getName() + { + return name; + } + + public Optional getSortOrder() + { + return sortOrder; + } + + public Optional getType() + { + return type; + } + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoInsertTableHandle.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoInsertTableHandle.java new file mode 100644 index 000000000..8fefb8146 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoInsertTableHandle.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.connector.ConnectorInsertTableHandle; +import io.prestosql.spi.connector.SchemaTableName; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class MongoInsertTableHandle + implements ConnectorInsertTableHandle +{ + private final SchemaTableName schemaTableName; + private final List columns; + + @JsonCreator + public MongoInsertTableHandle( + @JsonProperty("schemaTableName") SchemaTableName schemaTableName, + @JsonProperty("columns") List columns) + { + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @JsonProperty + public List getColumns() + { + return columns; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoMetadata.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoMetadata.java new file mode 100644 index 000000000..b887f6997 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoMetadata.java @@ -0,0 +1,310 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.log.Logger; +import io.airlift.slice.Slice; +import io.hetu.core.plugin.mongodb.MongoIndex.MongodbIndexKey; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorInsertTableHandle; +import io.prestosql.spi.connector.ConnectorMetadata; +import io.prestosql.spi.connector.ConnectorNewTableLayout; +import io.prestosql.spi.connector.ConnectorOutputMetadata; +import io.prestosql.spi.connector.ConnectorOutputTableHandle; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTableMetadata; +import io.prestosql.spi.connector.ConnectorTableProperties; +import io.prestosql.spi.connector.Constraint; +import io.prestosql.spi.connector.ConstraintApplicationResult; +import io.prestosql.spi.connector.LocalProperty; +import io.prestosql.spi.connector.NotFoundException; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.connector.SchemaTablePrefix; +import io.prestosql.spi.connector.SortingProperty; +import io.prestosql.spi.connector.TableNotFoundException; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.statistics.ComputedStatistics; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toList; + +public class MongoMetadata + implements ConnectorMetadata +{ + private static final Logger log = Logger.get(MongoMetadata.class); + + private final MongoSession mongoSession; + + private final AtomicReference rollbackAction = new AtomicReference<>(); + + public MongoMetadata(MongoSession mongoSession) + { + this.mongoSession = requireNonNull(mongoSession, "mongoSession is null"); + } + + @Override + public List listSchemaNames(ConnectorSession session) + { + return mongoSession.getAllSchemas(); + } + + @Override + public MongoTableHandle getTableHandle(ConnectorSession session, SchemaTableName tableName) + { + requireNonNull(tableName, "tableName is null"); + try { + return mongoSession.getTable(tableName).getTableHandle(); + } + catch (TableNotFoundException e) { + log.debug(e, "Table(%s) not found", tableName); + return null; + } + } + + @Override + public ConnectorTableMetadata getTableMetadata(ConnectorSession session, ConnectorTableHandle tableHandle) + { + requireNonNull(tableHandle, "tableHandle is null"); + SchemaTableName tableName = getTableName(tableHandle); + return getTableMetadata(session, tableName); + } + + @Override + public List listTables(ConnectorSession session, Optional optionalSchemaName) + { + List schemaNames = optionalSchemaName.map(ImmutableList::of) + .orElseGet(() -> (ImmutableList) listSchemaNames(session)); + ImmutableList.Builder tableNames = ImmutableList.builder(); + for (String schemaName : schemaNames) { + for (String tableName : mongoSession.getAllTables(schemaName)) { + tableNames.add(new SchemaTableName(schemaName, tableName.toLowerCase(ENGLISH))); + } + } + return tableNames.build(); + } + + @Override + public Map getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) + { + MongoTableHandle table = (MongoTableHandle) tableHandle; + List columns = mongoSession.getTable(table.getSchemaTableName()).getColumns(); + + ImmutableMap.Builder columnHandles = ImmutableMap.builder(); + for (MongoColumnHandle columnHandle : columns) { + columnHandles.put(columnHandle.getName(), columnHandle); + } + return columnHandles.build(); + } + + @Override + public Map> listTableColumns(ConnectorSession session, SchemaTablePrefix prefix) + { + requireNonNull(prefix, "prefix is null"); + ImmutableMap.Builder> columns = ImmutableMap.builder(); + for (SchemaTableName tableName : listTables(session, prefix)) { + try { + columns.put(tableName, getTableMetadata(session, tableName).getColumns()); + } + catch (NotFoundException e) { + // table disappeared during listing operation + } + } + return columns.build(); + } + + private List listTables(ConnectorSession session, SchemaTablePrefix prefix) + { + if (!prefix.getTable().isPresent()) { + return listTables(session, prefix.getSchema()); + } + return ImmutableList.of(prefix.toSchemaTableName()); + } + + @Override + public ColumnMetadata getColumnMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle columnHandle) + { + return ((MongoColumnHandle) columnHandle).toColumnMetadata(); + } + + @Override + public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) + { + mongoSession.createTable(tableMetadata.getTable(), buildColumnHandles(tableMetadata)); + } + + @Override + public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) + { + MongoTableHandle table = (MongoTableHandle) tableHandle; + + mongoSession.dropTable(table.getSchemaTableName()); + } + + @Override + public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) + { + throw new UnsupportedOperationException(); + } + + @Override + public void addColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) + { + mongoSession.addColumn(((MongoTableHandle) tableHandle).getSchemaTableName(), column); + } + + @Override + public ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout) + { + List columns = buildColumnHandles(tableMetadata); + + mongoSession.createTable(tableMetadata.getTable(), columns); + + setRollback(() -> mongoSession.dropTable(tableMetadata.getTable())); + + return new MongoOutputTableHandle( + tableMetadata.getTable(), + columns.stream().filter(c -> !c.isHidden()).collect(toList())); + } + + @Override + public Optional finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection fragments, Collection computedStatistics) + { + clearRollback(); + return Optional.empty(); + } + + @Override + public ConnectorInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) + { + MongoTableHandle table = (MongoTableHandle) tableHandle; + List columns = mongoSession.getTable(table.getSchemaTableName()).getColumns(); + + return new MongoInsertTableHandle( + table.getSchemaTableName(), + columns.stream().filter(c -> !c.isHidden()).collect(toList())); + } + + @Override + public Optional finishInsert(ConnectorSession session, ConnectorInsertTableHandle insertHandle, Collection fragments, Collection computedStatistics) + { + return Optional.empty(); + } + + @Override + public boolean usesLegacyTableLayouts() + { + return false; + } + + @Override + public ConnectorTableProperties getTableProperties(ConnectorSession session, ConnectorTableHandle table) + { + MongoTableHandle tableHandle = (MongoTableHandle) table; + + Optional> partitioningColumns = Optional.empty(); //TODO: sharding key + ImmutableList.Builder> localProperties = ImmutableList.builder(); + + MongoTable tableInfo = mongoSession.getTable(tableHandle.getSchemaTableName()); + Map columns = getColumnHandles(session, tableHandle); + + for (MongoIndex index : tableInfo.getIndexes()) { + for (MongodbIndexKey key : index.getKeys()) { + if (!key.getSortOrder().isPresent()) { + continue; + } + if (columns.get(key.getName()) != null) { + localProperties.add(new SortingProperty<>(columns.get(key.getName()), key.getSortOrder().get())); + } + } + } + + return new ConnectorTableProperties( + TupleDomain.all(), + Optional.empty(), + partitioningColumns, + Optional.empty(), + localProperties.build()); + } + + @Override + public Optional> applyFilter(ConnectorSession session, ConnectorTableHandle table, Constraint constraint) + { + MongoTableHandle handle = (MongoTableHandle) table; + + TupleDomain oldDomain = handle.getConstraint(); + TupleDomain newDomain = oldDomain.intersect(constraint.getSummary()); + if (oldDomain.equals(newDomain)) { + return Optional.empty(); + } + + handle = new MongoTableHandle( + handle.getSchemaTableName(), + newDomain); + + return Optional.of(new ConstraintApplicationResult<>(handle, constraint.getSummary())); + } + + private void setRollback(Runnable action) + { + checkState(rollbackAction.compareAndSet(null, action), "rollback action is already set"); + } + + private void clearRollback() + { + rollbackAction.set(null); + } + + public void rollback() + { + Optional.ofNullable(rollbackAction.getAndSet(null)).ifPresent(Runnable::run); + } + + private static SchemaTableName getTableName(ConnectorTableHandle tableHandle) + { + return ((MongoTableHandle) tableHandle).getSchemaTableName(); + } + + private ConnectorTableMetadata getTableMetadata(ConnectorSession session, SchemaTableName tableName) + { + MongoTableHandle tableHandle = mongoSession.getTable(tableName).getTableHandle(); + + List columns = ImmutableList.copyOf( + getColumnHandles(session, tableHandle).values().stream() + .map(MongoColumnHandle.class::cast) + .map(MongoColumnHandle::toColumnMetadata) + .collect(toList())); + + return new ConnectorTableMetadata(tableName, columns); + } + + private static List buildColumnHandles(ConnectorTableMetadata tableMetadata) + { + return tableMetadata.getColumns().stream() + .map(m -> new MongoColumnHandle(m.getName(), m.getType(), m.isHidden())) + .collect(toList()); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoOutputTableHandle.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoOutputTableHandle.java new file mode 100644 index 000000000..977b08a10 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoOutputTableHandle.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.connector.ConnectorOutputTableHandle; +import io.prestosql.spi.connector.SchemaTableName; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class MongoOutputTableHandle + implements ConnectorOutputTableHandle +{ + private final SchemaTableName schemaTableName; + private final List columns; + + @JsonCreator + public MongoOutputTableHandle( + @JsonProperty("schemaTableName") SchemaTableName schemaTableName, + @JsonProperty("columns") List columns) + { + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @JsonProperty + public List getColumns() + { + return columns; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSink.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSink.java new file mode 100644 index 000000000..cbc1ffccb --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSink.java @@ -0,0 +1,256 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Shorts; +import com.google.common.primitives.SignedBytes; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.InsertManyOptions; +import io.airlift.slice.Slice; +import io.prestosql.spi.Page; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.StandardErrorCode; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.connector.ConnectorPageSink; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.BooleanType; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.DoubleType; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.NamedTypeSignature; +import io.prestosql.spi.type.RealType; +import io.prestosql.spi.type.SmallintType; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.TimestampWithTimeZoneType; +import io.prestosql.spi.type.TinyintType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignatureParameter; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; +import org.bson.Document; +import org.bson.types.Binary; +import org.bson.types.ObjectId; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID; +import static io.hetu.core.plugin.mongodb.TypeUtils.isArrayType; +import static io.hetu.core.plugin.mongodb.TypeUtils.isMapType; +import static io.hetu.core.plugin.mongodb.TypeUtils.isRowType; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.Chars.padSpaces; +import static io.prestosql.spi.type.DateTimeEncoding.unpackMillisUtc; +import static io.prestosql.spi.type.Decimals.readBigDecimal; +import static java.lang.Float.intBitsToFloat; +import static java.lang.Math.toIntExact; +import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.CompletableFuture.completedFuture; + +public class MongoPageSink + implements ConnectorPageSink +{ + private final MongoSession mongoSession; + private final SchemaTableName schemaTableName; + private final List columns; + private final String implicitPrefix; + + public MongoPageSink( + MongoClientConfig config, + MongoSession mongoSession, + SchemaTableName schemaTableName, + List columns) + { + this.mongoSession = mongoSession; + this.schemaTableName = schemaTableName; + this.columns = columns; + this.implicitPrefix = requireNonNull(config.getImplicitRowFieldPrefix(), "config.getImplicitRowFieldPrefix() is null"); + } + + @Override + public CompletableFuture appendPage(Page page) + { + MongoCollection collection = mongoSession.getCollection(schemaTableName); + List batch = new ArrayList<>(page.getPositionCount()); + + for (int position = 0; position < page.getPositionCount(); position++) { + Document doc = new Document(); + + for (int channel = 0; channel < page.getChannelCount(); channel++) { + MongoColumnHandle column = columns.get(channel); + doc.append(column.getName(), getObjectValue(columns.get(channel).getType(), page.getBlock(channel), position)); + } + batch.add(doc); + } + + collection.insertMany(batch, new InsertManyOptions().ordered(true)); + return NOT_BLOCKED; + } + + private Object getObjectValue(Type type, Block block, int position) + { + if (block.isNull(position)) { + if (type.equals(OBJECT_ID)) { + return new ObjectId(); + } + return null; + } + + if (type.equals(OBJECT_ID)) { + return new ObjectId(block.getSlice(position, 0, block.getSliceLength(position)).getBytes()); + } + if (type.equals(BooleanType.BOOLEAN)) { + return type.getBoolean(block, position); + } + if (type.equals(BigintType.BIGINT)) { + return type.getLong(block, position); + } + if (type.equals(IntegerType.INTEGER)) { + return toIntExact(type.getLong(block, position)); + } + if (type.equals(SmallintType.SMALLINT)) { + return Shorts.checkedCast(type.getLong(block, position)); + } + if (type.equals(TinyintType.TINYINT)) { + return SignedBytes.checkedCast(type.getLong(block, position)); + } + if (type.equals(RealType.REAL)) { + return intBitsToFloat(toIntExact(type.getLong(block, position))); + } + if (type.equals(DoubleType.DOUBLE)) { + return type.getDouble(block, position); + } + if (type instanceof VarcharType) { + return type.getSlice(block, position).toStringUtf8(); + } + if (type instanceof CharType) { + return padSpaces(type.getSlice(block, position), ((CharType) type)).toStringUtf8(); + } + if (type.equals(VarbinaryType.VARBINARY)) { + return new Binary(type.getSlice(block, position).getBytes()); + } + if (type.equals(DateType.DATE)) { + long days = type.getLong(block, position); + return new Date(TimeUnit.DAYS.toMillis(days)); + } + if (type.equals(TimeType.TIME)) { + long millisUtc = type.getLong(block, position); + return new Date(millisUtc); + } + if (type.equals(TimestampType.TIMESTAMP)) { + long millisUtc = type.getLong(block, position); + return new Date(millisUtc); + } + if (type.equals(TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE)) { + long millisUtc = unpackMillisUtc(type.getLong(block, position)); + return new Date(millisUtc); + } + if (type instanceof DecimalType) { + return readBigDecimal((DecimalType) type, block, position); + } + if (isArrayType(type)) { + Type elementType = type.getTypeParameters().get(0); + + Block arrayBlock = (Block) block.getObject(position, Block.class); + + List list = new ArrayList<>(arrayBlock.getPositionCount()); + for (int i = 0; i < arrayBlock.getPositionCount(); i++) { + Object element = getObjectValue(elementType, arrayBlock, i); + list.add(element); + } + + return unmodifiableList(list); + } + if (isMapType(type)) { + Type keyType = type.getTypeParameters().get(0); + Type valueType = type.getTypeParameters().get(1); + + Block mapBlock = (Block) block.getObject(position, Block.class); + + // map type is converted into list of fixed keys document + List values = new ArrayList<>(mapBlock.getPositionCount() / 2); + for (int i = 0; i < mapBlock.getPositionCount(); i += 2) { + Map mapValue = new HashMap<>(); + mapValue.put("key", getObjectValue(keyType, mapBlock, i)); + mapValue.put("value", getObjectValue(valueType, mapBlock, i + 1)); + values.add(mapValue); + } + + return unmodifiableList(values); + } + if (isRowType(type)) { + Block rowBlock = (Block) block.getObject(position, Block.class); + + List fieldTypes = type.getTypeParameters(); + if (fieldTypes.size() != rowBlock.getPositionCount()) { + throw new PrestoException(StandardErrorCode.GENERIC_INTERNAL_ERROR, "Expected row value field count does not match type field count"); + } + + if (isImplicitRowType(type)) { + List rowValue = new ArrayList<>(); + for (int i = 0; i < rowBlock.getPositionCount(); i++) { + Object element = getObjectValue(fieldTypes.get(i), rowBlock, i); + rowValue.add(element); + } + return unmodifiableList(rowValue); + } + + Map rowValue = new HashMap<>(); + for (int i = 0; i < rowBlock.getPositionCount(); i++) { + rowValue.put( + type.getTypeSignature().getParameters().get(i).getNamedTypeSignature().getName().orElse("field" + i), + getObjectValue(fieldTypes.get(i), rowBlock, i)); + } + return unmodifiableMap(rowValue); + } + + throw new PrestoException(NOT_SUPPORTED, "unsupported type: " + type); + } + + private boolean isImplicitRowType(Type type) + { + return type.getTypeSignature().getParameters() + .stream() + .map(TypeSignatureParameter::getNamedTypeSignature) + .map(NamedTypeSignature::getName) + .filter(Optional::isPresent) + .map(Optional::get) + .allMatch(name -> name.startsWith(implicitPrefix)); + } + + @Override + public CompletableFuture> finish() + { + return completedFuture(ImmutableList.of()); + } + + @Override + public void abort() + { + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSinkProvider.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSinkProvider.java new file mode 100644 index 000000000..677fc5dd4 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSinkProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import io.prestosql.spi.connector.ConnectorInsertTableHandle; +import io.prestosql.spi.connector.ConnectorOutputTableHandle; +import io.prestosql.spi.connector.ConnectorPageSink; +import io.prestosql.spi.connector.ConnectorPageSinkProvider; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +import javax.inject.Inject; + +public class MongoPageSinkProvider + implements ConnectorPageSinkProvider +{ + private final MongoClientConfig config; + private final MongoSession mongoSession; + + @Inject + public MongoPageSinkProvider(MongoClientConfig config, MongoSession mongoSession) + { + this.config = config; + this.mongoSession = mongoSession; + } + + @Override + public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle) + { + MongoOutputTableHandle handle = (MongoOutputTableHandle) outputTableHandle; + return new MongoPageSink(config, mongoSession, handle.getSchemaTableName(), handle.getColumns()); + } + + @Override + public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle) + { + MongoInsertTableHandle handle = (MongoInsertTableHandle) insertTableHandle; + return new MongoPageSink(config, mongoSession, handle.getSchemaTableName(), handle.getColumns()); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSource.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSource.java new file mode 100644 index 000000000..7c7a370ef --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSource.java @@ -0,0 +1,344 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.primitives.Shorts; +import com.google.common.primitives.SignedBytes; +import com.mongodb.client.MongoCursor; +import io.airlift.slice.Slice; +import io.prestosql.spi.Page; +import io.prestosql.spi.PageBuilder; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.connector.ConnectorPageSource; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeSignatureParameter; +import io.prestosql.spi.type.VarbinaryType; +import io.prestosql.spi.type.VarcharType; +import org.bson.Document; +import org.bson.types.Binary; +import org.bson.types.Decimal128; +import org.bson.types.ObjectId; +import org.joda.time.chrono.ISOChronology; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Verify.verify; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID; +import static io.hetu.core.plugin.mongodb.TypeUtils.isArrayType; +import static io.hetu.core.plugin.mongodb.TypeUtils.isMapType; +import static io.hetu.core.plugin.mongodb.TypeUtils.isRowType; +import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.Chars.truncateToLengthAndTrimSpaces; +import static io.prestosql.spi.type.DateTimeEncoding.packDateTimeWithZone; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.Decimals.encodeScaledValue; +import static io.prestosql.spi.type.Decimals.encodeShortScaledValue; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; +import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static java.lang.Float.floatToIntBits; +import static java.lang.String.join; +import static java.util.stream.Collectors.toList; + +public class MongoPageSource + implements ConnectorPageSource +{ + private static final ISOChronology UTC_CHRONOLOGY = ISOChronology.getInstanceUTC(); + private static final int ROWS_PER_REQUEST = 1024; + + private final MongoCursor cursor; + private final List columnNames; + private final List columnTypes; + private Document currentDoc; + private long count; + private boolean finished; + + private final PageBuilder pageBuilder; + + public MongoPageSource( + MongoSession mongoSession, + MongoTableHandle tableHandle, + List columns) + { + this.columnNames = columns.stream().map(MongoColumnHandle::getName).collect(toList()); + this.columnTypes = columns.stream().map(MongoColumnHandle::getType).collect(toList()); + this.cursor = mongoSession.execute(tableHandle, columns); + currentDoc = null; + + pageBuilder = new PageBuilder(columnTypes); + } + + @Override + public long getCompletedBytes() + { + return count; + } + + @Override + public long getReadTimeNanos() + { + return 0; + } + + @Override + public boolean isFinished() + { + return finished; + } + + @Override + public long getSystemMemoryUsage() + { + return 0L; + } + + @Override + public Page getNextPage() + { + verify(pageBuilder.isEmpty()); + count = 0; + for (int i = 0; i < ROWS_PER_REQUEST; i++) { + if (!cursor.hasNext()) { + finished = true; + break; + } + currentDoc = cursor.next(); + count++; + + pageBuilder.declarePosition(); + for (int column = 0; column < columnTypes.size(); column++) { + BlockBuilder output = pageBuilder.getBlockBuilder(column); + appendTo(columnTypes.get(column), currentDoc.get(columnNames.get(column)), output); + } + } + + Page page = pageBuilder.build(); + pageBuilder.reset(); + return page; + } + + private void appendTo(Type type, Object value, BlockBuilder output) + { + if (value == null) { + output.appendNull(); + return; + } + + Class javaType = type.getJavaType(); + try { + if (javaType == boolean.class) { + type.writeBoolean(output, (Boolean) value); + } + else if (javaType == long.class) { + if (type.equals(BIGINT)) { + type.writeLong(output, ((Number) value).longValue()); + } + else if (type.equals(INTEGER)) { + type.writeLong(output, ((Number) value).intValue()); + } + else if (type.equals(SMALLINT)) { + type.writeLong(output, Shorts.checkedCast(((Number) value).longValue())); + } + else if (type.equals(TINYINT)) { + type.writeLong(output, SignedBytes.checkedCast(((Number) value).longValue())); + } + else if (type.equals(REAL)) { + //noinspection NumericCastThatLosesPrecision + type.writeLong(output, floatToIntBits(((float) ((Number) value).doubleValue()))); + } + else if (type instanceof DecimalType) { + type.writeLong(output, encodeShortScaledValue(((Decimal128) value).bigDecimalValue(), ((DecimalType) type).getScale())); + } + else if (type.equals(DATE)) { + long utcMillis = ((Date) value).getTime(); + type.writeLong(output, TimeUnit.MILLISECONDS.toDays(utcMillis)); + } + else if (type.equals(TIME)) { + type.writeLong(output, UTC_CHRONOLOGY.millisOfDay().get(((Date) value).getTime())); + } + else if (type.equals(TIMESTAMP)) { + // TODO provide correct TIMESTAMP mapping, and respecting session.isLegacyTimestamp() + type.writeLong(output, ((Date) value).getTime()); + } + else if (type.equals(TIMESTAMP_WITH_TIME_ZONE)) { + type.writeLong(output, packDateTimeWithZone(((Date) value).getTime(), UTC_KEY)); + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled type for " + javaType.getSimpleName() + ":" + type.getTypeSignature()); + } + } + else if (javaType == double.class) { + type.writeDouble(output, ((Number) value).doubleValue()); + } + else if (javaType == Slice.class) { + writeSlice(output, type, value); + } + else if (javaType == Block.class) { + writeBlock(output, type, value); + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled type for " + javaType.getSimpleName() + ":" + type.getTypeSignature()); + } + } + catch (ClassCastException ignore) { + // TODO remove (fail clearly), or hide behind a toggle + // returns null instead of raising exception + output.appendNull(); + } + } + + private String toVarcharValue(Object value) + { + if (value instanceof Collection) { + return "[" + join(", ", ((Collection) value).stream().map(this::toVarcharValue).collect(toList())) + "]"; + } + if (value instanceof Document) { + return ((Document) value).toJson(); + } + return String.valueOf(value); + } + + private void writeSlice(BlockBuilder output, Type type, Object value) + { + if (type instanceof VarcharType) { + type.writeSlice(output, utf8Slice(toVarcharValue(value))); + } + else if (type instanceof CharType) { + type.writeSlice(output, truncateToLengthAndTrimSpaces(utf8Slice((String) value), ((CharType) type))); + } + else if (type.equals(OBJECT_ID)) { + type.writeSlice(output, wrappedBuffer(((ObjectId) value).toByteArray())); + } + else if (type instanceof VarbinaryType) { + if (value instanceof Binary) { + type.writeSlice(output, wrappedBuffer(((Binary) value).getData())); + } + else { + output.appendNull(); + } + } + else if (type instanceof DecimalType) { + type.writeSlice(output, encodeScaledValue(((Decimal128) value).bigDecimalValue(), ((DecimalType) type).getScale())); + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled type for Slice: " + type.getTypeSignature()); + } + } + + private void writeBlock(BlockBuilder output, Type type, Object value) + { + if (isArrayType(type)) { + if (value instanceof List) { + BlockBuilder builder = output.beginBlockEntry(); + + ((List) value).forEach(element -> + appendTo(type.getTypeParameters().get(0), element, builder)); + + output.closeEntry(); + return; + } + } + else if (isMapType(type)) { + if (value instanceof List) { + BlockBuilder builder = output.beginBlockEntry(); + for (Object element : (List) value) { + if (!(element instanceof Map)) { + continue; + } + + Map document = (Map) element; + if (document.containsKey("key") && document.containsKey("value")) { + appendTo(type.getTypeParameters().get(0), document.get("key"), builder); + appendTo(type.getTypeParameters().get(1), document.get("value"), builder); + } + } + + output.closeEntry(); + return; + } + else if (value instanceof Map) { + BlockBuilder builder = output.beginBlockEntry(); + Map document = (Map) value; + for (Map.Entry entry : document.entrySet()) { + appendTo(type.getTypeParameters().get(0), entry.getKey(), builder); + appendTo(type.getTypeParameters().get(1), entry.getValue(), builder); + } + output.closeEntry(); + return; + } + } + else if (isRowType(type)) { + if (value instanceof Map) { + Map mapValue = (Map) value; + BlockBuilder builder = output.beginBlockEntry(); + + List fieldNames = new ArrayList<>(); + for (int i = 0; i < type.getTypeSignature().getParameters().size(); i++) { + TypeSignatureParameter parameter = type.getTypeSignature().getParameters().get(i); + fieldNames.add(parameter.getNamedTypeSignature().getName().orElse("field" + i)); + } + checkState(fieldNames.size() == type.getTypeParameters().size(), "fieldName doesn't match with type size : %s", type); + for (int index = 0; index < type.getTypeParameters().size(); index++) { + appendTo(type.getTypeParameters().get(index), mapValue.get(fieldNames.get(index)), builder); + } + output.closeEntry(); + return; + } + else if (value instanceof List) { + List listValue = (List) value; + BlockBuilder builder = output.beginBlockEntry(); + for (int index = 0; index < type.getTypeParameters().size(); index++) { + if (index < listValue.size()) { + appendTo(type.getTypeParameters().get(index), listValue.get(index), builder); + } + else { + builder.appendNull(); + } + } + output.closeEntry(); + return; + } + } + else { + throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unhandled type for Block: " + type.getTypeSignature()); + } + + // not a convertible value + output.appendNull(); + } + + @Override + public void close() + { + cursor.close(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSourceProvider.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSourceProvider.java new file mode 100644 index 000000000..c98bac501 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPageSourceProvider.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorPageSource; +import io.prestosql.spi.connector.ConnectorPageSourceProvider; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplit; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +import javax.inject.Inject; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class MongoPageSourceProvider + implements ConnectorPageSourceProvider +{ + private final MongoSession mongoSession; + + @Inject + public MongoPageSourceProvider(MongoSession mongoSession) + { + this.mongoSession = requireNonNull(mongoSession, "mongoSession is null"); + } + + @Override + public ConnectorPageSource createPageSource( + ConnectorTransactionHandle transaction, + ConnectorSession session, + ConnectorSplit split, + ConnectorTableHandle table, + List columns) + { + MongoTableHandle tableHandle = (MongoTableHandle) table; + + ImmutableList.Builder handles = ImmutableList.builder(); + for (ColumnHandle handle : requireNonNull(columns, "columns is null")) { + handles.add((MongoColumnHandle) handle); + } + + return new MongoPageSource(mongoSession, tableHandle, handles.build()); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPlugin.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPlugin.java new file mode 100644 index 000000000..5e532fa7e --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoPlugin.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import io.prestosql.spi.Plugin; +import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.type.Type; + +import java.util.Set; + +import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID; + +public class MongoPlugin + implements Plugin +{ + @Override + public Iterable getTypes() + { + return ImmutableList.of(OBJECT_ID); + } + + @Override + public Set> getFunctions() + { + return ImmutableSet.of(ObjectIdFunctions.class); + } + + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new MongoConnectorFactory("mongodb")); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSession.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSession.java new file mode 100644 index 000000000..b2a9635a7 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSession.java @@ -0,0 +1,679 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.primitives.Primitives; +import com.google.common.primitives.Shorts; +import com.google.common.primitives.SignedBytes; +import com.google.common.util.concurrent.UncheckedExecutionException; +import com.mongodb.MongoClient; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.IndexOptions; +import com.mongodb.client.result.DeleteResult; +import io.airlift.log.Logger; +import io.airlift.slice.Slice; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.SchemaNotFoundException; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.connector.TableNotFoundException; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.Range; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.type.IntegerType; +import io.prestosql.spi.type.NamedTypeSignature; +import io.prestosql.spi.type.RowFieldName; +import io.prestosql.spi.type.StandardTypes; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.TypeManager; +import io.prestosql.spi.type.TypeSignature; +import io.prestosql.spi.type.TypeSignatureParameter; +import io.prestosql.spi.type.VarcharType; +import org.bson.Document; +import org.bson.types.ObjectId; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Throwables.throwIfInstanceOf; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.HOURS; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + +public class MongoSession +{ + private static final Logger log = Logger.get(MongoSession.class); + private static final List SYSTEM_TABLES = Arrays.asList("system.indexes", "system.users", "system.version"); + + private static final String TABLE_NAME_KEY = "table"; + private static final String FIELDS_KEY = "fields"; + private static final String FIELDS_NAME_KEY = "name"; + private static final String FIELDS_TYPE_KEY = "type"; + private static final String FIELDS_HIDDEN_KEY = "hidden"; + + private static final String OR_OP = "$or"; + + private static final String EQ_OP = "$eq"; + private static final String NOT_EQ_OP = "$ne"; + private static final String GTE_OP = "$gte"; + private static final String GT_OP = "$gt"; + private static final String LT_OP = "$lt"; + private static final String LTE_OP = "$lte"; + private static final String IN_OP = "$in"; + + private final TypeManager typeManager; + private final MongoClient client; + + private final String schemaCollection; + private final boolean caseInsensitiveNameMatching; + private final int cursorBatchSize; + + private final LoadingCache tableCache; + private final String implicitPrefix; + + public MongoSession(TypeManager typeManager, MongoClient client, MongoClientConfig config) + { + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.client = requireNonNull(client, "client is null"); + this.schemaCollection = requireNonNull(config.getSchemaCollection(), "config.getSchemaCollection() is null"); + this.caseInsensitiveNameMatching = config.isCaseInsensitiveNameMatching(); + this.cursorBatchSize = config.getCursorBatchSize(); + this.implicitPrefix = requireNonNull(config.getImplicitRowFieldPrefix(), "config.getImplicitRowFieldPrefix() is null"); + + this.tableCache = CacheBuilder.newBuilder() + .expireAfterWrite(1, HOURS) // TODO: Configure + .refreshAfterWrite(1, MINUTES) + .build(CacheLoader.from(this::loadTableSchema)); + } + + public void shutdown() + { + client.close(); + } + + public List getAllSchemas() + { + return ImmutableList.copyOf(client.listDatabaseNames()).stream() + .map(name -> name.toLowerCase(ENGLISH)) + .collect(toImmutableList()); + } + + public Set getAllTables(String schema) + throws SchemaNotFoundException + { + String schemaName = toRemoteSchemaName(schema); + ImmutableSet.Builder builder = ImmutableSet.builder(); + + builder.addAll(ImmutableList.copyOf(client.getDatabase(schemaName).listCollectionNames()).stream() + .filter(name -> !name.equals(schemaCollection)) + .filter(name -> !SYSTEM_TABLES.contains(name)) + .collect(toSet())); + builder.addAll(getTableMetadataNames(schema)); + + return builder.build(); + } + + public MongoTable getTable(SchemaTableName tableName) + throws TableNotFoundException + { + try { + return tableCache.getUnchecked(tableName); + } + catch (UncheckedExecutionException e) { + throwIfInstanceOf(e.getCause(), PrestoException.class); + throw e; + } + } + + public void createTable(SchemaTableName name, List columns) + { + createTableMetadata(name, columns); + // collection is created implicitly + } + + public void dropTable(SchemaTableName tableName) + { + deleteTableMetadata(tableName); + getCollection(tableName).drop(); + + tableCache.invalidate(tableName); + } + + public void addColumn(SchemaTableName schemaTableName, ColumnMetadata columnMetadata) + { + Document metadata = getTableMetadata(schemaTableName); + + List columns = new ArrayList<>(getColumnMetadata(metadata)); + + Document newColumn = new Document(); + newColumn.append(FIELDS_NAME_KEY, columnMetadata.getName()); + newColumn.append(FIELDS_TYPE_KEY, columnMetadata.getType().getTypeSignature().toString()); + newColumn.append(FIELDS_HIDDEN_KEY, false); + columns.add(newColumn); + + String schemaName = toRemoteSchemaName(schemaTableName.getSchemaName()); + String tableName = toRemoteTableName(schemaName, schemaTableName.getTableName()); + + metadata.append(FIELDS_KEY, columns); + + MongoDatabase db = client.getDatabase(schemaName); + MongoCollection schema = db.getCollection(schemaCollection); + schema.findOneAndReplace(new Document(TABLE_NAME_KEY, tableName), metadata); + + tableCache.invalidate(schemaTableName); + } + + private MongoTable loadTableSchema(SchemaTableName tableName) + throws TableNotFoundException + { + Document tableMeta = getTableMetadata(tableName); + + ImmutableList.Builder columnHandles = ImmutableList.builder(); + + for (Document columnMetadata : getColumnMetadata(tableMeta)) { + MongoColumnHandle columnHandle = buildColumnHandle(columnMetadata); + columnHandles.add(columnHandle); + } + + MongoTableHandle tableHandle = new MongoTableHandle(tableName); + return new MongoTable(tableHandle, columnHandles.build(), getIndexes(tableName)); + } + + private MongoColumnHandle buildColumnHandle(Document columnMeta) + { + String name = columnMeta.getString(FIELDS_NAME_KEY); + String typeString = columnMeta.getString(FIELDS_TYPE_KEY); + boolean hidden = columnMeta.getBoolean(FIELDS_HIDDEN_KEY, false); + + Type type = typeManager.getType(TypeSignature.parseTypeSignature(typeString)); + + return new MongoColumnHandle(name, type, hidden); + } + + private List getColumnMetadata(Document doc) + { + if (!doc.containsKey(FIELDS_KEY)) { + return ImmutableList.of(); + } + + return (List) doc.get(FIELDS_KEY); + } + + public MongoCollection getCollection(SchemaTableName tableName) + { + return getCollection(tableName.getSchemaName(), tableName.getTableName()); + } + + private MongoCollection getCollection(String schema, String table) + { + String schemaName = toRemoteSchemaName(schema); + String tableName = toRemoteTableName(schemaName, table); + return client.getDatabase(schemaName).getCollection(tableName); + } + + public List getIndexes(SchemaTableName tableName) + { + if (isView(tableName)) { + return ImmutableList.of(); + } + return MongoIndex.parse(getCollection(tableName).listIndexes()); + } + + public MongoCursor execute(MongoTableHandle tableHandle, List columns) + { + Document output = new Document(); + for (MongoColumnHandle column : columns) { + output.append(column.getName(), 1); + } + MongoCollection collection = getCollection(tableHandle.getSchemaTableName()); + Document query = buildQuery(tableHandle.getConstraint()); + FindIterable iterable = collection.find(query).projection(output); + log.debug("Find documents: collection: %s, filter: %s, projection: %s", tableHandle.getSchemaTableName(), query.toJson(), output.toJson()); + + if (cursorBatchSize != 0) { + iterable.batchSize(cursorBatchSize); + } + + return iterable.iterator(); + } + + @VisibleForTesting + static Document buildQuery(TupleDomain tupleDomain) + { + Document query = new Document(); + if (tupleDomain.getDomains().isPresent()) { + for (Map.Entry entry : tupleDomain.getDomains().get().entrySet()) { + MongoColumnHandle column = (MongoColumnHandle) entry.getKey(); + Optional predicate = buildPredicate(column, entry.getValue()); + predicate.ifPresent(query::putAll); + } + } + + return query; + } + + private static Optional buildPredicate(MongoColumnHandle column, Domain domain) + { + String name = column.getName(); + Type type = column.getType(); + if (domain.getValues().isNone() && domain.isNullAllowed()) { + return Optional.of(documentOf(name, isNullPredicate())); + } + if (domain.getValues().isAll() && !domain.isNullAllowed()) { + return Optional.of(documentOf(name, isNotNullPredicate())); + } + + List singleValues = new ArrayList<>(); + List disjuncts = new ArrayList<>(); + for (Range range : domain.getValues().getRanges().getOrderedRanges()) { + if (range.isSingleValue()) { + Optional translated = translateValue(range.getSingleValue(), type); + if (!translated.isPresent()) { + return Optional.empty(); + } + singleValues.add(translated.get()); + } + else { + Document rangeConjuncts = new Document(); + if (!range.getLow().isLowerUnbounded()) { + Optional translated = translateValue(range.getLow().getValue(), type); + if (!translated.isPresent()) { + return Optional.empty(); + } + switch (range.getLow().getBound()) { + case ABOVE: + rangeConjuncts.put(GT_OP, translated.get()); + break; + case EXACTLY: + rangeConjuncts.put(GTE_OP, translated.get()); + break; + case BELOW: + throw new IllegalArgumentException("Low Marker should never use BELOW bound: " + range); + default: + throw new AssertionError("Unhandled bound: " + range.getLow().getBound()); + } + } + if (!range.getHigh().isUpperUnbounded()) { + Optional translated = translateValue(range.getHigh().getValue(), type); + if (!translated.isPresent()) { + return Optional.empty(); + } + switch (range.getHigh().getBound()) { + case ABOVE: + throw new IllegalArgumentException("High Marker should never use ABOVE bound: " + range); + case EXACTLY: + rangeConjuncts.put(LTE_OP, translated.get()); + break; + case BELOW: + rangeConjuncts.put(LT_OP, translated.get()); + break; + default: + throw new AssertionError("Unhandled bound: " + range.getHigh().getBound()); + } + } + // If rangeConjuncts is null, then the range was ALL, which should already have been checked for + verify(!rangeConjuncts.isEmpty()); + disjuncts.add(rangeConjuncts); + } + } + + // Add back all of the possible single values either as an equality or an IN predicate + if (singleValues.size() == 1) { + disjuncts.add(documentOf(EQ_OP, singleValues.get(0))); + } + else if (singleValues.size() > 1) { + disjuncts.add(documentOf(IN_OP, singleValues)); + } + + if (domain.isNullAllowed()) { + disjuncts.add(isNullPredicate()); + } + + return Optional.of(orPredicate(disjuncts.stream() + .map(disjunct -> new Document(name, disjunct)) + .collect(toImmutableList()))); + } + + private static Optional translateValue(Object prestoNativeValue, Type type) + { + requireNonNull(prestoNativeValue, "prestoNativeValue is null"); + requireNonNull(type, "type is null"); + checkArgument(Primitives.wrap(type.getJavaType()).isInstance(prestoNativeValue), "%s (%s) is not a valid representation for %s", prestoNativeValue, prestoNativeValue.getClass(), type); + + if (type == TINYINT) { + return Optional.of((long) SignedBytes.checkedCast(((Long) prestoNativeValue))); + } + + if (type == SMALLINT) { + return Optional.of((long) Shorts.checkedCast(((Long) prestoNativeValue))); + } + + if (type == IntegerType.INTEGER) { + return Optional.of((long) toIntExact(((Long) prestoNativeValue))); + } + + if (type == BIGINT) { + return Optional.of(prestoNativeValue); + } + + if (type instanceof ObjectIdType) { + return Optional.of(new ObjectId(((Slice) prestoNativeValue).getBytes())); + } + + if (type instanceof VarcharType) { + return Optional.of(((Slice) prestoNativeValue).toStringUtf8()); + } + + return Optional.empty(); + } + + private static Document documentOf(String key, Object value) + { + return new Document(key, value); + } + + private static Document orPredicate(List values) + { + checkState(!values.isEmpty()); + if (values.size() == 1) { + return values.get(0); + } + return new Document(OR_OP, values); + } + + private static Document isNullPredicate() + { + return documentOf(EQ_OP, null); + } + + private static Document isNotNullPredicate() + { + return documentOf(NOT_EQ_OP, null); + } + + // Internal Schema management + private Document getTableMetadata(SchemaTableName schemaTableName) + throws TableNotFoundException + { + String schemaName = toRemoteSchemaName(schemaTableName.getSchemaName()); + String tableName = toRemoteTableName(schemaName, schemaTableName.getTableName()); + + MongoDatabase db = client.getDatabase(schemaName); + MongoCollection schema = db.getCollection(schemaCollection); + + Document doc = schema + .find(new Document(TABLE_NAME_KEY, tableName)).first(); + + if (doc == null) { + if (!collectionExists(db, tableName)) { + throw new TableNotFoundException(schemaTableName); + } + else { + Document metadata = new Document(TABLE_NAME_KEY, tableName); + metadata.append(FIELDS_KEY, guessTableFields(schemaName, tableName)); + + schema.createIndex(new Document(TABLE_NAME_KEY, 1), new IndexOptions().unique(true)); + schema.insertOne(metadata); + + return metadata; + } + } + + return doc; + } + + public boolean collectionExists(MongoDatabase db, String collectionName) + { + for (String name : db.listCollectionNames()) { + if (name.equalsIgnoreCase(collectionName)) { + return true; + } + } + return false; + } + + private Set getTableMetadataNames(String schemaName) + throws TableNotFoundException + { + MongoDatabase db = client.getDatabase(schemaName); + MongoCursor cursor = db.getCollection(schemaCollection) + .find().projection(new Document(TABLE_NAME_KEY, true)).iterator(); + + HashSet names = new HashSet<>(); + while (cursor.hasNext()) { + names.add((cursor.next()).getString(TABLE_NAME_KEY)); + } + + return names; + } + + private void createTableMetadata(SchemaTableName schemaTableName, List columns) + throws TableNotFoundException + { + String schemaName = schemaTableName.getSchemaName(); + String tableName = schemaTableName.getTableName(); + + MongoDatabase db = client.getDatabase(schemaName); + Document metadata = new Document(TABLE_NAME_KEY, tableName); + + ArrayList fields = new ArrayList<>(); + if (!columns.stream().anyMatch(c -> c.getName().equals("_id"))) { + fields.add(new MongoColumnHandle("_id", OBJECT_ID, true).getDocument()); + } + + fields.addAll(columns.stream() + .map(MongoColumnHandle::getDocument) + .collect(toList())); + + metadata.append(FIELDS_KEY, fields); + + MongoCollection schema = db.getCollection(schemaCollection); + schema.createIndex(new Document(TABLE_NAME_KEY, 1), new IndexOptions().unique(true)); + schema.insertOne(metadata); + } + + private boolean deleteTableMetadata(SchemaTableName schemaTableName) + { + String schemaName = toRemoteSchemaName(schemaTableName.getSchemaName()); + String tableName = toRemoteTableName(schemaName, schemaTableName.getTableName()); + + MongoDatabase db = client.getDatabase(schemaName); + if (!collectionExists(db, tableName) && + db.getCollection(schemaCollection).find(new Document(TABLE_NAME_KEY, tableName)).first().isEmpty()) { + return false; + } + + DeleteResult result = db.getCollection(schemaCollection) + .deleteOne(new Document(TABLE_NAME_KEY, tableName)); + + return result.getDeletedCount() == 1; + } + + private List guessTableFields(String schemaName, String tableName) + { + MongoDatabase db = client.getDatabase(schemaName); + Document doc = db.getCollection(tableName).find().first(); + if (doc == null) { + // no records at the collection + return ImmutableList.of(); + } + + ImmutableList.Builder builder = ImmutableList.builder(); + + for (String key : doc.keySet()) { + Object value = doc.get(key); + Optional fieldType = guessFieldType(value); + if (fieldType.isPresent()) { + Document metadata = new Document(); + metadata.append(FIELDS_NAME_KEY, key); + metadata.append(FIELDS_TYPE_KEY, fieldType.get().toString()); + metadata.append(FIELDS_HIDDEN_KEY, + key.equals("_id") && fieldType.get().equals(OBJECT_ID.getTypeSignature())); + + builder.add(metadata); + } + else { + log.debug("Unable to guess field type from %s : %s", value == null ? "null" : value.getClass().getName(), value); + } + } + + return builder.build(); + } + + private Optional guessFieldType(Object value) + { + if (value == null) { + return Optional.empty(); + } + + TypeSignature typeSignature = null; + if (value instanceof String) { + typeSignature = createUnboundedVarcharType().getTypeSignature(); + } + else if (value instanceof Integer || value instanceof Long) { + typeSignature = BIGINT.getTypeSignature(); + } + else if (value instanceof Boolean) { + typeSignature = BOOLEAN.getTypeSignature(); + } + else if (value instanceof Float || value instanceof Double) { + typeSignature = DOUBLE.getTypeSignature(); + } + else if (value instanceof Date) { + typeSignature = TIMESTAMP.getTypeSignature(); + } + else if (value instanceof ObjectId) { + typeSignature = OBJECT_ID.getTypeSignature(); + } + else if (value instanceof List) { + List> subTypes = ((List) value).stream() + .map(this::guessFieldType) + .collect(toList()); + + if (subTypes.isEmpty() || subTypes.stream().anyMatch(Optional::isPresent)) { + return Optional.empty(); + } + + Set signatures = subTypes.stream().map(Optional::get).collect(toSet()); + if (signatures.size() == 1) { + typeSignature = new TypeSignature(StandardTypes.ARRAY, signatures.stream() + .map(TypeSignatureParameter::of) + .collect(Collectors.toList())); + } + else { + // TODO: presto cli doesn't handle empty field name row type yet + typeSignature = new TypeSignature(StandardTypes.ROW, + IntStream.range(0, subTypes.size()) + .mapToObj(idx -> TypeSignatureParameter.of( + new NamedTypeSignature(Optional.of(new RowFieldName(format("%s%d", implicitPrefix, idx + 1), false)), subTypes.get(idx).get()))) + .collect(toList())); + } + } + else if (value instanceof Document) { + List parameters = new ArrayList<>(); + + for (String key : ((Document) value).keySet()) { + Optional fieldType = guessFieldType(((Document) value).get(key)); + if (fieldType.isPresent()) { + parameters.add(TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName(key, false)), fieldType.get()))); + } + } + if (!parameters.isEmpty()) { + typeSignature = new TypeSignature(StandardTypes.ROW, parameters); + } + } + + return Optional.ofNullable(typeSignature); + } + + private String toRemoteSchemaName(String schemaName) + { + verify(schemaName.equals(schemaName.toLowerCase(ENGLISH)), "schemaName not in lower-case: %s", schemaName); + if (!caseInsensitiveNameMatching) { + return schemaName; + } + for (String remoteSchemaName : client.listDatabaseNames()) { + if (schemaName.equals(remoteSchemaName.toLowerCase(ENGLISH))) { + return remoteSchemaName; + } + } + return schemaName; + } + + private String toRemoteTableName(String schemaName, String tableName) + { + verify(tableName.equals(tableName.toLowerCase(ENGLISH)), "tableName not in lower-case: %s", tableName); + if (!caseInsensitiveNameMatching) { + return tableName; + } + for (String remoteTableName : client.getDatabase(schemaName).listCollectionNames()) { + if (tableName.equals(remoteTableName.toLowerCase(ENGLISH))) { + return remoteTableName; + } + } + return tableName; + } + + private boolean isView(SchemaTableName tableName) + { + Document listCollectionsCommand = new Document(new ImmutableMap.Builder() + .put("listCollections", 1.0) + .put("filter", documentOf("name", tableName.getTableName())) + .put("nameOnly", true) + .build()); + Document cursor = client.getDatabase(tableName.getSchemaName()).runCommand(listCollectionsCommand).get("cursor", Document.class); + List firstBatch = cursor.get("firstBatch", List.class); + if (firstBatch.isEmpty()) { + return false; + } + String type = firstBatch.get(0).getString("type"); + return "view".equals(type); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplit.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplit.java new file mode 100644 index 000000000..b67ba0fa5 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplit.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.HostAddress; +import io.prestosql.spi.connector.ConnectorSplit; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class MongoSplit + implements ConnectorSplit +{ + private final List addresses; + + @JsonCreator + public MongoSplit(@JsonProperty("addresses") List addresses) + { + this.addresses = ImmutableList.copyOf(requireNonNull(addresses, "addresses is null")); + } + + @Override + public boolean isRemotelyAccessible() + { + return true; + } + + @Override + @JsonProperty + public List getAddresses() + { + return addresses; + } + + @Override + public Object getInfo() + { + return this; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplitManager.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplitManager.java new file mode 100644 index 000000000..e810c35f3 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoSplitManager.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import io.prestosql.spi.HostAddress; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplitManager; +import io.prestosql.spi.connector.ConnectorSplitSource; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTransactionHandle; +import io.prestosql.spi.connector.FixedSplitSource; + +import javax.inject.Inject; + +import java.util.List; + +import static io.prestosql.spi.HostAddress.fromParts; +import static java.util.stream.Collectors.toList; + +public class MongoSplitManager + implements ConnectorSplitManager +{ + private final List addresses; + + @Inject + public MongoSplitManager(MongoClientConfig config) + { + this.addresses = config.getSeeds().stream() + .map(s -> fromParts(s.getHost(), s.getPort())) + .collect(toList()); + } + + @Override + public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy) + { + MongoSplit split = new MongoSplit(addresses); + + return new FixedSplitSource(ImmutableList.of(split)); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTable.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTable.java new file mode 100644 index 000000000..7aef26c7a --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTable.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public class MongoTable +{ + private final MongoTableHandle tableHandle; + private final List columns; + private final List indexes; + + public MongoTable(MongoTableHandle tableHandle, List columns, List indexes) + { + this.tableHandle = tableHandle; + this.columns = ImmutableList.copyOf(columns); + this.indexes = ImmutableList.copyOf(indexes); + } + + public MongoTableHandle getTableHandle() + { + return tableHandle; + } + + public List getColumns() + { + return columns; + } + + public List getIndexes() + { + return indexes; + } + + @Override + public int hashCode() + { + return tableHandle.hashCode(); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (!(obj instanceof MongoTable)) { + return false; + } + MongoTable that = (MongoTable) obj; + return this.tableHandle.equals(that.tableHandle); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("tableHandle", tableHandle) + .toString(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTableHandle.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTableHandle.java new file mode 100644 index 000000000..4bf77d011 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTableHandle.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.predicate.TupleDomain; + +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class MongoTableHandle + implements ConnectorTableHandle +{ + private final SchemaTableName schemaTableName; + private final TupleDomain constraint; + + public MongoTableHandle(SchemaTableName schemaTableName) + { + this(schemaTableName, TupleDomain.all()); + } + + @JsonCreator + public MongoTableHandle( + @JsonProperty("schemaTableName") SchemaTableName schemaTableName, + @JsonProperty("constraint") TupleDomain constraint) + { + this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); + this.constraint = requireNonNull(constraint, "constraint is null"); + } + + @JsonProperty + public SchemaTableName getSchemaTableName() + { + return schemaTableName; + } + + @JsonProperty + public TupleDomain getConstraint() + { + return constraint; + } + + @Override + public int hashCode() + { + return Objects.hash(schemaTableName, constraint); + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + MongoTableHandle other = (MongoTableHandle) obj; + return Objects.equals(this.schemaTableName, other.schemaTableName) && + Objects.equals(this.constraint, other.constraint); + } + + @Override + public String toString() + { + return schemaTableName.toString(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTransactionHandle.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTransactionHandle.java new file mode 100644 index 000000000..132470967 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/MongoTransactionHandle.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.connector.ConnectorTransactionHandle; + +import java.util.Objects; +import java.util.UUID; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class MongoTransactionHandle + implements ConnectorTransactionHandle +{ + private final UUID uuid; + + public MongoTransactionHandle() + { + this(UUID.randomUUID()); + } + + @JsonCreator + public MongoTransactionHandle(@JsonProperty("uuid") UUID uuid) + { + this.uuid = requireNonNull(uuid, "uuid is null"); + } + + @JsonProperty + public UUID getUuid() + { + return uuid; + } + + @Override + public boolean equals(Object obj) + { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + MongoTransactionHandle other = (MongoTransactionHandle) obj; + return Objects.equals(uuid, other.uuid); + } + + @Override + public int hashCode() + { + return Objects.hash(uuid); + } + + @Override + public String toString() + { + return toStringHelper(this) + .add("uuid", uuid) + .toString(); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdFunctions.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdFunctions.java new file mode 100644 index 000000000..af3860d4a --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdFunctions.java @@ -0,0 +1,178 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.base.CharMatcher; +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.airlift.slice.XxHash64; +import io.prestosql.spi.function.Description; +import io.prestosql.spi.function.IsNull; +import io.prestosql.spi.function.LiteralParameter; +import io.prestosql.spi.function.LiteralParameters; +import io.prestosql.spi.function.ScalarFunction; +import io.prestosql.spi.function.ScalarOperator; +import io.prestosql.spi.function.SqlNullable; +import io.prestosql.spi.function.SqlType; +import io.prestosql.spi.type.StandardTypes; +import org.bson.types.ObjectId; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.spi.function.OperatorType.BETWEEN; +import static io.prestosql.spi.function.OperatorType.CAST; +import static io.prestosql.spi.function.OperatorType.EQUAL; +import static io.prestosql.spi.function.OperatorType.GREATER_THAN; +import static io.prestosql.spi.function.OperatorType.GREATER_THAN_OR_EQUAL; +import static io.prestosql.spi.function.OperatorType.HASH_CODE; +import static io.prestosql.spi.function.OperatorType.INDETERMINATE; +import static io.prestosql.spi.function.OperatorType.IS_DISTINCT_FROM; +import static io.prestosql.spi.function.OperatorType.LESS_THAN; +import static io.prestosql.spi.function.OperatorType.LESS_THAN_OR_EQUAL; +import static io.prestosql.spi.function.OperatorType.NOT_EQUAL; +import static io.prestosql.spi.function.OperatorType.XX_HASH_64; +import static io.prestosql.spi.type.DateTimeEncoding.packDateTimeWithZone; +import static io.prestosql.spi.type.TimeZoneKey.UTC_KEY; +import static java.lang.Math.toIntExact; +import static java.util.concurrent.TimeUnit.SECONDS; + +public final class ObjectIdFunctions +{ + private ObjectIdFunctions() {} + + @Description("Mongodb ObjectId") + @ScalarFunction + @SqlType("ObjectId") + public static Slice objectid() + { + return Slices.wrappedBuffer(new ObjectId().toByteArray()); + } + + @Description("Mongodb ObjectId from the given string") + @ScalarFunction + @SqlType("ObjectId") + public static Slice objectid(@SqlType(StandardTypes.VARCHAR) Slice value) + { + return Slices.wrappedBuffer(new ObjectId(CharMatcher.is(' ').removeFrom(value.toStringUtf8())).toByteArray()); + } + + @ScalarFunction + @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) // ObjectId's timestamp is a point in time + public static long objectidTimestamp(@SqlType("ObjectId") Slice value) + { + int epochSeconds = new ObjectId(value.getBytes()).getTimestamp(); + return packDateTimeWithZone(SECONDS.toMillis(epochSeconds), UTC_KEY); + } + + @ScalarOperator(CAST) + @LiteralParameters("x") + @SqlType("varchar(x)") + public static Slice castToVarchar(@LiteralParameter("x") long x, @SqlType("ObjectId") Slice value) + { + String hexString = new ObjectId(value.getBytes()).toString(); + if (hexString.length() > x) { + hexString = hexString.substring(0, toIntExact(x)); + } + return utf8Slice(hexString); + } + + @ScalarOperator(EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean equal(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return left.equals(right); + } + + @ScalarOperator(IS_DISTINCT_FROM) + @SqlType(StandardTypes.BOOLEAN) + public static boolean isDistinctFrom(@SqlType("ObjectId") Slice left, @IsNull boolean leftNull, @SqlType("ObjectId") Slice right, @IsNull boolean rightNull) + { + if (leftNull != rightNull) { + return true; + } + if (leftNull) { + return false; + } + return notEqual(left, right); + } + + @ScalarOperator(NOT_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + @SqlNullable + public static Boolean notEqual(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return !left.equals(right); + } + + @ScalarOperator(GREATER_THAN) + @SqlType(StandardTypes.BOOLEAN) + public static boolean greaterThan(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return compareTo(left, right) > 0; + } + + @ScalarOperator(GREATER_THAN_OR_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + public static boolean greaterThanOrEqual(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return compareTo(left, right) >= 0; + } + + @ScalarOperator(LESS_THAN) + @SqlType(StandardTypes.BOOLEAN) + public static boolean lessThan(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return compareTo(left, right) < 0; + } + + @ScalarOperator(LESS_THAN_OR_EQUAL) + @SqlType(StandardTypes.BOOLEAN) + public static boolean lessThanOrEqual(@SqlType("ObjectId") Slice left, @SqlType("ObjectId") Slice right) + { + return compareTo(left, right) <= 0; + } + + @ScalarOperator(BETWEEN) + @SqlType(StandardTypes.BOOLEAN) + public static boolean between(@SqlType("ObjectId") Slice value, @SqlType("ObjectId") Slice min, @SqlType("ObjectId") Slice max) + { + return compareTo(value, min) >= 0 && compareTo(value, max) <= 0; + } + + @ScalarOperator(HASH_CODE) + @SqlType(StandardTypes.BIGINT) + public static long hashCode(@SqlType("ObjectId") Slice value) + { + return new ObjectId(value.getBytes()).hashCode(); + } + + private static int compareTo(Slice left, Slice right) + { + return new ObjectId(left.getBytes()).compareTo(new ObjectId(right.getBytes())); + } + + @ScalarOperator(INDETERMINATE) + @SqlType(StandardTypes.BOOLEAN) + public static boolean indeterminate(@SqlType("ObjectId") Slice value, @IsNull boolean isNull) + { + return isNull; + } + + @ScalarOperator(XX_HASH_64) + @SqlType(StandardTypes.BIGINT) + public static long xxHash64(@SqlType("ObjectId") Slice value) + { + return XxHash64.hash(value); + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdType.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdType.java new file mode 100644 index 000000000..2ce8cc865 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ObjectIdType.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.airlift.slice.Slice; +import io.prestosql.spi.block.Block; +import io.prestosql.spi.block.BlockBuilder; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.type.AbstractVariableWidthType; +import io.prestosql.spi.type.SqlVarbinary; +import io.prestosql.spi.type.TypeSignature; +import org.bson.types.ObjectId; + +import java.io.IOException; + +public class ObjectIdType + extends AbstractVariableWidthType +{ + public static final ObjectIdType OBJECT_ID = new ObjectIdType(); + + @JsonCreator + public ObjectIdType() + { + super(new TypeSignature("ObjectId"), Slice.class); + } + + @Override + public boolean isComparable() + { + return true; + } + + @Override + public boolean isOrderable() + { + return true; + } + + @Override + public Object getObjectValue(ConnectorSession session, Block block, int position) + { + if (block.isNull(position)) { + return null; + } + + // TODO: There's no way to represent string value of a custom type + return new SqlVarbinary(block.getSlice(position, 0, block.getSliceLength(position)).getBytes()); + } + + @Override + public boolean equalTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) + { + int leftLength = leftBlock.getSliceLength(leftPosition); + int rightLength = rightBlock.getSliceLength(rightPosition); + if (leftLength != rightLength) { + return false; + } + return leftBlock.equals(leftPosition, 0, rightBlock, rightPosition, 0, leftLength); + } + + @Override + public long hash(Block block, int position) + { + return block.hash(position, 0, block.getSliceLength(position)); + } + + @Override + public int compareTo(Block leftBlock, int leftPosition, Block rightBlock, int rightPosition) + { + int leftLength = leftBlock.getSliceLength(leftPosition); + int rightLength = rightBlock.getSliceLength(rightPosition); + return leftBlock.compareTo(leftPosition, 0, leftLength, rightBlock, rightPosition, 0, rightLength); + } + + @Override + public void appendTo(Block block, int position, BlockBuilder blockBuilder) + { + if (block.isNull(position)) { + blockBuilder.appendNull(); + } + else { + block.writeBytesTo(position, 0, block.getSliceLength(position), blockBuilder); + blockBuilder.closeEntry(); + } + } + + @Override + public Slice getSlice(Block block, int position) + { + return block.getSlice(position, 0, block.getSliceLength(position)); + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value) + { + writeSlice(blockBuilder, value, 0, value.length()); + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int length) + { + blockBuilder.writeBytes(value, offset, length).closeEntry(); + } + + public static class ObjectIdSerializer + extends JsonSerializer + { + @Override + public void serialize(ObjectId objectId, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException + { + jsonGenerator.writeString(objectId.toString()); + } + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ReadPreferenceType.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ReadPreferenceType.java new file mode 100644 index 000000000..6dbbcc14f --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/ReadPreferenceType.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.mongodb.ReadPreference; + +import static java.util.Objects.requireNonNull; + +public enum ReadPreferenceType +{ + PRIMARY(ReadPreference.primary()), + PRIMARY_PREFERRED(ReadPreference.primaryPreferred()), + SECONDARY(ReadPreference.secondary()), + SECONDARY_PREFERRED(ReadPreference.secondaryPreferred()), + NEAREST(ReadPreference.nearest()); + + private final ReadPreference readPreference; + + ReadPreferenceType(ReadPreference readPreference) + { + this.readPreference = requireNonNull(readPreference, "readPreference is null"); + } + + public ReadPreference getReadPreference() + { + return readPreference; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/TypeUtils.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/TypeUtils.java new file mode 100644 index 000000000..4302a4a0c --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/TypeUtils.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import io.prestosql.spi.type.ArrayType; +import io.prestosql.spi.type.MapType; +import io.prestosql.spi.type.RowType; +import io.prestosql.spi.type.Type; + +public final class TypeUtils +{ + private TypeUtils() {} + + public static boolean isArrayType(Type type) + { + return type instanceof ArrayType; + } + + public static boolean isMapType(Type type) + { + return type instanceof MapType; + } + + public static boolean isRowType(Type type) + { + return type instanceof RowType; + } +} diff --git a/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/WriteConcernType.java b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/WriteConcernType.java new file mode 100644 index 000000000..751ea9e51 --- /dev/null +++ b/hetu-mongodb/src/main/java/io/hetu/core/plugin/mongodb/WriteConcernType.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.mongodb.WriteConcern; + +import static java.util.Objects.requireNonNull; + +public enum WriteConcernType +{ + ACKNOWLEDGED(WriteConcern.ACKNOWLEDGED), + FSYNC_SAFE(WriteConcern.FSYNC_SAFE), + FSYNCED(WriteConcern.FSYNCED), + JOURNAL_SAFEY(WriteConcern.JOURNAL_SAFE), + JOURNALED(WriteConcern.JOURNALED), + MAJORITY(WriteConcern.MAJORITY), + NORMAL(WriteConcern.NORMAL), + REPLICA_ACKNOWLEDGED(WriteConcern.REPLICA_ACKNOWLEDGED), + REPLICAS_SAFE(WriteConcern.REPLICAS_SAFE), + SAFE(WriteConcern.SAFE), + UNACKNOWLEDGED(WriteConcern.UNACKNOWLEDGED); + + private final WriteConcern writeConcern; + + WriteConcernType(WriteConcern writeConcern) + { + this.writeConcern = requireNonNull(writeConcern, "writeConcern is null"); + } + + public WriteConcern getWriteConcern() + { + return writeConcern; + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoQueryRunner.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoQueryRunner.java new file mode 100644 index 000000000..e5ee0edbe --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoQueryRunner.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.mongodb.MongoClient; +import io.airlift.log.Logger; +import io.airlift.log.Logging; +import io.airlift.tpch.TpchTable; +import io.prestosql.Session; +import io.prestosql.plugin.tpch.TpchPlugin; +import io.prestosql.tests.DistributedQueryRunner; + +import java.util.Map; + +import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.prestosql.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.prestosql.testing.TestingSession.testSessionBuilder; +import static io.prestosql.tests.QueryAssertions.copyTpchTables; + +public final class MongoQueryRunner +{ + private static final String TPCH_SCHEMA = "tpch"; + + private MongoQueryRunner() {} + + public static DistributedQueryRunner createMongoQueryRunner(MongoServer server, TpchTable... tables) + throws Exception + { + return createMongoQueryRunner(server, ImmutableMap.of(), ImmutableList.copyOf(tables)); + } + + public static DistributedQueryRunner createMongoQueryRunner(MongoServer server, Map extraProperties, Iterable> tables) + throws Exception + { + DistributedQueryRunner queryRunner = null; + try { + queryRunner = DistributedQueryRunner.builder(createSession()) + .setExtraProperties(extraProperties) + .build(); + + queryRunner.installPlugin(new TpchPlugin()); + queryRunner.createCatalog("tpch", "tpch"); + + Map properties = ImmutableMap.of( + "mongodb.case-insensitive-name-matching", "true", + "mongodb.seeds", server.getAddress().toString(), + "mongodb.socket-keep-alive", "true"); + + queryRunner.installPlugin(new MongoPlugin()); + queryRunner.createCatalog("mongodb", "mongodb", properties); + + copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables); + return queryRunner; + } + catch (Throwable e) { + closeAllSuppress(e, queryRunner); + throw e; + } + } + + public static Session createSession() + { + return testSessionBuilder() + .setCatalog("mongodb") + .setSchema(TPCH_SCHEMA) + .build(); + } + + public static MongoClient createMongoClient(MongoServer server) + { + return new MongoClient(server.getAddress().getHost(), server.getAddress().getPort()); + } + + public static void main(String[] args) + throws Exception + { + Logging.initialize(); + DistributedQueryRunner queryRunner = createMongoQueryRunner( + new MongoServer(), + ImmutableMap.of("http-server.http.port", "8080"), + TpchTable.getTables()); + Thread.sleep(10); + Logger log = Logger.get(MongoQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoServer.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoServer.java new file mode 100644 index 000000000..a8c79b489 --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/MongoServer.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.net.HostAndPort; +import org.testcontainers.containers.MongoDBContainer; + +import java.io.Closeable; + +public class MongoServer + implements Closeable +{ + private static final int MONGO_PORT = 27017; + + private final MongoDBContainer dockerContainer; + + public MongoServer() + { + this.dockerContainer = new MongoDBContainer("mongo:3.4.0") + .withEnv("MONGO_INITDB_DATABASE", "tpch") + .withCommand("--bind_ip 0.0.0.0"); + this.dockerContainer.start(); + } + + public HostAndPort getAddress() + { + return HostAndPort.fromParts(dockerContainer.getContainerIpAddress(), dockerContainer.getMappedPort(MONGO_PORT)); + } + + @Override + public void close() + { + dockerContainer.close(); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoClientConfig.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoClientConfig.java new file mode 100644 index 000000000..8d198544f --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoClientConfig.java @@ -0,0 +1,107 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableMap; +import com.mongodb.MongoCredential; +import org.testng.annotations.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static org.testng.Assert.assertEquals; + +public class TestMongoClientConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(MongoClientConfig.class) + .setSchemaCollection("_schema") + .setCaseInsensitiveNameMatching(false) + .setSeeds("") + .setCredentials("") + .setMinConnectionsPerHost(0) + .setConnectionsPerHost(100) + .setMaxWaitTime(120_000) + .setConnectionTimeout(10_000) + .setSocketTimeout(0) + .setSocketKeepAlive(false) + .setSslEnabled(false) + .setMaxConnectionIdleTime(0) + .setCursorBatchSize(0) + .setReadPreference(ReadPreferenceType.PRIMARY) + .setWriteConcern(WriteConcernType.ACKNOWLEDGED) + .setRequiredReplicaSetName(null) + .setImplicitRowFieldPrefix("_pos")); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = new ImmutableMap.Builder() + .put("mongodb.schema-collection", "_my_schema") + .put("mongodb.case-insensitive-name-matching", "true") + .put("mongodb.seeds", "host1,host2:27016") + .put("mongodb.credentials", "username:password@collection") + .put("mongodb.min-connections-per-host", "1") + .put("mongodb.connections-per-host", "99") + .put("mongodb.max-wait-time", "120001") + .put("mongodb.connection-timeout", "9999") + .put("mongodb.socket-timeout", "1") + .put("mongodb.socket-keep-alive", "true") + .put("mongodb.ssl.enabled", "true") + .put("mongodb.max-connection-idle-time", "180000") + .put("mongodb.cursor-batch-size", "1") + .put("mongodb.read-preference", "NEAREST") + .put("mongodb.write-concern", "UNACKNOWLEDGED") + .put("mongodb.required-replica-set", "replica_set") + .put("mongodb.implicit-row-field-prefix", "_prefix") + .build(); + + MongoClientConfig expected = new MongoClientConfig() + .setSchemaCollection("_my_schema") + .setCaseInsensitiveNameMatching(true) + .setSeeds("host1", "host2:27016") + .setCredentials("username:password@collection") + .setMinConnectionsPerHost(1) + .setConnectionsPerHost(99) + .setMaxWaitTime(120_001) + .setConnectionTimeout(9_999) + .setSocketTimeout(1) + .setSocketKeepAlive(true) + .setSslEnabled(true) + .setMaxConnectionIdleTime(180_000) + .setCursorBatchSize(1) + .setReadPreference(ReadPreferenceType.NEAREST) + .setWriteConcern(WriteConcernType.UNACKNOWLEDGED) + .setRequiredReplicaSetName("replica_set") + .setImplicitRowFieldPrefix("_prefix"); + + assertFullMapping(properties, expected); + } + + @Test + public void testSpecialCharacterCredential() + { + MongoClientConfig config = new MongoClientConfig() + .setCredentials("username:P@ss:w0rd@database"); + + MongoCredential credential = config.getCredentials().get(0); + MongoCredential expected = MongoCredential.createCredential("username", "database", "P@ss:w0rd".toCharArray()); + assertEquals(credential, expected); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoDistributedQueries.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoDistributedQueries.java new file mode 100644 index 000000000..d3f046936 --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoDistributedQueries.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableMap; +import io.airlift.tpch.TpchTable; +import io.prestosql.tests.AbstractTestDistributedQueries; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import static io.hetu.core.plugin.mongodb.MongoQueryRunner.createMongoQueryRunner; + +@Test +public class TestMongoDistributedQueries + extends AbstractTestDistributedQueries +{ + private MongoServer server; + + public TestMongoDistributedQueries() + { + this(new MongoServer()); + } + + public TestMongoDistributedQueries(MongoServer mongoServer) + { + super(() -> createMongoQueryRunner(mongoServer, ImmutableMap.of(), TpchTable.getTables())); + this.server = mongoServer; + } + + @AfterClass(alwaysRun = true) + public final void destroy() + { + server.close(); + } + + @Override + protected boolean supportsViews() + { + return false; + } + + @Override + public void testRenameTable() + { + // the connector does not support renaming tables + } + + @Override + public void testRenameColumn() + { + // the connector does not support renaming columns + } + + @Override + public void testDropColumn() + { + // the connector does not support dropping columns + } + + @Override + public void testDelete() + { + // the connector does not support delete + } + + @Override + public void testCommentTable() + { + // the connector does not support comment on table + assertQueryFails("COMMENT ON TABLE orders IS 'hello'", "This connector does not support setting table comments"); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoIntegrationSmokeTest.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoIntegrationSmokeTest.java new file mode 100644 index 000000000..487236a96 --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoIntegrationSmokeTest.java @@ -0,0 +1,366 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.mongodb.MongoClient; +import com.mongodb.client.MongoCollection; +import io.prestosql.testing.MaterializedResult; +import io.prestosql.testing.MaterializedRow; +import io.prestosql.tests.AbstractTestIntegrationSmokeTest; +import org.bson.Document; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.Arrays; + +import static io.airlift.tpch.TpchTable.ORDERS; +import static io.hetu.core.plugin.mongodb.MongoQueryRunner.createMongoClient; +import static io.hetu.core.plugin.mongodb.MongoQueryRunner.createMongoQueryRunner; +import static java.lang.String.format; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; + +@Test(singleThreaded = true) +public class TestMongoIntegrationSmokeTest + extends AbstractTestIntegrationSmokeTest +{ + private MongoServer server; + private MongoClient client; + + public TestMongoIntegrationSmokeTest() + { + this(new MongoServer()); + } + + public TestMongoIntegrationSmokeTest(MongoServer mongoServer) + { + super(() -> createMongoQueryRunner(mongoServer, ORDERS)); + this.server = mongoServer; + this.client = createMongoClient(server); + } + + @AfterClass(alwaysRun = true) + public final void destroy() + { + server.close(); + client.close(); + } + + @Test + public void createTableWithEveryType() + { + String query = "" + + "CREATE TABLE test_types_table AS " + + "SELECT" + + " 'foo' _varchar" + + ", cast('bar' as varbinary) _varbinary" + + ", cast(1 as bigint) _bigint" + + ", 3.14E0 _double" + + ", true _boolean" + + ", DATE '1980-05-07' _date" + + ", TIMESTAMP '1980-05-07 11:22:33.456' _timestamp" + + ", ObjectId('ffffffffffffffffffffffff') _objectid"; + + assertUpdate(query, 1); + + MaterializedResult results = getQueryRunner().execute(getSession(), "SELECT * FROM test_types_table").toTestTypes(); + assertEquals(results.getRowCount(), 1); + MaterializedRow row = results.getMaterializedRows().get(0); + assertEquals(row.getField(0), "foo"); + assertEquals(row.getField(1), "bar".getBytes(UTF_8)); + assertEquals(row.getField(2), 1L); + assertEquals(row.getField(3), 3.14); + assertEquals(row.getField(4), true); + assertEquals(row.getField(5), LocalDate.of(1980, 5, 7)); + assertEquals(row.getField(6), LocalDateTime.of(1980, 5, 7, 11, 22, 33, 456_000_000)); + assertUpdate("DROP TABLE test_types_table"); + + assertFalse(getQueryRunner().tableExists(getSession(), "test_types_table")); + } + + @Test + public void testInsertWithEveryType() + { + String createSql = "" + + "CREATE TABLE test_insert_types_table " + + "(" + + " vc varchar" + + ", vb varbinary" + + ", bi bigint" + + ", d double" + + ", b boolean" + + ", dt date" + + ", ts timestamp" + + ", objid objectid" + + ")"; + getQueryRunner().execute(getSession(), createSql); + + String insertSql = "" + + "INSERT INTO test_insert_types_table " + + "SELECT" + + " 'foo' _varchar" + + ", cast('bar' as varbinary) _varbinary" + + ", cast(1 as bigint) _bigint" + + ", 3.14E0 _double" + + ", true _boolean" + + ", DATE '1980-05-07' _date" + + ", TIMESTAMP '1980-05-07 11:22:33.456' _timestamp" + + ", ObjectId('ffffffffffffffffffffffff') _objectid"; + getQueryRunner().execute(getSession(), insertSql); + + MaterializedResult results = getQueryRunner().execute(getSession(), "SELECT * FROM test_insert_types_table").toTestTypes(); + assertEquals(results.getRowCount(), 1); + MaterializedRow row = results.getMaterializedRows().get(0); + assertEquals(row.getField(0), "foo"); + assertEquals(row.getField(1), "bar".getBytes(UTF_8)); + assertEquals(row.getField(2), 1L); + assertEquals(row.getField(3), 3.14); + assertEquals(row.getField(4), true); + assertEquals(row.getField(5), LocalDate.of(1980, 5, 7)); + assertEquals(row.getField(6), LocalDateTime.of(1980, 5, 7, 11, 22, 33, 456_000_000)); + assertUpdate("DROP TABLE test_insert_types_table"); + assertFalse(getQueryRunner().tableExists(getSession(), "test_insert_types_table")); + } + + @Test + public void testArrays() + { + assertUpdate("CREATE TABLE tmp_array1 AS SELECT ARRAY[1, 2, NULL] AS col", 1); + assertQuery("SELECT col[2] FROM tmp_array1", "SELECT 2"); + assertQuery("SELECT col[3] FROM tmp_array1", "SELECT NULL"); + + assertUpdate("CREATE TABLE tmp_array2 AS SELECT ARRAY[1.0E0, 2.5E0, 3.5E0] AS col", 1); + assertQuery("SELECT col[2] FROM tmp_array2", "SELECT 2.5"); + + assertUpdate("CREATE TABLE tmp_array3 AS SELECT ARRAY['puppies', 'kittens', NULL] AS col", 1); + assertQuery("SELECT col[2] FROM tmp_array3", "SELECT 'kittens'"); + assertQuery("SELECT col[3] FROM tmp_array3", "SELECT NULL"); + + assertUpdate("CREATE TABLE tmp_array4 AS SELECT ARRAY[TRUE, NULL] AS col", 1); + assertQuery("SELECT col[1] FROM tmp_array4", "SELECT TRUE"); + assertQuery("SELECT col[2] FROM tmp_array4", "SELECT NULL"); + + assertUpdate("CREATE TABLE tmp_array5 AS SELECT ARRAY[ARRAY[1, 2], NULL, ARRAY[3, 4]] AS col", 1); + assertQuery("SELECT col[1][2] FROM tmp_array5", "SELECT 2"); + + assertUpdate("CREATE TABLE tmp_array6 AS SELECT ARRAY[ARRAY['\"hi\"'], NULL, ARRAY['puppies']] AS col", 1); + assertQuery("SELECT col[1][1] FROM tmp_array6", "SELECT '\"hi\"'"); + assertQuery("SELECT col[3][1] FROM tmp_array6", "SELECT 'puppies'"); + } + + @Test + public void testTemporalArrays() + { + assertUpdate("CREATE TABLE tmp_array7 AS SELECT ARRAY[DATE '2014-09-30'] AS col", 1); + assertOneNotNullResult("SELECT col[1] FROM tmp_array7"); + assertUpdate("CREATE TABLE tmp_array8 AS SELECT ARRAY[TIMESTAMP '2001-08-22 03:04:05.321'] AS col", 1); + assertOneNotNullResult("SELECT col[1] FROM tmp_array8"); + } + + @Test + public void testSkipUnknownTypes() + { + Document document1 = new Document("col", Document.parse("{\"key1\": \"value1\", \"key2\": null}")); + client.getDatabase("test").getCollection("tmp_guess_schema1").insertOne(document1); + assertQuery("SHOW COLUMNS FROM test.tmp_guess_schema1", "SELECT 'col', 'row(key1 varchar)', '', ''"); + assertQuery("SELECT col.key1 FROM test.tmp_guess_schema1", "SELECT 'value1'"); + + Document document2 = new Document("col", new Document("key1", null)); + client.getDatabase("test").getCollection("tmp_guess_schema2").insertOne(document2); + assertQueryReturnsEmptyResult("SHOW COLUMNS FROM test.tmp_guess_schema2"); + } + + @Test + public void testMaps() + { + assertUpdate("CREATE TABLE tmp_map1 AS SELECT MAP(ARRAY[0,1], ARRAY[2,NULL]) AS col", 1); + assertQuery("SELECT col[0] FROM tmp_map1", "SELECT 2"); + assertQuery("SELECT col[1] FROM tmp_map1", "SELECT NULL"); + + assertUpdate("CREATE TABLE tmp_map2 AS SELECT MAP(ARRAY[1.0E0], ARRAY[2.5E0]) AS col", 1); + assertQuery("SELECT col[1.0] FROM tmp_map2", "SELECT 2.5"); + + assertUpdate("CREATE TABLE tmp_map3 AS SELECT MAP(ARRAY['puppies'], ARRAY['kittens']) AS col", 1); + assertQuery("SELECT col['puppies'] FROM tmp_map3", "SELECT 'kittens'"); + + assertUpdate("CREATE TABLE tmp_map4 AS SELECT MAP(ARRAY[TRUE], ARRAY[FALSE]) AS col", "SELECT 1"); + assertQuery("SELECT col[TRUE] FROM tmp_map4", "SELECT FALSE"); + + assertUpdate("CREATE TABLE tmp_map5 AS SELECT MAP(ARRAY[1.0E0], ARRAY[ARRAY[1, 2]]) AS col", 1); + assertQuery("SELECT col[1.0][2] FROM tmp_map5", "SELECT 2"); + + assertUpdate("CREATE TABLE tmp_map6 AS SELECT MAP(ARRAY[DATE '2014-09-30'], ARRAY[DATE '2014-09-29']) AS col", 1); + assertOneNotNullResult("SELECT col[DATE '2014-09-30'] FROM tmp_map6"); + assertUpdate("CREATE TABLE tmp_map7 AS SELECT MAP(ARRAY[TIMESTAMP '2001-08-22 03:04:05.321'], ARRAY[TIMESTAMP '2001-08-22 03:04:05.321']) AS col", 1); + assertOneNotNullResult("SELECT col[TIMESTAMP '2001-08-22 03:04:05.321'] FROM tmp_map7"); + + assertUpdate("CREATE TABLE test.tmp_map8 (col MAP)"); + client.getDatabase("test").getCollection("tmp_map8").insertOne(new Document( + ImmutableMap.of("col", new Document(ImmutableMap.of("key1", "value1", "key2", "value2"))))); + assertQuery("SELECT col['key1'] FROM test.tmp_map8", "SELECT 'value1'"); + + assertUpdate("CREATE TABLE test.tmp_map9 (col VARCHAR)"); + client.getDatabase("test").getCollection("tmp_map9").insertOne(new Document( + ImmutableMap.of("col", new Document(ImmutableMap.of("key1", "value1", "key2", "value2"))))); + assertQuery("SELECT col FROM test.tmp_map9", "SELECT '{ \"key1\" : \"value1\", \"key2\" : \"value2\" }'"); + + assertUpdate("CREATE TABLE test.tmp_map10 (col VARCHAR)"); + client.getDatabase("test").getCollection("tmp_map10").insertOne(new Document( + ImmutableMap.of("col", ImmutableList.of(new Document(ImmutableMap.of("key1", "value1", "key2", "value2")), + new Document(ImmutableMap.of("key3", "value3", "key4", "value4")))))); + assertQuery("SELECT col FROM test.tmp_map10", "SELECT '[{ \"key1\" : \"value1\", \"key2\" : \"value2\" }, { \"key3\" : \"value3\", \"key4\" : \"value4\" }]'"); + + assertUpdate("CREATE TABLE test.tmp_map11 (col VARCHAR)"); + client.getDatabase("test").getCollection("tmp_map11").insertOne(new Document( + ImmutableMap.of("col", 10))); + assertQuery("SELECT col FROM test.tmp_map11", "SELECT '10'"); + + assertUpdate("CREATE TABLE test.tmp_map12 (col VARCHAR)"); + client.getDatabase("test").getCollection("tmp_map12").insertOne(new Document( + ImmutableMap.of("col", Arrays.asList(10, null, 11)))); + assertQuery("SELECT col FROM test.tmp_map12", "SELECT '[10, null, 11]'"); + } + + @Test + public void testCollectionNameContainsDots() + { + assertUpdate("CREATE TABLE \"tmp.dot1\" AS SELECT 'foo' _varchar", 1); + assertQuery("SELECT _varchar FROM \"tmp.dot1\"", "SELECT 'foo'"); + assertUpdate("DROP TABLE \"tmp.dot1\""); + } + + @Test + public void testObjectIds() + { + String values = "VALUES " + + " (10, NULL, NULL)," + + " (11, ObjectId('ffffffffffffffffffffffff'), ObjectId('ffffffffffffffffffffffff'))," + + " (12, ObjectId('ffffffffffffffffffffffff'), ObjectId('aaaaaaaaaaaaaaaaaaaaaaaa'))," + + " (13, ObjectId('000000000000000000000000'), ObjectId('000000000000000000000000'))," + + " (14, ObjectId('ffffffffffffffffffffffff'), NULL)," + + " (15, NULL, ObjectId('ffffffffffffffffffffffff'))"; + String inlineTable = format("(%s) AS t(i, one, two)", values); + + assertUpdate("DROP TABLE IF EXISTS tmp_objectid"); + assertUpdate("CREATE TABLE tmp_objectid AS SELECT * FROM " + inlineTable, 6); + + // IS NULL + assertQuery("SELECT i FROM " + inlineTable + " WHERE one IS NULL", "VALUES 10, 15"); + assertQuery("SELECT i FROM tmp_objectid WHERE one IS NULL", "SELECT 0 WHERE false"); // NULL gets replaced with new unique ObjectId in MongoPageSink, this affects other test cases + + // CAST AS varchar + assertQuery( + "SELECT i, CAST(one AS varchar) FROM " + inlineTable + " WHERE i <= 13", + "VALUES (10, NULL), (11, 'ffffffffffffffffffffffff'), (12, 'ffffffffffffffffffffffff'), (13, '000000000000000000000000')"); + + // EQUAL + assertQuery("SELECT i FROM tmp_objectid WHERE one = two", "VALUES 11, 13"); + assertQuery("SELECT i FROM tmp_objectid WHERE one = ObjectId('ffffffffffffffffffffffff')", "VALUES 11, 12, 14"); + + // IS DISTINCT FROM + assertQuery("SELECT i FROM " + inlineTable + " WHERE one IS DISTINCT FROM two", "VALUES 12, 14, 15"); + assertQuery("SELECT i FROM " + inlineTable + " WHERE one IS NOT DISTINCT FROM two", "VALUES 10, 11, 13"); + + assertQuery("SELECT i FROM tmp_objectid WHERE one IS DISTINCT FROM two", "VALUES 10, 12, 14, 15"); + assertQuery("SELECT i FROM tmp_objectid WHERE one IS NOT DISTINCT FROM two", "VALUES 11, 13"); + + // Join on ObjectId + assertQuery( + format("SELECT l.i, r.i FROM (%1$s) AS l(i, one, two) JOIN (%1$s) AS r(i, one, two) ON l.one = r.two", values), + "VALUES (11, 11), (14, 11), (11, 15), (12, 15), (12, 11), (14, 15), (13, 13)"); + + // Group by ObjectId (IS DISTINCT FROM) + assertQuery("SELECT array_agg(i ORDER BY i) FROM " + inlineTable + " GROUP BY one", "VALUES ((10, 15)), ((11, 12, 14)), ((13))"); + assertQuery("SELECT i FROM " + inlineTable + " GROUP BY one, i", "VALUES 10, 11, 12, 13, 14, 15"); + + // Group by Row(ObjectId) (ID DISTINCT FROM in @OperatorDependency) + assertQuery( + "SELECT r.i, count(*) FROM (SELECT CAST(row(one, i) AS row(one ObjectId, i bigint)) r FROM " + inlineTable + ") GROUP BY r", + "VALUES (10, 1), (11, 1), (12, 1), (13, 1), (14, 1), (15, 1)"); + assertQuery( + "SELECT r.x, CAST(r.one AS varchar), count(*) FROM (SELECT CAST(row(one, i / 3 * 3) AS row(one ObjectId, x bigint)) r FROM " + inlineTable + ") GROUP BY r", + "VALUES (9, NULL, 1), (9, 'ffffffffffffffffffffffff', 1), (12, 'ffffffffffffffffffffffff', 2), (12, '000000000000000000000000', 1), (15, NULL, 1)"); + + assertUpdate("DROP TABLE tmp_objectid"); + } + + @Test + public void testCaseInsensitive() + throws Exception + { + MongoCollection collection = client.getDatabase("testCase").getCollection("testInsensitive"); + collection.insertOne(new Document(ImmutableMap.of("Name", "abc", "Value", 1))); + + assertQuery("SHOW SCHEMAS IN mongodb LIKE 'testcase'", "SELECT 'testcase'"); + assertQuery("SHOW TABLES IN testcase", "SELECT 'testinsensitive'"); + assertQuery( + "SHOW COLUMNS FROM testcase.testInsensitive", + "VALUES ('name', 'varchar', '', ''), ('value', 'bigint', '', '')"); + + assertQuery("SELECT name, value FROM testcase.testinsensitive", "SELECT 'abc', 1"); + assertUpdate("INSERT INTO testcase.testinsensitive VALUES('def', 2)", 1); + + assertQuery("SELECT value FROM testcase.testinsensitive WHERE name = 'def'", "SELECT 2"); + assertUpdate("DROP TABLE testcase.testinsensitive"); + } + + @Test + public void testSelectView() + { + assertUpdate("CREATE TABLE test.view_base AS SELECT 'foo' _varchar", 1); + client.getDatabase("test").createView("test_view", "view_base", ImmutableList.of()); + assertQuery("SELECT * FROM test.view_base", "SELECT 'foo'"); + assertUpdate("DROP TABLE test.test_view"); + assertUpdate("DROP TABLE test.view_base"); + } + + @Test + public void testDropTable() + { + assertUpdate("CREATE TABLE test.drop_table(col bigint)"); + assertUpdate("DROP TABLE test.drop_table"); + assertQueryFails("SELECT * FROM test.drop_table", ".*Table mongodb.test.drop_table does not exist"); + } + + @Test + public void testNullPredicates() + { + assertUpdate("CREATE TABLE test.null_predicates(name varchar, value integer)"); + + MongoCollection collection = client.getDatabase("test").getCollection("null_predicates"); + collection.insertOne(new Document(ImmutableMap.of("name", "abc", "value", 1))); + collection.insertOne(new Document(ImmutableMap.of("name", "abcd"))); + collection.insertOne(new Document(Document.parse("{\"name\": \"abcde\", \"value\": null}"))); + + assertQuery("SELECT count(*) FROM test.null_predicates WHERE value IS NULL OR rand() = 42", "SELECT 2"); + assertQuery("SELECT count(*) FROM test.null_predicates WHERE value IS NULL", "SELECT 2"); + assertQuery("SELECT count(*) FROM test.null_predicates WHERE value IS NOT NULL", "SELECT 1"); + + assertUpdate("DROP TABLE test.null_predicates"); + } + + private void assertOneNotNullResult(String query) + { + MaterializedResult results = getQueryRunner().execute(getSession(), query).toTestTypes(); + assertEquals(results.getRowCount(), 1); + assertEquals(results.getMaterializedRows().get(0).getFieldCount(), 1); + assertNotNull(results.getMaterializedRows().get(0).getField(0)); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoPlugin.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoPlugin.java new file mode 100644 index 000000000..822ef406c --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoPlugin.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableMap; +import io.prestosql.spi.connector.Connector; +import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.type.Type; +import io.prestosql.testing.TestingConnectorContext; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID; +import static org.testng.Assert.assertEquals; + +public class TestMongoPlugin +{ + private MongoServer server; + private String seed; + + @BeforeClass + public void start() + { + server = new MongoServer(); + seed = server.getAddress().toString(); + } + + @Test + public void testCreateConnector() + { + MongoPlugin plugin = new MongoPlugin(); + + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + Connector connector = factory.create("test", ImmutableMap.of("mongodb.seeds", seed), new TestingConnectorContext()); + + Type type = getOnlyElement(plugin.getTypes()); + assertEquals(type, OBJECT_ID); + + connector.shutdown(); + } + + @AfterClass(alwaysRun = true) + public void destroy() + { + server.close(); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSession.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSession.java new file mode 100644 index 000000000..b90005a4d --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSession.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.predicate.ValueSet; +import org.bson.Document; +import org.testng.annotations.Test; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.prestosql.spi.predicate.Range.equal; +import static io.prestosql.spi.predicate.Range.greaterThan; +import static io.prestosql.spi.predicate.Range.greaterThanOrEqual; +import static io.prestosql.spi.predicate.Range.lessThan; +import static io.prestosql.spi.predicate.Range.range; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.util.Arrays.asList; +import static org.testng.Assert.assertEquals; + +public class TestMongoSession +{ + private static final MongoColumnHandle COL1 = new MongoColumnHandle("col1", BIGINT, false); + private static final MongoColumnHandle COL2 = new MongoColumnHandle("col2", createUnboundedVarcharType(), false); + private static final MongoColumnHandle COL3 = new MongoColumnHandle("col3", createUnboundedVarcharType(), false); + + @Test + public void testBuildQuery() + { + TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + COL1, Domain.create(ValueSet.ofRanges(range(BIGINT, 100L, false, 200L, true)), false), + COL2, Domain.singleValue(createUnboundedVarcharType(), utf8Slice("a value")))); + + Document query = MongoSession.buildQuery(tupleDomain); + Document expected = new Document() + .append(COL1.getName(), new Document().append("$gt", 100L).append("$lte", 200L)) + .append(COL2.getName(), new Document("$eq", "a value")); + assertEquals(query, expected); + } + + @Test + public void testBuildQueryStringType() + { + TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + COL3, Domain.create(ValueSet.ofRanges(range(createUnboundedVarcharType(), utf8Slice("hello"), false, utf8Slice("world"), true)), false), + COL2, Domain.create(ValueSet.ofRanges(greaterThanOrEqual(createUnboundedVarcharType(), utf8Slice("a value"))), false))); + + Document query = MongoSession.buildQuery(tupleDomain); + Document expected = new Document() + .append(COL3.getName(), new Document().append("$gt", "hello").append("$lte", "world")) + .append(COL2.getName(), new Document("$gte", "a value")); + assertEquals(query, expected); + } + + @Test + public void testBuildQueryIn() + { + TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + COL2, Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("hello")), equal(createUnboundedVarcharType(), utf8Slice("world"))), false))); + + Document query = MongoSession.buildQuery(tupleDomain); + Document expected = new Document(COL2.getName(), new Document("$in", ImmutableList.of("hello", "world"))); + assertEquals(query, expected); + } + + @Test + public void testBuildQueryOr() + { + TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + COL1, Domain.create(ValueSet.ofRanges(lessThan(BIGINT, 100L), greaterThan(BIGINT, 200L)), false))); + + Document query = MongoSession.buildQuery(tupleDomain); + Document expected = new Document("$or", asList( + new Document(COL1.getName(), new Document("$lt", 100L)), + new Document(COL1.getName(), new Document("$gt", 200L)))); + assertEquals(query, expected); + } + + @Test + public void testBuildQueryNull() + { + TupleDomain tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of( + COL1, Domain.create(ValueSet.ofRanges(greaterThan(BIGINT, 200L)), true))); + + Document query = MongoSession.buildQuery(tupleDomain); + Document expected = new Document("$or", asList( + new Document(COL1.getName(), new Document("$gt", 200L)), + new Document(COL1.getName(), new Document("$eq", null)))); + assertEquals(query, expected); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSplit.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSplit.java new file mode 100644 index 000000000..14b33708b --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoSplit.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import io.airlift.json.JsonCodec; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class TestMongoSplit +{ + private final JsonCodec codec = JsonCodec.jsonCodec(MongoSplit.class); + + @Test + public void testJsonRoundTrip() + { + MongoSplit expected = new MongoSplit(ImmutableList.of()); + + String json = codec.toJson(expected); + MongoSplit actual = codec.fromJson(json); + + assertEquals(actual.getAddresses(), ImmutableList.of()); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoTableHandle.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoTableHandle.java new file mode 100644 index 000000000..8d91a2dc3 --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestMongoTableHandle.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import io.airlift.json.JsonCodec; +import io.prestosql.spi.connector.SchemaTableName; +import org.testng.annotations.Test; + +import static org.testng.Assert.assertEquals; + +public class TestMongoTableHandle +{ + private final JsonCodec codec = JsonCodec.jsonCodec(MongoTableHandle.class); + + @Test + public void testRoundTrip() + { + MongoTableHandle expected = new MongoTableHandle(new SchemaTableName("schema", "table")); + + String json = codec.toJson(expected); + MongoTableHandle actual = codec.fromJson(json); + + assertEquals(actual.getSchemaTableName(), expected.getSchemaTableName()); + } +} diff --git a/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestObjectIdFunctions.java b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestObjectIdFunctions.java new file mode 100644 index 000000000..49f411213 --- /dev/null +++ b/hetu-mongodb/src/test/java/io/hetu/core/plugin/mongodb/TestObjectIdFunctions.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.mongodb; + +import com.google.common.collect.ImmutableList; +import io.prestosql.operator.scalar.AbstractTestFunctions; +import io.prestosql.spi.type.SqlTimestampWithTimeZone; +import io.prestosql.spi.type.TimeZoneKey; +import io.prestosql.spi.type.Type; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.time.ZonedDateTime; + +import static io.prestosql.metadata.FunctionExtractor.extractFunctions; +import static io.prestosql.operator.scalar.ApplyFunction.APPLY_FUNCTION; +import static io.prestosql.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; +import static java.time.ZoneOffset.UTC; + +public class TestObjectIdFunctions + extends AbstractTestFunctions +{ + @BeforeClass + protected void registerFunctions() + { + MongoPlugin plugin = new MongoPlugin(); + for (Type type : plugin.getTypes()) { + functionAssertions.addType(type); + } + functionAssertions.getMetadata().addFunctions(extractFunctions(plugin.getFunctions())); + functionAssertions.getMetadata().addFunctions(ImmutableList.of(APPLY_FUNCTION)); + } + + @Test + public void testObjectidTimestamp() + { + assertFunction( + "objectid_timestamp(ObjectId('1234567890abcdef12345678'))", + TIMESTAMP_WITH_TIME_ZONE, + toTimestampWithTimeZone(ZonedDateTime.of(1979, 9, 5, 22, 51, 36, 0, UTC))); + } + + private SqlTimestampWithTimeZone toTimestampWithTimeZone(ZonedDateTime zonedDateTime) + { + return new SqlTimestampWithTimeZone(zonedDateTime.toInstant().toEpochMilli(), TimeZoneKey.getTimeZoneKey(zonedDateTime.getZone().getId())); + } +} diff --git a/hetu-server/src/main/provisio/hetu.xml b/hetu-server/src/main/provisio/hetu.xml index 08f3f8150..8bc1ce82d 100644 --- a/hetu-server/src/main/provisio/hetu.xml +++ b/hetu-server/src/main/provisio/hetu.xml @@ -186,13 +186,13 @@ - + --> - + - --> + diff --git a/pom.xml b/pom.xml index 6d5412a21..f1e066ba9 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,7 @@ hetu-seed-store hetu-state-store hetu-hbase + hetu-mongodb hetu-carbondata hetu-filesystem-client hetu-metastore