From 68e87e04914d593f345065d1a31712c2082360d0 Mon Sep 17 00:00:00 2001 From: Peikun Chen Date: Thu, 24 Dec 2020 02:31:02 +0800 Subject: [PATCH] Support auto connector config for webui --- .../plugin/carbondata/CarbondataPlugin.java | 21 ++++ .../plugin/datacenter/DataCenterConfig.java | 9 ++ .../plugin/datacenter/DataCenterPlugin.java | 20 ++++ hetu-docs/en/admin/web-interface.md | 76 ------------- hetu-docs/zh/admin/web-interface.md | 76 ------------- .../io/hetu/core/plugin/hana/HanaPlugin.java | 26 +++++ .../hetu/core/plugin/hbase/HBasePlugin.java | 21 ++++ .../core/plugin/hbase/conf/HBaseConfig.java | 15 +++ .../plugin/opengauss/OpenGaussPlugin.java | 21 ++++ .../hetu/core/plugin/oracle/OraclePlugin.java | 22 +++- .../io/hetu/core/plugin/vdm/VdmPlugin.java | 20 ++++ .../prestosql/plugin/jdbc/BaseJdbcConfig.java | 9 ++ .../plugin/jdbc/JdbcMetadataConfig.java | 4 + .../elasticsearch/ElasticsearchConfig.java | 20 ++++ .../elasticsearch/ElasticsearchPlugin.java | 20 ++++ .../io/prestosql/plugin/hive/HiveConfig.java | 7 ++ .../io/prestosql/plugin/hive/HivePlugin.java | 23 ++++ .../thrift/StaticMetastoreConfig.java | 5 + .../io/prestosql/plugin/jmx/JmxPlugin.java | 18 +++ .../plugin/kafka/KafkaConnectorConfig.java | 12 ++ .../prestosql/plugin/kafka/KafkaPlugin.java | 18 +++ .../plugin/localfile/LocalFilePlugin.java | 18 +++ .../etc/connector_properties.json.template | 103 ----------------- .../queryeditorui/QueryEditorConfig.java | 13 --- .../queryeditorui/QueryEditorUIModule.java | 7 -- .../queryeditorui/protocol/Connector.java | 14 +-- .../resources/ConnectorResource.java | 17 +-- .../store/connectors/ConnectorCache.java | 61 ++++------ .../io/prestosql/server/PluginManager.java | 2 + .../prestosql/plugin/memory/MemoryConfig.java | 5 + .../prestosql/plugin/memory/MemoryPlugin.java | 19 ++++ .../prestosql/plugin/mysql/MySqlPlugin.java | 21 ++++ .../plugin/postgresql/PostgreSqlPlugin.java | 21 ++++ .../main/java/io/prestosql/spi/Plugin.java | 7 ++ .../spi/function/ConnectorConfig.java | 58 ++++++++++ .../io/prestosql/spi/function/Mandatory.java | 60 ++++++++++ .../spi/queryeditorui/ConnectorUtil.java | 74 ++++++++++++ .../ConnectorWithProperties.java | 107 ++++++++++++++++-- .../spi/queryeditorui/PropertyType.java | 34 ++++++ .../plugin/sqlserver/SqlServerPlugin.java | 21 ++++ .../prestosql/plugin/thrift/ThriftPlugin.java | 29 +++++ .../prestosql/plugin/tpcds/TpcdsPlugin.java | 15 +++ .../io/prestosql/plugin/tpch/TpchPlugin.java | 15 +++ 43 files changed, 832 insertions(+), 352 deletions(-) delete mode 100644 presto-main/etc/connector_properties.json.template create mode 100644 presto-spi/src/main/java/io/prestosql/spi/function/ConnectorConfig.java create mode 100644 presto-spi/src/main/java/io/prestosql/spi/function/Mandatory.java create mode 100644 presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorUtil.java rename {presto-main/src/main/java/io/prestosql/queryeditorui/protocol => presto-spi/src/main/java/io/prestosql/spi/queryeditorui}/ConnectorWithProperties.java (65%) create mode 100644 presto-spi/src/main/java/io/prestosql/spi/queryeditorui/PropertyType.java diff --git a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPlugin.java b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPlugin.java index 690247d75..355dc2714 100644 --- a/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPlugin.java +++ b/hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPlugin.java @@ -15,9 +15,22 @@ package io.hetu.core.plugin.carbondata; import com.google.common.collect.ImmutableList; import io.prestosql.plugin.hive.HivePlugin; +import io.prestosql.plugin.hive.metastore.thrift.StaticMetastoreConfig; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; import org.apache.carbondata.core.datastore.impl.FileFactory; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "Carbondata: Query data stored in a Carbondata warehouse", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html", + configLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html#configuration") public class CarbondataPlugin extends HivePlugin { @@ -36,4 +49,12 @@ public class CarbondataPlugin { return ImmutableList.of(new CarbondataConnectorFactory("carbondata", getClassLoader())); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = CarbondataPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(StaticMetastoreConfig.class.getDeclaredMethods())); + } } diff --git a/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java b/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java index 0dc3cf938..6c73f3bdb 100644 --- a/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java +++ b/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java @@ -20,6 +20,7 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.prestosql.spi.function.Mandatory; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -143,6 +144,10 @@ public class DataCenterConfig * @param connectionUrl the connection url of data center. * @return DataCenterConfig object. */ + @Mandatory(name = "connection-url", + description = "The connection URL of remote OpenLooKeng data center", + defaultValue = "http://host:port", + required = true) @Config("connection-url") public DataCenterConfig setConnectionUrl(URI connectionUrl) { @@ -239,6 +244,10 @@ public class DataCenterConfig * @param connectionUser the connection user name. * @return DataCenterConfig object. */ + @Mandatory(name = "connection-user", + description = "User to connect to remote data center", + defaultValue = "lk", + required = true) @Config("connection-user") public DataCenterConfig setConnectionUser(String connectionUser) { diff --git a/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java b/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java index 2ff70ef1d..eac66e95a 100644 --- a/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java +++ b/hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java @@ -18,12 +18,24 @@ package io.hetu.core.plugin.datacenter; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.util.Arrays; +import java.util.Optional; /** * Data center plugin. * * @since 2020-02-11 */ +@ConnectorConfig(connectorLabel = "DataCenter: Query data on remote OpenLooKeng data center", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/datacenter.html", + configLink = "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration") public class DataCenterPlugin implements Plugin { @@ -32,4 +44,12 @@ public class DataCenterPlugin { return ImmutableList.of(new DataCenterConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = DataCenterPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(DataCenterConfig.class.getDeclaredMethods())); + } } diff --git a/hetu-docs/en/admin/web-interface.md b/hetu-docs/en/admin/web-interface.md index 4cfa730a7..6d7a86811 100644 --- a/hetu-docs/en/admin/web-interface.md +++ b/hetu-docs/en/admin/web-interface.md @@ -24,79 +24,3 @@ For more detailed information about a query, simply click the query ID link. The The summary section has a button to kill the currently running query. There are two visualizations available in the summary section: task execution and timeline. The full JSON document containing information and statistics about the query is available by clicking the *JSON* link. These visualizations and other statistics can be used to analyze where time is being spent for a query. - -## Connector Properties File Configuration - -The web interface supports expanding and adding new connectors in the interface by setting the connector properties file.The connector properties file can be set by referring to the template`presto-main/etc/connector_properties.json.template`. - -`etc/connector_properties.json` is the default path of the web connector configuration file，you can change the path by set and modify `hetu.queryeditor-ui.server.connector-properties-json` property of `etc/config.properties`. - -The web connector configuration file supports the following properties: - -| Property Name | Description| Required | -|----------|----------|----------| -| `connectorWithProperties` | Yes | Connector with properties | -| `user` | Yes | The user who created the connector | -| `uuid` | Yes | Connector unique identification | -| `docLink` | Yes | Link to official introduction document of the connector | -| `configLink` | Yes | Link to official configuration document of the connector | - -The properties that `connectorWithProperties` supported are as follows: - -| Property Name | Description| Required | Default Value | -|----------|----------|----------|----------| -| `connectorName`| Yes | Connector name | | -| `connectorLabel`| Yes | Connector label | | -| `propertiesEnabled` | No | Whether to enable the connector properties configuration panel or not | false | -| `catalogConfigFilesEnabled` | No | Whether to enable properties file upload for current connector configuration or not | false | -| `globalConfigFilesEnabled` | No | Whether to enable properties file upload for system configuration (system configuration are shared by all connectors) or not | false | -| `properties` | Yes | Connector properties | | - -The properties that `properties` supported are as follows: - -| Property Name | Description| Required | Default Value | -|----------|----------|----------|----------| -| `name`| Yes | Property name | | -| `value`| Yes | Property value | | -| `description` | Yes | Property description | | -| `required` | No | Property required | false | -| `readOnly` | No | If the read-only mode is enabled, the value modification is not supported | false | -| `type` | Yes | Property type | | - -Example, datacenter connector properties configuration of the connector properties file is as follow： - -``` json -[ - { - "connectorWithProperties": { - "connectorName": "dc", - "connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center", - "propertiesEnabled": true, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [ - { - "name": "connection-url", - "value": "http://localhost:8080", - "description": "The connection URL of remote OpenLooKeng data center", - "required" : true, - "type" : "string" - }, - { - "name": "connection-user", - "value": "lk", - "description": "User to connect to remote data center", - "required" : true, - "readOnly": true, - "type": "string" - } - ] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e", - "docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html", - "configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration" - }, - ...... -] -``` \ No newline at end of file diff --git a/hetu-docs/zh/admin/web-interface.md b/hetu-docs/zh/admin/web-interface.md index 6c83851a6..a61fc194e 100644 --- a/hetu-docs/zh/admin/web-interface.md +++ b/hetu-docs/zh/admin/web-interface.md @@ -21,79 +21,3 @@ openLooKeng提供了一个用于监视和管理查询的Web界面。Web界面可 有关查询的详细信息，请单击查询ID链接。查询详细信息页有一个摘要部分、查询的各个阶段的图形表示和任务列表。可以点击每个任务ID以获得有关该任务的更多信息。 摘要部分有一个按钮，用于终止当前正在运行的查询。在摘要部分有两个可视化：任务执行和时间线。通过单击JSON链接，可以获得包含有关查询的信息和统计信息的完整JSON文档。这些可视化和其他统计信息可用于分析查询所花费的时间。 - -## 连接器属性文件设置 - -Web界面支持通过连接器属性文件设置的方式在界面拓展和添加新的连接器，连接器属性文件可以参考项目提供的模板`presto-main/etc/connector_properties.json.template`进行设置。 - -系统默认使用`etc/connector_properties.json`作为Web界面连接器属性文件路径，可以通过设置`etc/config.properties`中的`hetu.queryeditor-ui.server.connector-properties-json`属性修改文件路径。 - -Web连接器配置文件支持配置的属性如下： - -| 属性名称 | 是否必选 | 描述 | -|----------|----------|----------| -| `connectorWithProperties` | 是 | 连接器详细配置参数配置 | -| `user` | 是 | 建立连接器的用户 | -| `uuid` | 是 | 连接器唯一标识 | -| `docLink` | 是 | 连接器官方介绍文档链接 | -| `configLink` | 是 | 连接器属性配置文档链接 | - -其中，`connectorWithProperties`支持配置的属性如下： - -| 属性名称| 是否必选 | 描述 | 默认值 | -|----------|----------|----------|----------| -| `connectorName`| 是 | 连接器名 | | -| `connectorLabel`| 是 | 连接器描述 | | -| `propertiesEnabled` | 否 | 是否启用连接器属性配置面板，启用界面才支持配置连接器属性设置功能 | false | -| `catalogConfigFilesEnabled` | 否 | 是否启用当前连接器的配置文件上传功能 | false | -| `globalConfigFilesEnabled` | 否 | 是否启用系统配置文件（系统配置文件的配置项所有连接器共享）上传功能 | false | -| `properties` | 是 | 连接器属性配置项 | | - -其中，`properties`支持配置的属性如下： - -| 属性名称 | 是否必选 | 描述 | 默认值 | -|----------|----------|----------|----------| -| `name`| 是 | 属性名 | | -| `value`| 是 | 属性值 | | -| `description` | 是 | 属性描述 | | -| `required` | 否 | 该属性是否必填 | false | -| `readOnly` | 否 | 是否开启只读模式，启用的话在界面不支持修改属性值 | false | -| `type` | 是 | 配置项数据类型 | | - -示例，连接器属性文件中配置DataCenter连接器属性： - -``` json -[ - { - "connectorWithProperties": { - "connectorName": "dc", - "connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center", - "propertiesEnabled": true, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [ - { - "name": "connection-url", - "value": "http://localhost:8080", - "description": "The connection URL of remote OpenLooKeng data center", - "required" : true, - "type" : "string" - }, - { - "name": "connection-user", - "value": "lk", - "description": "User to connect to remote data center", - "required" : true, - "readOnly": true, - "type": "string" - } - ] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e", - "docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html", - "configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration" - }, - ...... -] -``` diff --git a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaPlugin.java b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaPlugin.java index 3771dac5a..bbec4d90a 100644 --- a/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaPlugin.java +++ b/hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaPlugin.java @@ -14,13 +14,27 @@ */ package io.hetu.core.plugin.hana; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; +import io.prestosql.plugin.jdbc.JdbcMetadataConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Optional; /** * HanaPlugin class * * @since 2019-07-10 */ +@ConnectorConfig(connectorLabel = "Hana: Query and create tables on an external Hana database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/hana.html", + configLink = "https://openlookeng.io/docs/docs/connector/hana.html#configuration") public class HanaPlugin extends JdbcPlugin { @@ -32,4 +46,16 @@ public class HanaPlugin // name of the connector and the module implementation super("hana", new HanaClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = HanaPlugin.class.getAnnotation(ConnectorConfig.class); + ArrayList methods = new ArrayList<>(); + methods.addAll(Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + methods.addAll(Arrays.asList(JdbcMetadataConfig.class.getDeclaredMethods())); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, methods); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:sap://host:port"); + return connectorWithProperties; + } } diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/HBasePlugin.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/HBasePlugin.java index e16bff8e0..38dc906cd 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/HBasePlugin.java +++ b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/HBasePlugin.java @@ -16,15 +16,28 @@ package io.hetu.core.plugin.hbase; import com.google.common.collect.ImmutableList; import com.google.inject.Module; +import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.connector.HBaseConnectorFactory; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.util.Arrays; +import java.util.Optional; /** * HBasePlugin * * @since 2020-03-18 */ +@ConnectorConfig(connectorLabel = "HBase: Query and create tables on an external Apache HBase instance", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/hbase.html", + configLink = "https://openlookeng.io/docs/docs/connector/hbase.html#configuration") public class HBasePlugin implements Plugin { @@ -58,4 +71,12 @@ public class HBasePlugin } return classLoader; } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = HBasePlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(HBaseConfig.class.getDeclaredMethods())); + } } diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java index f637fc6db..080e257ca 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java +++ b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java @@ -15,6 +15,7 @@ package io.hetu.core.plugin.hbase.conf; import io.airlift.configuration.Config; +import io.prestosql.spi.function.Mandatory; /** * hbase.properties @@ -133,6 +134,10 @@ public class HBaseConfig return zkQuorum; } + @Mandatory(name = "hbase.zookeeper.quorum", + description = "Zookeeper cluster address", + defaultValue = "host1,host2", + required = true) @Config("hbase.zookeeper.quorum") public void setZkQuorum(String zkQuorum) { @@ -144,6 +149,10 @@ public class HBaseConfig return zkClientPort; } + @Mandatory(name = "hbase.zookeeper.property.clientPort", + description = "Zookeeper client port", + defaultValue = "2181", + required = true) @Config("hbase.zookeeper.property.clientPort") public void setZkClientPort(String zkClientPort) { @@ -155,6 +164,9 @@ public class HBaseConfig return zkZnodeParent; } + @Mandatory(name = "hbase.zookeeper.znode.parent", + description = "Zookeeper znode parent of hbase", + defaultValue = "/hbase") @Config("hbase.zookeeper.znode.parent") public void setZkZnodeParent(String zkZnodeParent) { @@ -166,6 +178,9 @@ public class HBaseConfig return metastoreType; } + @Mandatory(name = "hbase.metastore.type", + description = "The storage of hbase metadata", + defaultValue = "hetuMetastore") @Config("hbase.metastore.type") public void setMetastoreType(String metastoreType) { diff --git a/hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussPlugin.java b/hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussPlugin.java index 89928dd5a..339155dc1 100644 --- a/hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussPlugin.java +++ b/hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussPlugin.java @@ -15,8 +15,19 @@ package io.hetu.core.plugin.opengauss; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "OpenGauss : Query and create tables on an external OpenGauss database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/opengauss.html", + configLink = "https://openlookeng.io/docs/docs/connector/opengauss.html#configuration") public class OpenGaussPlugin extends JdbcPlugin { @@ -24,4 +35,14 @@ public class OpenGaussPlugin { super("opengauss", new OpenGaussClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = OpenGaussPlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:postgresql://host:port/database"); + return connectorWithProperties; + } } diff --git a/hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OraclePlugin.java b/hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OraclePlugin.java index c090b1c01..dba9e6ff7 100644 --- a/hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OraclePlugin.java +++ b/hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OraclePlugin.java @@ -15,14 +15,24 @@ package io.hetu.core.plugin.oracle; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.util.Arrays; +import java.util.Optional; /** * OraclePlugin * * @since 2019-07-06 */ - +@ConnectorConfig(connectorLabel = "Oracle : Query and create tables on an external Oracle database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/oracle.html", + configLink = "https://openlookeng.io/docs/docs/connector/oracle.html#configuration") public class OraclePlugin extends JdbcPlugin { @@ -34,4 +44,14 @@ public class OraclePlugin // name of the connector and the module implementation super("oracle", new OracleClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = OraclePlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:oracle:thin:@host:port/ORCLCDB"); + return connectorWithProperties; + } } diff --git a/hetu-vdm/src/main/java/io/hetu/core/plugin/vdm/VdmPlugin.java b/hetu-vdm/src/main/java/io/hetu/core/plugin/vdm/VdmPlugin.java index b1490680b..8c427b222 100644 --- a/hetu-vdm/src/main/java/io/hetu/core/plugin/vdm/VdmPlugin.java +++ b/hetu-vdm/src/main/java/io/hetu/core/plugin/vdm/VdmPlugin.java @@ -17,6 +17,12 @@ package io.hetu.core.plugin.vdm; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.util.Arrays; +import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; @@ -26,6 +32,12 @@ import static com.google.common.base.Strings.isNullOrEmpty; * * @since 2020-02-27 */ +@ConnectorConfig(connectorLabel = "VDM : Virtualize Data Market", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/vdm.html", + configLink = "https://openlookeng.io/docs/docs/connector/vdm.html#usage") public class VdmPlugin implements Plugin { @@ -56,4 +68,12 @@ public class VdmPlugin { return ImmutableList.of(new VdmConnectorFactory(name, VdmPlugin.class.getClassLoader())); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = VdmPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(VdmConfig.class.getDeclaredMethods())); + } } diff --git a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java index 57d673284..1012d117b 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java @@ -18,6 +18,7 @@ import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.ConfigSecuritySensitive; import io.airlift.units.Duration; import io.airlift.units.MinDuration; +import io.prestosql.spi.function.Mandatory; import javax.annotation.Nullable; import javax.validation.constraints.Min; @@ -267,6 +268,10 @@ public class BaseJdbcConfig return connectionUser; } + @Mandatory(name = "connection-user", + description = "User to connect to remote database", + defaultValue = "root", + required = true) @Config("connection-user") public BaseJdbcConfig setConnectionUser(String connectionUser) { @@ -280,6 +285,10 @@ public class BaseJdbcConfig return connectionPassword; } + @Mandatory(name = "connection-password", + description = "Password of user to connect to remote database", + defaultValue = "secret", + required = true) @Config("connection-password") @ConfigSecuritySensitive public BaseJdbcConfig setConnectionPassword(String connectionPassword) diff --git a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcMetadataConfig.java b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcMetadataConfig.java index 90736e208..ab7bbc940 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcMetadataConfig.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcMetadataConfig.java @@ -17,6 +17,7 @@ import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; import io.airlift.units.Duration; import io.airlift.units.MinDuration; +import io.prestosql.spi.function.Mandatory; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; @@ -37,6 +38,9 @@ public class JdbcMetadataConfig return allowDropTable; } + @Mandatory(name = "allow-drop-table", + description = "Allow connector to drop tables", + defaultValue = "true") @Config("allow-drop-table") @ConfigDescription("Allow connector to drop tables") public JdbcMetadataConfig setAllowDropTable(boolean allowDropTable) diff --git a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchConfig.java b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchConfig.java index 3c2e40b2d..0c76b8580 100644 --- a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchConfig.java +++ b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchConfig.java @@ -19,6 +19,7 @@ import io.airlift.configuration.ConfigSecuritySensitive; import io.airlift.configuration.DefunctConfig; import io.airlift.units.Duration; import io.airlift.units.MinDuration; +import io.prestosql.spi.function.Mandatory; import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; @@ -78,6 +79,10 @@ public class ElasticsearchConfig return host; } + @Mandatory(name = "elasticsearch.host", + description = "Elasticsearch host", + defaultValue = "host", + required = true) @Config("elasticsearch.host") public ElasticsearchConfig setHost(String host) { @@ -90,6 +95,9 @@ public class ElasticsearchConfig return port; } + @Mandatory(name = "elasticsearch.port", + description = "Elasticsearch port", + defaultValue = "9200") @Config("elasticsearch.port") public ElasticsearchConfig setPort(int port) { @@ -103,6 +111,9 @@ public class ElasticsearchConfig return defaultSchema; } + @Mandatory(name = "elasticsearch.default-schema-name", + description = "Default schema name to use", + defaultValue = "default") @Config("elasticsearch.default-schema-name") @ConfigDescription("Default schema name to use") public ElasticsearchConfig setDefaultSchema(String defaultSchema) @@ -118,6 +129,9 @@ public class ElasticsearchConfig return scrollSize; } + @Mandatory(name = "elasticsearch.scroll-size", + description = "Scroll batch size", + defaultValue = "1000") @Config("elasticsearch.scroll-size") @ConfigDescription("Scroll batch size") public ElasticsearchConfig setScrollSize(int scrollSize) @@ -132,6 +146,9 @@ public class ElasticsearchConfig return scrollTimeout; } + @Mandatory(name = "elasticsearch.scroll-timeout", + description = "Scroll timeout", + defaultValue = "1m") @Config("elasticsearch.scroll-timeout") @ConfigDescription("Scroll timeout") public ElasticsearchConfig setScrollTimeout(Duration scrollTimeout) @@ -146,6 +163,9 @@ public class ElasticsearchConfig return requestTimeout; } + @Mandatory(name = "elasticsearch.request-timeout", + description = "Elasticsearch request timeout", + defaultValue = "2s") @Config("elasticsearch.request-timeout") @ConfigDescription("Elasticsearch request timeout") public ElasticsearchConfig setRequestTimeout(Duration requestTimeout) diff --git a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPlugin.java b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPlugin.java index 5f19188be..842ca96df 100644 --- a/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPlugin.java +++ b/presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPlugin.java @@ -17,9 +17,21 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; + +import java.util.Arrays; +import java.util.Optional; import static java.util.Objects.requireNonNull; +@ConnectorConfig(connectorLabel = "Elasticsearch: Allow access to Elasticsearch data from openLooKeng", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/elasticsearch.html", + configLink = "https://openlookeng.io/docs/docs/connector/elasticsearch.html#configuration") public class ElasticsearchPlugin implements Plugin { @@ -41,4 +53,12 @@ public class ElasticsearchPlugin { return ImmutableList.of(connectorFactory); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = ElasticsearchPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(ElasticsearchConfig.class.getDeclaredMethods())); + } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java index f32782a09..27e8b5824 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java @@ -26,6 +26,8 @@ import io.airlift.units.MinDataSize; import io.airlift.units.MinDuration; import io.prestosql.orc.OrcWriteValidation.OrcWriteValidationMode; import io.prestosql.plugin.hive.s3.S3FileSystemType; +import io.prestosql.spi.function.Mandatory; +import io.prestosql.spi.queryeditorui.PropertyType; import org.joda.time.DateTimeZone; import javax.annotation.Nullable; @@ -551,6 +553,11 @@ public class HiveConfig return resourceConfigFiles; } + @Mandatory(name = "hive.config.resources", + description = "An optional comma-separated list of HDFS configuration files. These files must exist on the machines running openLooKeng. Only specify this if absolutely necessary to access HDFS. Ensure to upload these files.", + defaultValue = "core-site.xml,hdfs-site.xml", + readOnly = true, + type = PropertyType.FILES) @Config("hive.config.resources") public HiveConfig setResourceConfigFiles(String files) { diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePlugin.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePlugin.java index c7cbf2cf8..eb964c7ff 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePlugin.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/HivePlugin.java @@ -15,15 +15,28 @@ package io.prestosql.plugin.hive; import com.google.common.collect.ImmutableList; import io.prestosql.plugin.hive.metastore.HiveMetastore; +import io.prestosql.plugin.hive.metastore.thrift.StaticMetastoreConfig; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Strings.isNullOrEmpty; import static java.util.Objects.requireNonNull; +@ConnectorConfig(connectorLabel = "Hive: Query data stored in a Hive data warehouse", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/hive.html", + configLink = "https://openlookeng.io/docs/docs/connector/hive.html#configuration") public class HivePlugin implements Plugin { @@ -47,4 +60,14 @@ public class HivePlugin { return ImmutableList.of(new HiveConnectorFactory(name, HivePlugin.class.getClassLoader(), metastore)); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = HivePlugin.class.getAnnotation(ConnectorConfig.class); + ArrayList methods = new ArrayList<>(); + methods.addAll(Arrays.asList(StaticMetastoreConfig.class.getDeclaredMethods())); + methods.addAll(Arrays.asList(HiveConfig.class.getDeclaredMethods())); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, methods); + } } diff --git a/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/StaticMetastoreConfig.java b/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/StaticMetastoreConfig.java index 1c69f5a8a..f58c368f6 100644 --- a/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/StaticMetastoreConfig.java +++ b/presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/StaticMetastoreConfig.java @@ -17,6 +17,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; +import io.prestosql.spi.function.Mandatory; import javax.validation.constraints.NotNull; @@ -38,6 +39,10 @@ public class StaticMetastoreConfig return metastoreUris; } + @Mandatory(name = "hive.metastore.uri", + description = "The URI(s) of the Hive metastore to connect to using the Thrift protocol. If multiple URIs are provided, the first URI is used by default and the rest of the URIs are fallback metastores. This property is required. Example: thrift://192.0.2.3:9083 or thrift://192.0.2.3:9083,thrift://192.0.2.4:9083", + defaultValue = "thrift://host:port", + required = true) @Config("hive.metastore.uri") @ConfigDescription("Hive metastore URIs (comma separated)") public StaticMetastoreConfig setMetastoreUris(String uris) diff --git a/presto-jmx/src/main/java/io/prestosql/plugin/jmx/JmxPlugin.java b/presto-jmx/src/main/java/io/prestosql/plugin/jmx/JmxPlugin.java index 05ffedfa1..4f5fde52f 100644 --- a/presto-jmx/src/main/java/io/prestosql/plugin/jmx/JmxPlugin.java +++ b/presto-jmx/src/main/java/io/prestosql/plugin/jmx/JmxPlugin.java @@ -16,7 +16,17 @@ package io.prestosql.plugin.jmx; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "JMX: Query JMX information from all nodes in a openLooKeng cluster", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/jmx.html", + configLink = "https://openlookeng.io/docs/docs/connector/jmx.html#configuration") public class JmxPlugin implements Plugin { @@ -25,4 +35,12 @@ public class JmxPlugin { return ImmutableList.of(new JmxConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = JmxPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(JmxConnectorConfig.class.getDeclaredMethods())); + } } diff --git a/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaConnectorConfig.java b/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaConnectorConfig.java index 65c711d1a..4a93d5e40 100644 --- a/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaConnectorConfig.java +++ b/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaConnectorConfig.java @@ -21,6 +21,7 @@ import io.airlift.units.DataSize.Unit; import io.airlift.units.Duration; import io.airlift.units.MinDuration; import io.prestosql.spi.HostAddress; +import io.prestosql.spi.function.Mandatory; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; @@ -88,6 +89,10 @@ public class KafkaConnectorConfig return tableNames; } + @Mandatory(name = "kafka.table-names", + description = "List of all tables provided by the catalog", + defaultValue = "table1,table2", + required = true) @Config("kafka.table-names") public KafkaConnectorConfig setTableNames(String tableNames) { @@ -101,6 +106,9 @@ public class KafkaConnectorConfig return defaultSchema; } + @Mandatory(name = "kafka.default-schema", + description = "Default schema name to use", + defaultValue = "default") @Config("kafka.default-schema") public KafkaConnectorConfig setDefaultSchema(String defaultSchema) { @@ -114,6 +122,10 @@ public class KafkaConnectorConfig return nodes; } + @Mandatory(name = "kafka.nodes", + description = "List of nodes in the Kafka cluster ", + defaultValue = "host1:port,host2:port", + required = true) @Config("kafka.nodes") public KafkaConnectorConfig setNodes(String nodes) { diff --git a/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaPlugin.java b/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaPlugin.java index fcc941082..e17b7b795 100644 --- a/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaPlugin.java +++ b/presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaPlugin.java @@ -18,7 +18,11 @@ import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -28,6 +32,12 @@ import static java.util.Objects.requireNonNull; /** * Presto plugin to use Apache Kafka as a data source. */ +@ConnectorConfig(connectorLabel = "Kafka: Allow the use of Apache Kafka topics as tables in openLooKeng", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/kafka.html", + configLink = "https://openlookeng.io/docs/docs/connector/kafka.html#configuration") public class KafkaPlugin implements Plugin { @@ -44,4 +54,12 @@ public class KafkaPlugin { return ImmutableList.of(new KafkaConnectorFactory(tableDescriptionSupplier)); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = KafkaPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(KafkaConnectorConfig.class.getDeclaredMethods())); + } } diff --git a/presto-local-file/src/main/java/io/prestosql/plugin/localfile/LocalFilePlugin.java b/presto-local-file/src/main/java/io/prestosql/plugin/localfile/LocalFilePlugin.java index 7593c85f0..f6611cbb3 100644 --- a/presto-local-file/src/main/java/io/prestosql/plugin/localfile/LocalFilePlugin.java +++ b/presto-local-file/src/main/java/io/prestosql/plugin/localfile/LocalFilePlugin.java @@ -16,7 +16,17 @@ package io.prestosql.plugin.localfile; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "LocalFile: Query data stored on the local file system of each worker", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/localfile.html", + configLink = "https://openlookeng.io/docs/docs/connector/localfile.html#configuration") public class LocalFilePlugin implements Plugin { @@ -25,4 +35,12 @@ public class LocalFilePlugin { return ImmutableList.of(new LocalFileConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = LocalFilePlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(LocalFileConfig.class.getDeclaredMethods())); + } } diff --git a/presto-main/etc/connector_properties.json.template b/presto-main/etc/connector_properties.json.template deleted file mode 100644 index 898f01b49..000000000 --- a/presto-main/etc/connector_properties.json.template +++ /dev/null @@ -1,103 +0,0 @@ -[ - { - "connectorWithProperties": { - "connectorName": "tpch", - "connectorLabel": "TPC-H : TPC Benchmark™ H, data is generated on the fly", - "propertiesEnabled": false, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9a", - "docLink": "https://openlookeng.io/docs/docs/connector/tpch.html", - "configLink": "https://openlookeng.io/docs/docs/connector/tpch.html#configuration" - }, - { - "connectorWithProperties": { - "connectorName": "tpcds", - "connectorLabel": "TPC-DS : TPC Benchmark™ DS, data is generated on the fly", - "propertiesEnabled": false, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9b", - "docLink": "https://openlookeng.io/docs/docs/connector/tpcds.html", - "configLink": "https://openlookeng.io/docs/docs/connector/tpcds.html#configuration" - }, - { - "connectorWithProperties": { - "connectorName": "memory", - "connectorLabel": "Memory: All data and metadata is only stored in memory", - "propertiesEnabled": false, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9d", - "docLink": "https://openlookeng.io/docs/docs/connector/memory.html", - "configLink": "https://openlookeng.io/docs/docs/connector/memory.html#configuration" - }, - { - "connectorWithProperties": { - "connectorName": "hive-hadoop2", - "connectorLabel": "Hive-Hadoop2: Data stored in Hive data warehouse / HDFS", - "propertiesEnabled": true, - "catalogConfigFilesEnabled": true, - "globalConfigFilesEnabled": true, - "properties": [ - { - "name": "hive.metastore.uri", - "value": "thrift://localhost:9083", - "description": "The URI(s) of the Hive metastore to connect to using the Thrift protocol.\nIf multiple URIs are provided, the first URI is used by default and the rest of the URIs are fallback metastores.\nThis property is required.\nExample: thrift://192.0.2.3:9083 or thrift://192.0.2.3:9083,thrift://192.0.2.4:9083", - "required" : true, - "type" : "string" - }, - { - "name": "hive.config.resources", - "value": "core-site.xml,hdfs-site.xml", - "description": "An optional comma-separated list of HDFS configuration files.\nThese files must exist on the machines running openLooKeng.\nOnly specify this if absolutely necessary to access HDFS.\n\nEnsure to upload these files.", - "readOnly": true, - "type": "files" - } - ] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9c", - "docLink": "https://openlookeng.io/docs/docs/connector/hive.html", - "configLink": "https://openlookeng.io/docs/docs/connector/hive.html#configuration" - }, - { - "connectorWithProperties": { - "connectorName": "dc", - "connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center", - "propertiesEnabled": true, - "catalogConfigFilesEnabled": false, - "globalConfigFilesEnabled": false, - "properties": [ - { - "name": "connection-url", - "value": "http://localhost:8080", - "description": "The connection URL of remote OpenLooKeng data center", - "required" : true, - "type" : "string" - }, - { - "name": "connection-user", - "value": "lk", - "description": "User to connect to remote data center", - "required" : true, - "readOnly": true, - "type": "string" - } - ] - }, - "user": "lk", - "uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e", - "docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html", - "configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration" - } -] diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java b/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java index 1e1666789..7f1412b3e 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java +++ b/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java @@ -31,7 +31,6 @@ public class QueryEditorConfig private String coordinatorUri; private String featuredQueriesPath = "etc/featured_queries.json"; private String userQueriesPath = "etc/user_queries.json"; - private String connectorsListPath = "etc/connector_properties.json"; private int maxResultCount = 1000; private DataSize maxResultSize = new DataSize(1, DataSize.Unit.GIGABYTE); private Optional sharedSecret = Optional.empty(); @@ -106,23 +105,11 @@ public class QueryEditorConfig return this; } - @Config("hetu.queryeditor-ui.server.connector-properties-json") - public QueryEditorConfig setConnectorsListPath(String connectorsListPath) - { - this.connectorsListPath = connectorsListPath; - return this; - } - public String getUserQueriesPath() { return userQueriesPath; } - public String getConnectorsListPath() - { - return connectorsListPath; - } - @NotNull public Optional getSharedSecret() { diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java b/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java index 69067319a..337bc4c3d 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java +++ b/presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java @@ -50,7 +50,6 @@ import io.prestosql.queryeditorui.resources.TablesResource; import io.prestosql.queryeditorui.resources.UIExecuteResource; import io.prestosql.queryeditorui.resources.UserResource; import io.prestosql.queryeditorui.security.UiAuthenticator; -import io.prestosql.queryeditorui.store.connectors.ConnectorCache; import io.prestosql.queryeditorui.store.files.ExpiringFileStore; import io.prestosql.queryeditorui.store.history.JobHistoryStore; import io.prestosql.queryeditorui.store.history.LocalJobHistoryStore; @@ -280,10 +279,4 @@ public class QueryEditorUIModule { return new InMemoryQueryStore(new File(queryEditorConfig.getFeaturedQueriesPath()), new File(queryEditorConfig.getUserQueriesPath())); } - - @Provides - public ConnectorCache provideConnectorCache(QueryEditorConfig queryEditorConfig) throws IOException - { - return new ConnectorCache(new File(queryEditorConfig.getConnectorsListPath())); - } } diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/Connector.java b/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/Connector.java index f0ca19f53..b9a78d05f 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/Connector.java +++ b/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/Connector.java @@ -16,6 +16,7 @@ package io.prestosql.queryeditorui.protocol; import com.fasterxml.jackson.annotation.JsonProperty; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; import java.util.UUID; @@ -26,9 +27,6 @@ public class Connector @JsonProperty private ConnectorWithProperties connectorWithProperties; - @JsonProperty - private String user; - @JsonProperty private UUID uuid; @@ -38,12 +36,6 @@ public class Connector @JsonProperty private String configLink; - @JsonProperty - public String getUser() - { - return user; - } - @JsonProperty public UUID getUuid() { @@ -72,13 +64,11 @@ public class Connector { } - public Connector(@JsonProperty("user") String userName, - @JsonProperty("uuid") UUID uuid, + public Connector(@JsonProperty("uuid") UUID uuid, @JsonProperty("docLink") String docLink, @JsonProperty("configLink") String configLink, @JsonProperty("connectorWithProperties") ConnectorWithProperties connectorWithProperties) { - this.user = requireNonNull(userName, "userName is null"); this.connectorWithProperties = requireNonNull(connectorWithProperties, "Properties is null"); this.uuid = requireNonNull(uuid, "uuid is null"); this.docLink = requireNonNull(docLink, "docLink is null"); diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/resources/ConnectorResource.java b/presto-main/src/main/java/io/prestosql/queryeditorui/resources/ConnectorResource.java index 3cfc5c1e9..13cf3e77c 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/resources/ConnectorResource.java +++ b/presto-main/src/main/java/io/prestosql/queryeditorui/resources/ConnectorResource.java @@ -14,9 +14,6 @@ */ package io.prestosql.queryeditorui.resources; -import com.google.inject.Inject; -import io.prestosql.queryeditorui.protocol.Connector; -import io.prestosql.queryeditorui.security.UiAuthenticator; import io.prestosql.queryeditorui.store.connectors.ConnectorCache; import javax.servlet.http.HttpServletRequest; @@ -27,25 +24,13 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.util.List; - @Path("/api/connectors") public class ConnectorResource { - private final ConnectorCache connectorCache; - - @Inject - public ConnectorResource(ConnectorCache connectorCache) - { - this.connectorCache = connectorCache; - } - @GET @Produces(MediaType.APPLICATION_JSON) public Response getConnectors(@Context HttpServletRequest servletRequest) { - String user = UiAuthenticator.getUser(servletRequest); - List lk = connectorCache.getConnectors(user); - return Response.ok(lk).build(); + return Response.ok(ConnectorCache.getConnectors()).build(); } } diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/store/connectors/ConnectorCache.java b/presto-main/src/main/java/io/prestosql/queryeditorui/store/connectors/ConnectorCache.java index 63d7450bc..547414ac1 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/store/connectors/ConnectorCache.java +++ b/presto-main/src/main/java/io/prestosql/queryeditorui/store/connectors/ConnectorCache.java @@ -12,60 +12,41 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.prestosql.queryeditorui.store.connectors; -import io.airlift.json.JsonCodec; -import io.airlift.json.JsonCodecFactory; -import io.airlift.json.ObjectMapperProvider; import io.prestosql.queryeditorui.protocol.Connector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import io.prestosql.spi.Plugin; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; +import java.util.Optional; +import java.util.UUID; + +import static java.util.Collections.unmodifiableList; public class ConnectorCache { - private static final Logger LOG = LoggerFactory.getLogger(ConnectorCache.class); + private ConnectorCache() {} - static final JsonCodec> CONNECTORS_CODEC = - new JsonCodecFactory(new ObjectMapperProvider()).listJsonCodec(Connector.class); - private List connectorsList = new ArrayList<>(); + private static final List CONNECTORS_CACHE = new ArrayList<>(); - public ConnectorCache(File connectorsJsonPath) throws IOException + public static void addCatalogConfig(Plugin plugin, String connectorName) { - loadConnectorsAndProperties(connectorsJsonPath); + Optional connectorWithProperties = plugin.getConnectorWithProperties(); + if (connectorWithProperties.isPresent() && connectorName != null) { + connectorWithProperties.get().setConnectorName(connectorName); + Connector connector = new Connector( + UUID.randomUUID(), + connectorWithProperties.get().getDocLink(), + connectorWithProperties.get().getConfigLink(), + connectorWithProperties.get()); + CONNECTORS_CACHE.add(connector); + } } - public List getConnectors(String user) + public static List getConnectors() { - return connectorsList.stream() - .filter(q -> q.getUser().equals(user)) - .collect(Collectors.toList()); - } - - private synchronized void loadConnectorsAndProperties(File connectorsJsonPath) throws IOException - { - if (!connectorsJsonPath.exists()) { - return; - } - try (FileInputStream fin = new FileInputStream(connectorsJsonPath)) { - StringBuilder jsonString = new StringBuilder(); - byte[] buffer = new byte[4096]; - int n; - while ((n = fin.read(buffer)) > 0) { - jsonString.append(new String(buffer, 0, n)); - } - connectorsList.addAll(CONNECTORS_CODEC.fromJson(jsonString.toString())); - } - catch (IOException e) { - LOG.error("Connectors properties file not found", e); - throw e; - } + return unmodifiableList(CONNECTORS_CACHE); } } diff --git a/presto-main/src/main/java/io/prestosql/server/PluginManager.java b/presto-main/src/main/java/io/prestosql/server/PluginManager.java index a11657689..adcd12f8c 100644 --- a/presto-main/src/main/java/io/prestosql/server/PluginManager.java +++ b/presto-main/src/main/java/io/prestosql/server/PluginManager.java @@ -26,6 +26,7 @@ import io.prestosql.filesystem.FileSystemClientManager; import io.prestosql.heuristicindex.HeuristicIndexerManager; import io.prestosql.metadata.MetadataManager; import io.prestosql.metastore.HetuMetaStoreManager; +import io.prestosql.queryeditorui.store.connectors.ConnectorCache; import io.prestosql.security.AccessControlManager; import io.prestosql.seedstore.SeedStoreManager; import io.prestosql.server.security.PasswordAuthenticatorManager; @@ -260,6 +261,7 @@ public class PluginManager for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) { log.info("Registering connector %s", connectorFactory.getName()); connectorManager.addConnectorFactory(connectorFactory); + ConnectorCache.addCatalogConfig(plugin, connectorFactory.getName()); } for (SessionPropertyConfigurationManagerFactory sessionConfigFactory : plugin.getSessionPropertyConfigurationManagerFactories()) { diff --git a/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryConfig.java b/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryConfig.java index 5fb630dce..88c42d66c 100644 --- a/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryConfig.java +++ b/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryConfig.java @@ -15,6 +15,7 @@ package io.prestosql.plugin.memory; import io.airlift.configuration.Config; import io.airlift.units.DataSize; +import io.prestosql.spi.function.Mandatory; import javax.validation.constraints.NotNull; @@ -42,6 +43,10 @@ public class MemoryConfig return maxDataPerNode; } + @Mandatory(name = "memory.max-data-per-node", + description = "Define memory limit for pages stored in this connector per each node", + defaultValue = "128MB", + required = true) @Config("memory.max-data-per-node") public MemoryConfig setMaxDataPerNode(DataSize maxDataPerNode) { diff --git a/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryPlugin.java b/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryPlugin.java index 3dbeacdfb..397cb7684 100644 --- a/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryPlugin.java +++ b/presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryPlugin.java @@ -16,12 +16,31 @@ package io.prestosql.plugin.memory; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "Memory : Stores all data and metadata in RAM on workers", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/memory.html", + configLink = "https://openlookeng.io/docs/docs/connector/memory.html#configuration") public final class MemoryPlugin implements Plugin { + @Override public Iterable getConnectorFactories() { return ImmutableList.of(new MemoryConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = MemoryPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(MemoryConfig.class.getDeclaredMethods())); + } } diff --git a/presto-mysql/src/main/java/io/prestosql/plugin/mysql/MySqlPlugin.java b/presto-mysql/src/main/java/io/prestosql/plugin/mysql/MySqlPlugin.java index 3d5b3ba58..0af21ef6b 100644 --- a/presto-mysql/src/main/java/io/prestosql/plugin/mysql/MySqlPlugin.java +++ b/presto-mysql/src/main/java/io/prestosql/plugin/mysql/MySqlPlugin.java @@ -13,8 +13,19 @@ */ package io.prestosql.plugin.mysql; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "MySql : Query and create tables on an external MySql database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/mysql.html", + configLink = "https://openlookeng.io/docs/docs/connector/mysql.html#configuration") public class MySqlPlugin extends JdbcPlugin { @@ -22,4 +33,14 @@ public class MySqlPlugin { super("mysql", new MySqlClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = MySqlPlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:mysql://host:port"); + return connectorWithProperties; + } } diff --git a/presto-postgresql/src/main/java/io/prestosql/plugin/postgresql/PostgreSqlPlugin.java b/presto-postgresql/src/main/java/io/prestosql/plugin/postgresql/PostgreSqlPlugin.java index 549f0a2c9..b8a0a362d 100644 --- a/presto-postgresql/src/main/java/io/prestosql/plugin/postgresql/PostgreSqlPlugin.java +++ b/presto-postgresql/src/main/java/io/prestosql/plugin/postgresql/PostgreSqlPlugin.java @@ -13,8 +13,19 @@ */ package io.prestosql.plugin.postgresql; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "PostgreSQL : Query and create tables on an external PostgreSQL database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/postgresql.html", + configLink = "https://openlookeng.io/docs/docs/connector/postgresql.html#configuration") public class PostgreSqlPlugin extends JdbcPlugin { @@ -22,4 +33,14 @@ public class PostgreSqlPlugin { super("postgresql", new PostgreSqlClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = PostgreSqlPlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:postgresql://host:port/database"); + return connectorWithProperties; + } } diff --git a/presto-spi/src/main/java/io/prestosql/spi/Plugin.java b/presto-spi/src/main/java/io/prestosql/spi/Plugin.java index 3cfb0302c..98951768e 100644 --- a/presto-spi/src/main/java/io/prestosql/spi/Plugin.java +++ b/presto-spi/src/main/java/io/prestosql/spi/Plugin.java @@ -19,6 +19,7 @@ import io.prestosql.spi.eventlistener.EventListenerFactory; import io.prestosql.spi.filesystem.HetuFileSystemClientFactory; import io.prestosql.spi.heuristicindex.IndexFactory; import io.prestosql.spi.metastore.HetuMetaStoreFactory; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; import io.prestosql.spi.resourcegroups.ResourceGroupConfigurationManagerFactory; import io.prestosql.spi.security.PasswordAuthenticatorFactory; import io.prestosql.spi.security.SystemAccessControlFactory; @@ -30,6 +31,7 @@ import io.prestosql.spi.type.ParametricType; import io.prestosql.spi.type.Type; import java.io.File; +import java.util.Optional; import java.util.Set; import static java.util.Collections.emptyList; @@ -122,6 +124,11 @@ public interface Plugin return emptyList(); } + default Optional getConnectorWithProperties() + { + return Optional.empty(); + } + default void setExternalFunctionsDir(File externalFuncsDir) {} diff --git a/presto-spi/src/main/java/io/prestosql/spi/function/ConnectorConfig.java b/presto-spi/src/main/java/io/prestosql/spi/function/ConnectorConfig.java new file mode 100644 index 000000000..c47940171 --- /dev/null +++ b/presto-spi/src/main/java/io/prestosql/spi/function/ConnectorConfig.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.spi.function; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface ConnectorConfig +{ + /** + * Connector label + */ + String connectorLabel() default ""; + + /** + * Whether to enable the connector properties configuration panel or not + */ + boolean propertiesEnabled() default false; + + /** + * Whether to enable properties file upload for current connector configuration or not + */ + boolean catalogConfigFilesEnabled() default false; + + /** + * Whether to enable properties file upload for system configuration (system configuration are shared by all connectors) or not + */ + boolean globalConfigFilesEnabled() default false; + + /** + * Link to official introduction document of the connector + */ + String docLink() default ""; + + /** + * Link to official configuration document of the connector + */ + String configLink() default ""; +} diff --git a/presto-spi/src/main/java/io/prestosql/spi/function/Mandatory.java b/presto-spi/src/main/java/io/prestosql/spi/function/Mandatory.java new file mode 100644 index 000000000..89b33dcf1 --- /dev/null +++ b/presto-spi/src/main/java/io/prestosql/spi/function/Mandatory.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.spi.function; + +import io.prestosql.spi.queryeditorui.PropertyType; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface Mandatory +{ + /** + * Property name + */ + String name() default ""; + + /** + * Property value + */ + String defaultValue() default ""; + + /** + * Property description + */ + String description() default ""; + + /** + * Property required + */ + boolean required() default false; + + /** + * If the read-only mode is enabled, the value modification is not supported + */ + boolean readOnly() default false; + + /** + * Property type + */ + PropertyType type() default PropertyType.STRING; +} diff --git a/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorUtil.java b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorUtil.java new file mode 100644 index 000000000..410e3f522 --- /dev/null +++ b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorUtil.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.prestosql.spi.queryeditorui; + +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.function.Mandatory; + +import java.lang.reflect.Method; +import java.util.Optional; + +public class ConnectorUtil +{ + private ConnectorUtil() + { + } + + public static Optional assembleConnectorProperties(ConnectorConfig connectorConfig, Iterable methods) + { + if (connectorConfig == null) { + return Optional.empty(); + } + else { + ConnectorWithProperties connectorWithProperties = new ConnectorWithProperties(); + connectorWithProperties.setConnectorLabel(Optional.of(connectorConfig.connectorLabel())); + connectorWithProperties.setPropertiesEnabled(connectorConfig.propertiesEnabled()); + connectorWithProperties.setCatalogConfigFilesEnabled(connectorConfig.catalogConfigFilesEnabled()); + connectorWithProperties.setGlobalConfigFilesEnabled(connectorConfig.globalConfigFilesEnabled()); + connectorWithProperties.setDocLink(connectorConfig.docLink()); + connectorWithProperties.setConfigLink(connectorConfig.configLink()); + + for (Method method : methods) { + ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties(); + Mandatory mandatory = method.getAnnotation(Mandatory.class); + if (mandatory != null) { + properties.setName(mandatory.name()); + properties.setDescription(mandatory.description()); + properties.setValue(mandatory.defaultValue()); + properties.setRequired(Optional.of(mandatory.required())); + properties.setReadOnly(Optional.of(mandatory.readOnly())); + properties.setType(Optional.of(mandatory.type().stringValue())); + connectorWithProperties.addProperties(properties); + } + } + return Optional.of(connectorWithProperties); + } + } + + public static void addConnUrlProperty(Optional connectorWithProperties, String conUrl) + { + if (connectorWithProperties.isPresent()) { + ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties(); + properties.setName("connection-url"); + properties.setDescription("The connection URL of remote database"); + properties.setValue(conUrl); + properties.setRequired(Optional.of(true)); + properties.setReadOnly(Optional.of(false)); + properties.setType(Optional.of(PropertyType.STRING.stringValue())); + connectorWithProperties.get().addProperties(properties); + } + } +} diff --git a/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/ConnectorWithProperties.java b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorWithProperties.java similarity index 65% rename from presto-main/src/main/java/io/prestosql/queryeditorui/protocol/ConnectorWithProperties.java rename to presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorWithProperties.java index cade12b70..34a3ccb1e 100644 --- a/presto-main/src/main/java/io/prestosql/queryeditorui/protocol/ConnectorWithProperties.java +++ b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorWithProperties.java @@ -13,11 +13,12 @@ * limitations under the License. */ -package io.prestosql.queryeditorui.protocol; +package io.prestosql.spi.queryeditorui; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -33,13 +34,19 @@ public class ConnectorWithProperties @JsonProperty private boolean propertiesEnabled; + @JsonProperty private boolean catalogConfigFilesEnabled; + @JsonProperty private boolean globalConfigFilesEnabled; @JsonProperty - private List connectorProperties; + private List connectorProperties = new ArrayList<>(); + + private String docLink; + + private String configLink; public ConnectorWithProperties() { @@ -61,6 +68,56 @@ public class ConnectorWithProperties this.connectorProperties = requireNonNull(connectorProperties, "Properties is null"); } + public void setConnectorName(String connectorName) + { + this.connectorName = requireNonNull(connectorName, "connectorName is null"); + } + + public void setConnectorLabel(Optional connectorLabel) + { + this.connectorLabel = requireNonNull(connectorLabel, "connectorLabel is null"); + } + + public void setPropertiesEnabled(boolean propertiesEnabled) + { + this.propertiesEnabled = propertiesEnabled; + } + + public void setCatalogConfigFilesEnabled(boolean catalogConfigFilesEnabled) + { + this.catalogConfigFilesEnabled = catalogConfigFilesEnabled; + } + + public void setGlobalConfigFilesEnabled(boolean globalConfigFilesEnabled) + { + this.globalConfigFilesEnabled = globalConfigFilesEnabled; + } + + public void addProperties(Properties properties) + { + this.connectorProperties.add(properties); + } + + public void setDocLink(String docLink) + { + this.docLink = docLink; + } + + public void setConfigLink(String configLink) + { + this.configLink = configLink; + } + + public String getDocLink() + { + return docLink; + } + + public String getConfigLink() + { + return configLink; + } + @JsonProperty public String getConnectorName() { @@ -102,12 +159,16 @@ public class ConnectorWithProperties public static class Properties { - private final String name; - private final String value; - private final String description; - private final Optional type; - private final Optional required; - private final Optional readOnly; + private String name; + private String value; + private String description; + private Optional type; + private Optional required; + private Optional readOnly; + + public Properties() + { + } @JsonCreator public Properties( @@ -126,6 +187,36 @@ public class ConnectorWithProperties this.readOnly = readOnly; } + public void setName(String name) + { + this.name = name; + } + + public void setValue(String value) + { + this.value = value; + } + + public void setDescription(String description) + { + this.description = description; + } + + public void setType(Optional type) + { + this.type = type; + } + + public void setRequired(Optional required) + { + this.required = required; + } + + public void setReadOnly(Optional readOnly) + { + this.readOnly = readOnly; + } + @JsonProperty public String getName() { diff --git a/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/PropertyType.java b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/PropertyType.java new file mode 100644 index 000000000..0a113774c --- /dev/null +++ b/presto-spi/src/main/java/io/prestosql/spi/queryeditorui/PropertyType.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.spi.queryeditorui; + +public enum PropertyType +{ + STRING("string"), + + FILES("files"); + + String propertyType; + + PropertyType(String propertyType) + { + this.propertyType = propertyType; + } + + public String stringValue() + { + return propertyType; + } +} diff --git a/presto-sqlserver/src/main/java/io/prestosql/plugin/sqlserver/SqlServerPlugin.java b/presto-sqlserver/src/main/java/io/prestosql/plugin/sqlserver/SqlServerPlugin.java index c90cec718..9dd023575 100644 --- a/presto-sqlserver/src/main/java/io/prestosql/plugin/sqlserver/SqlServerPlugin.java +++ b/presto-sqlserver/src/main/java/io/prestosql/plugin/sqlserver/SqlServerPlugin.java @@ -13,8 +13,19 @@ */ package io.prestosql.plugin.sqlserver; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; import io.prestosql.plugin.jdbc.JdbcPlugin; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Arrays; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "SQL Server : Query and create tables on an external SQL Server database", + propertiesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/sqlserver.html", + configLink = "https://openlookeng.io/docs/docs/connector/sqlserver.html#configuration") public class SqlServerPlugin extends JdbcPlugin { @@ -22,4 +33,14 @@ public class SqlServerPlugin { super("sqlserver", new SqlServerClientModule()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = SqlServerPlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods())); + ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:sqlserver://[serverName[\\instanceName][:portNumber]]"); + return connectorWithProperties; + } } diff --git a/presto-thrift/src/main/java/io/prestosql/plugin/thrift/ThriftPlugin.java b/presto-thrift/src/main/java/io/prestosql/plugin/thrift/ThriftPlugin.java index 06017067d..92f6e87cd 100644 --- a/presto-thrift/src/main/java/io/prestosql/plugin/thrift/ThriftPlugin.java +++ b/presto-thrift/src/main/java/io/prestosql/plugin/thrift/ThriftPlugin.java @@ -17,8 +17,12 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Module; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; import java.util.List; +import java.util.Optional; import java.util.ServiceLoader; import static com.google.common.base.Preconditions.checkArgument; @@ -26,6 +30,12 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; +@ConnectorConfig(connectorLabel = "Thrift : Connect to Thrift servers to integrate with external storage systems", + propertiesEnabled = true, + catalogConfigFilesEnabled = true, + globalConfigFilesEnabled = true, + docLink = "https://openlookeng.io/docs/docs/connector/thrift.html", + configLink = "https://openlookeng.io/docs/docs/connector/thrift.html#configuration") public class ThriftPlugin implements Plugin { @@ -62,4 +72,23 @@ public class ThriftPlugin List list = ImmutableList.copyOf(loader); return list.isEmpty() ? new ThriftPluginInfo() : getOnlyElement(list); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = ThriftPlugin.class.getAnnotation(ConnectorConfig.class); + Optional connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, + ImmutableList.of()); + if (connectorWithProperties.isPresent()) { + ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties(); + properties.setName("presto.thrift.client.addresses"); + properties.setDescription("Location of Thrift servers"); + properties.setValue("host1:port,host2:port"); + properties.setRequired(Optional.of(true)); + properties.setReadOnly(Optional.of(false)); + properties.setType(Optional.of("string")); + connectorWithProperties.get().addProperties(properties); + } + return connectorWithProperties; + } } diff --git a/presto-tpcds/src/main/java/io/prestosql/plugin/tpcds/TpcdsPlugin.java b/presto-tpcds/src/main/java/io/prestosql/plugin/tpcds/TpcdsPlugin.java index 9681c062d..2f7762289 100644 --- a/presto-tpcds/src/main/java/io/prestosql/plugin/tpcds/TpcdsPlugin.java +++ b/presto-tpcds/src/main/java/io/prestosql/plugin/tpcds/TpcdsPlugin.java @@ -16,7 +16,15 @@ package io.prestosql.plugin.tpcds; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "TPC-DS : TPC Benchmark™ DS, data is generated on the fly", + docLink = "https://openlookeng.io/docs/docs/connector/tpcds.html", + configLink = "https://openlookeng.io/docs/docs/connector/tpcds.html#configuration") public class TpcdsPlugin implements Plugin { @@ -25,4 +33,11 @@ public class TpcdsPlugin { return ImmutableList.of(new TpcdsConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = TpcdsPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, ImmutableList.of()); + } } diff --git a/presto-tpch/src/main/java/io/prestosql/plugin/tpch/TpchPlugin.java b/presto-tpch/src/main/java/io/prestosql/plugin/tpch/TpchPlugin.java index 35051c0c4..bfae8a7d0 100644 --- a/presto-tpch/src/main/java/io/prestosql/plugin/tpch/TpchPlugin.java +++ b/presto-tpch/src/main/java/io/prestosql/plugin/tpch/TpchPlugin.java @@ -16,7 +16,15 @@ package io.prestosql.plugin.tpch; import com.google.common.collect.ImmutableList; import io.prestosql.spi.Plugin; import io.prestosql.spi.connector.ConnectorFactory; +import io.prestosql.spi.function.ConnectorConfig; +import io.prestosql.spi.queryeditorui.ConnectorUtil; +import io.prestosql.spi.queryeditorui.ConnectorWithProperties; +import java.util.Optional; + +@ConnectorConfig(connectorLabel = "TPC-H : TPC Benchmark™ H, data is generated on the fly", + docLink = "https://openlookeng.io/docs/docs/connector/tpch.html", + configLink = "https://openlookeng.io/docs/docs/connector/tpch.html#configuration") public class TpchPlugin implements Plugin { @@ -25,4 +33,11 @@ public class TpchPlugin { return ImmutableList.of(new TpchConnectorFactory()); } + + @Override + public Optional getConnectorWithProperties() + { + ConnectorConfig connectorConfig = TpchPlugin.class.getAnnotation(ConnectorConfig.class); + return ConnectorUtil.assembleConnectorProperties(connectorConfig, ImmutableList.of()); + } }