jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!521 Add auto catalog configuration for webui dynamic catalog adding 

Merge pull request !521 from Peikun Chen/feature/webui
This commit is contained in:
i-robot 2021-01-30 11:02:14 +08:00 committed by Gitee
parent f2e7d3667c 68e87e0491
commit 35400ef6f5
43 changed files with 832 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hetu-carbondata/src/main/java/io/hetu/core/plugin/carbondata/CarbondataPlugin.java
View File

@ -15,9 +15,22 @@ package io.hetu.core.plugin.carbondata;
import com.google.common.collect.ImmutableList;
import io.prestosql.plugin.hive.HivePlugin;
import io.prestosql.plugin.hive.metastore.thrift.StaticMetastoreConfig;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "Carbondata: Query data stored in a Carbondata warehouse",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html",
configLink = "https://openlookeng.io/zh-cn/docs/docs/connector/carbondata.html#configuration")
public class CarbondataPlugin
extends HivePlugin
{
@ -36,4 +49,12 @@ public class CarbondataPlugin
{
return ImmutableList.of(new CarbondataConnectorFactory("carbondata", getClassLoader()));
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = CarbondataPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(StaticMetastoreConfig.class.getDeclaredMethods()));
}
}

9
hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterConfig.java
View File

@ -20,6 +20,7 @@ import io.airlift.configuration.ConfigDescription;
import io.airlift.configuration.ConfigSecuritySensitive;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import io.prestosql.spi.function.Mandatory;
import javax.annotation.Nullable;
import javax.validation.constraints.NotNull;
@ -143,6 +144,10 @@ public class DataCenterConfig
* @param connectionUrl the connection url of data center.
* @return DataCenterConfig object.
*/
@Mandatory(name = "connection-url",
description = "The connection URL of remote OpenLooKeng data center",
defaultValue = "http://host:port",
required = true)
@Config("connection-url")
public DataCenterConfig setConnectionUrl(URI connectionUrl)
{
@ -239,6 +244,10 @@ public class DataCenterConfig
* @param connectionUser the connection user name.
* @return DataCenterConfig object.
*/
@Mandatory(name = "connection-user",
description = "User to connect to remote data center",
defaultValue = "lk",
required = true)
@Config("connection-user")
public DataCenterConfig setConnectionUser(String connectionUser)
{

20
hetu-datacenter/src/main/java/io/hetu/core/plugin/datacenter/DataCenterPlugin.java
View File

@ -18,12 +18,24 @@ package io.hetu.core.plugin.datacenter;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
/**
* Data center plugin.
*
* @since 2020-02-11
*/
@ConnectorConfig(connectorLabel = "DataCenter: Query data on remote OpenLooKeng data center",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/datacenter.html",
configLink = "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration")
public class DataCenterPlugin
implements Plugin
{
@ -32,4 +44,12 @@ public class DataCenterPlugin
{
return ImmutableList.of(new DataCenterConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = DataCenterPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(DataCenterConfig.class.getDeclaredMethods()));
}
}

76
hetu-docs/en/admin/web-interface.md
View File

@ -24,79 +24,3 @@ For more detailed information about a query, simply click the query ID link. The
The summary section has a button to kill the currently running query. There are two visualizations available in the summary section: task execution and timeline. The full JSON document containing information
and statistics about the query is available by clicking the *JSON* link. These visualizations and other statistics can be used to analyze where time is being spent for a query.
## Connector Properties File Configuration
The web interface supports expanding and adding new connectors in the interface by setting the connector properties file.The connector properties file can be set by referring to the template`presto-main/etc/connector_properties.json.template`.
`etc/connector_properties.json` is the default path of the web connector configuration file，you can change the path by set and modify `hetu.queryeditor-ui.server.connector-properties-json` property of `etc/config.properties`.
The web connector configuration file supports the following properties:
| Property Name | Description| Required |
|----------|----------|----------|
| `connectorWithProperties` | Yes | Connector with properties |
| `user` | Yes | The user who created the connector |
| `uuid` | Yes | Connector unique identification |
| `docLink` | Yes | Link to official introduction document of the connector |
| `configLink` | Yes | Link to official configuration document of the connector |
The properties that `connectorWithProperties` supported are as follows:
| Property Name | Description| Required | Default Value |
|----------|----------|----------|----------|
| `connectorName`| Yes | Connector name | |
| `connectorLabel`| Yes | Connector label | |
| `propertiesEnabled` | No | Whether to enable the connector properties configuration panel or not | false |
| `catalogConfigFilesEnabled` | No | Whether to enable properties file upload for current connector configuration or not | false |
| `globalConfigFilesEnabled` | No | Whether to enable properties file upload for system configuration (system configuration are shared by all connectors) or not | false |
| `properties` | Yes | Connector properties | |
The properties that `properties` supported are as follows:
| Property Name | Description| Required | Default Value |
|----------|----------|----------|----------|
| `name`| Yes | Property name | |
| `value`| Yes | Property value | |
| `description` | Yes | Property description | |
| `required` | No | Property required | false |
| `readOnly` | No | If the read-only mode is enabled, the value modification is not supported | false |
| `type` | Yes | Property type | |
Example, datacenter connector properties configuration of the connector properties file is as follow：
``` json
[
{
"connectorWithProperties": {
"connectorName": "dc",
"connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center",
"propertiesEnabled": true,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": [
{
"name": "connection-url",
"value": "http://localhost:8080",
"description": "The connection URL of remote OpenLooKeng data center",
"required" : true,
"type" : "string"
},
{
"name": "connection-user",
"value": "lk",
"description": "User to connect to remote data center",
"required" : true,
"readOnly": true,
"type": "string"
}
]
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e",
"docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html",
"configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration"
},
......
]
```

76
hetu-docs/zh/admin/web-interface.md
View File

@ -21,79 +21,3 @@ openLooKeng提供了一个用于监视和管理查询的Web界面。Web界面可
有关查询的详细信息，请单击查询ID链接。查询详细信息页有一个摘要部分、查询的各个阶段的图形表示和任务列表。可以点击每个任务ID以获得有关该任务的更多信息。
摘要部分有一个按钮，用于终止当前正在运行的查询。在摘要部分有两个可视化：任务执行和时间线。通过单击JSON链接，可以获得包含有关查询的信息和统计信息的完整JSON文档。这些可视化和其他统计信息可用于分析查询所花费的时间。
## 连接器属性文件设置
Web界面支持通过连接器属性文件设置的方式在界面拓展和添加新的连接器，连接器属性文件可以参考项目提供的模板`presto-main/etc/connector_properties.json.template`进行设置。
系统默认使用`etc/connector_properties.json`作为Web界面连接器属性文件路径，可以通过设置`etc/config.properties`中的`hetu.queryeditor-ui.server.connector-properties-json`属性修改文件路径。
Web连接器配置文件支持配置的属性如下：
| 属性名称 | 是否必选 | 描述 |
|----------|----------|----------|
| `connectorWithProperties` | 是 | 连接器详细配置参数配置 |
| `user` | 是 | 建立连接器的用户 |
| `uuid` | 是 | 连接器唯一标识 |
| `docLink` | 是 | 连接器官方介绍文档链接 |
| `configLink` | 是 | 连接器属性配置文档链接 |
其中，`connectorWithProperties`支持配置的属性如下：
| 属性名称| 是否必选 | 描述 | 默认值 |
|----------|----------|----------|----------|
| `connectorName`| 是 | 连接器名 | |
| `connectorLabel`| 是 | 连接器描述 | |
| `propertiesEnabled` | 否 | 是否启用连接器属性配置面板，启用界面才支持配置连接器属性设置功能 | false |
| `catalogConfigFilesEnabled` | 否 | 是否启用当前连接器的配置文件上传功能 | false |
| `globalConfigFilesEnabled` | 否 | 是否启用系统配置文件（系统配置文件的配置项所有连接器共享）上传功能 | false |
| `properties` | 是 | 连接器属性配置项 | |
其中，`properties`支持配置的属性如下：
| 属性名称 | 是否必选 | 描述 | 默认值 |
|----------|----------|----------|----------|
| `name`| 是 | 属性名 | |
| `value`| 是 | 属性值 | |
| `description` | 是 | 属性描述 | |
| `required` | 否 | 该属性是否必填 | false |
| `readOnly` | 否 | 是否开启只读模式，启用的话在界面不支持修改属性值 | false |
| `type` | 是 | 配置项数据类型 | |
示例，连接器属性文件中配置DataCenter连接器属性：
``` json
[
{
"connectorWithProperties": {
"connectorName": "dc",
"connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center",
"propertiesEnabled": true,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": [
{
"name": "connection-url",
"value": "http://localhost:8080",
"description": "The connection URL of remote OpenLooKeng data center",
"required" : true,
"type" : "string"
},
{
"name": "connection-user",
"value": "lk",
"description": "User to connect to remote data center",
"required" : true,
"readOnly": true,
"type": "string"
}
]
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e",
"docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html",
"configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration"
},
......
]
```

26
hetu-hana/src/main/java/io/hetu/core/plugin/hana/HanaPlugin.java
View File

@ -14,13 +14,27 @@
*/
package io.hetu.core.plugin.hana;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcMetadataConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Optional;
/**
* HanaPlugin class
*
* @since 2019-07-10
*/
@ConnectorConfig(connectorLabel = "Hana: Query and create tables on an external Hana database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/hana.html",
configLink = "https://openlookeng.io/docs/docs/connector/hana.html#configuration")
public class HanaPlugin
extends JdbcPlugin
{
@ -32,4 +46,16 @@ public class HanaPlugin
// name of the connector and the module implementation
super("hana", new HanaClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = HanaPlugin.class.getAnnotation(ConnectorConfig.class);
ArrayList<Method> methods = new ArrayList<>();
methods.addAll(Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
methods.addAll(Arrays.asList(JdbcMetadataConfig.class.getDeclaredMethods()));
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig, methods);
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:sap://host:port");
return connectorWithProperties;
}
}

21
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/HBasePlugin.java
View File

@ -16,15 +16,28 @@ package io.hetu.core.plugin.hbase;
import com.google.common.collect.ImmutableList;
import com.google.inject.Module;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.connector.HBaseConnectorFactory;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
/**
* HBasePlugin
*
* @since 2020-03-18
*/
@ConnectorConfig(connectorLabel = "HBase: Query and create tables on an external Apache HBase instance",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/hbase.html",
configLink = "https://openlookeng.io/docs/docs/connector/hbase.html#configuration")
public class HBasePlugin
implements Plugin
{
@ -58,4 +71,12 @@ public class HBasePlugin
}
return classLoader;
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = HBasePlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(HBaseConfig.class.getDeclaredMethods()));
}
}

15
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java
View File

@ -15,6 +15,7 @@
package io.hetu.core.plugin.hbase.conf;
import io.airlift.configuration.Config;
import io.prestosql.spi.function.Mandatory;
/**
* hbase.properties
@ -133,6 +134,10 @@ public class HBaseConfig
return zkQuorum;
}
@Mandatory(name = "hbase.zookeeper.quorum",
description = "Zookeeper cluster address",
defaultValue = "host1,host2",
required = true)
@Config("hbase.zookeeper.quorum")
public void setZkQuorum(String zkQuorum)
{
@ -144,6 +149,10 @@ public class HBaseConfig
return zkClientPort;
}
@Mandatory(name = "hbase.zookeeper.property.clientPort",
description = "Zookeeper client port",
defaultValue = "2181",
required = true)
@Config("hbase.zookeeper.property.clientPort")
public void setZkClientPort(String zkClientPort)
{
@ -155,6 +164,9 @@ public class HBaseConfig
return zkZnodeParent;
}
@Mandatory(name = "hbase.zookeeper.znode.parent",
description = "Zookeeper znode parent of hbase",
defaultValue = "/hbase")
@Config("hbase.zookeeper.znode.parent")
public void setZkZnodeParent(String zkZnodeParent)
{
@ -166,6 +178,9 @@ public class HBaseConfig
return metastoreType;
}
@Mandatory(name = "hbase.metastore.type",
description = "The storage of hbase metadata",
defaultValue = "hetuMetastore")
@Config("hbase.metastore.type")
public void setMetastoreType(String metastoreType)
{

21
hetu-opengauss/src/main/java/io/hetu/core/plugin/opengauss/OpenGaussPlugin.java
View File

@ -15,8 +15,19 @@
package io.hetu.core.plugin.opengauss;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "OpenGauss : Query and create tables on an external OpenGauss database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/opengauss.html",
configLink = "https://openlookeng.io/docs/docs/connector/opengauss.html#configuration")
public class OpenGaussPlugin
extends JdbcPlugin
{
@ -24,4 +35,14 @@ public class OpenGaussPlugin
{
super("opengauss", new OpenGaussClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = OpenGaussPlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:postgresql://host:port/database");
return connectorWithProperties;
}
}

22
hetu-oracle/src/main/java/io/hetu/core/plugin/oracle/OraclePlugin.java
View File

@ -15,14 +15,24 @@
package io.hetu.core.plugin.oracle;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
/**
* OraclePlugin
*
* @since 2019-07-06
*/
@ConnectorConfig(connectorLabel = "Oracle : Query and create tables on an external Oracle database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/oracle.html",
configLink = "https://openlookeng.io/docs/docs/connector/oracle.html#configuration")
public class OraclePlugin
extends JdbcPlugin
{
@ -34,4 +44,14 @@ public class OraclePlugin
// name of the connector and the module implementation
super("oracle", new OracleClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = OraclePlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:oracle:thin:@host:port/ORCLCDB");
return connectorWithProperties;
}
}

20
hetu-vdm/src/main/java/io/hetu/core/plugin/vdm/VdmPlugin.java
View File

@ -17,6 +17,12 @@ package io.hetu.core.plugin.vdm;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.isNullOrEmpty;
@ -26,6 +32,12 @@ import static com.google.common.base.Strings.isNullOrEmpty;
*
* @since 2020-02-27
*/
@ConnectorConfig(connectorLabel = "VDM : Virtualize Data Market",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/vdm.html",
configLink = "https://openlookeng.io/docs/docs/connector/vdm.html#usage")
public class VdmPlugin
implements Plugin
{
@ -56,4 +68,12 @@ public class VdmPlugin
{
return ImmutableList.of(new VdmConnectorFactory(name, VdmPlugin.class.getClassLoader()));
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = VdmPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(VdmConfig.class.getDeclaredMethods()));
}
}

9
presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java
View File

@ -18,6 +18,7 @@ import io.airlift.configuration.ConfigDescription;
import io.airlift.configuration.ConfigSecuritySensitive;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import io.prestosql.spi.function.Mandatory;
import javax.annotation.Nullable;
import javax.validation.constraints.Min;
@ -267,6 +268,10 @@ public class BaseJdbcConfig
return connectionUser;
}
@Mandatory(name = "connection-user",
description = "User to connect to remote database",
defaultValue = "root",
required = true)
@Config("connection-user")
public BaseJdbcConfig setConnectionUser(String connectionUser)
{
@ -280,6 +285,10 @@ public class BaseJdbcConfig
return connectionPassword;
}
@Mandatory(name = "connection-password",
description = "Password of user to connect to remote database",
defaultValue = "secret",
required = true)
@Config("connection-password")
@ConfigSecuritySensitive
public BaseJdbcConfig setConnectionPassword(String connectionPassword)

4
presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcMetadataConfig.java
View File

@ -17,6 +17,7 @@ import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import io.prestosql.spi.function.Mandatory;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
@ -37,6 +38,9 @@ public class JdbcMetadataConfig
return allowDropTable;
}
@Mandatory(name = "allow-drop-table",
description = "Allow connector to drop tables",
defaultValue = "true")
@Config("allow-drop-table")
@ConfigDescription("Allow connector to drop tables")
public JdbcMetadataConfig setAllowDropTable(boolean allowDropTable)

20
presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchConfig.java
View File

@ -19,6 +19,7 @@ import io.airlift.configuration.ConfigSecuritySensitive;
import io.airlift.configuration.DefunctConfig;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import io.prestosql.spi.function.Mandatory;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
@ -78,6 +79,10 @@ public class ElasticsearchConfig
return host;
}
@Mandatory(name = "elasticsearch.host",
description = "Elasticsearch host",
defaultValue = "host",
required = true)
@Config("elasticsearch.host")
public ElasticsearchConfig setHost(String host)
{
@ -90,6 +95,9 @@ public class ElasticsearchConfig
return port;
}
@Mandatory(name = "elasticsearch.port",
description = "Elasticsearch port",
defaultValue = "9200")
@Config("elasticsearch.port")
public ElasticsearchConfig setPort(int port)
{
@ -103,6 +111,9 @@ public class ElasticsearchConfig
return defaultSchema;
}
@Mandatory(name = "elasticsearch.default-schema-name",
description = "Default schema name to use",
defaultValue = "default")
@Config("elasticsearch.default-schema-name")
@ConfigDescription("Default schema name to use")
public ElasticsearchConfig setDefaultSchema(String defaultSchema)
@ -118,6 +129,9 @@ public class ElasticsearchConfig
return scrollSize;
}
@Mandatory(name = "elasticsearch.scroll-size",
description = "Scroll batch size",
defaultValue = "1000")
@Config("elasticsearch.scroll-size")
@ConfigDescription("Scroll batch size")
public ElasticsearchConfig setScrollSize(int scrollSize)
@ -132,6 +146,9 @@ public class ElasticsearchConfig
return scrollTimeout;
}
@Mandatory(name = "elasticsearch.scroll-timeout",
description = "Scroll timeout",
defaultValue = "1m")
@Config("elasticsearch.scroll-timeout")
@ConfigDescription("Scroll timeout")
public ElasticsearchConfig setScrollTimeout(Duration scrollTimeout)
@ -146,6 +163,9 @@ public class ElasticsearchConfig
return requestTimeout;
}
@Mandatory(name = "elasticsearch.request-timeout",
description = "Elasticsearch request timeout",
defaultValue = "2s")
@Config("elasticsearch.request-timeout")
@ConfigDescription("Elasticsearch request timeout")
public ElasticsearchConfig setRequestTimeout(Duration requestTimeout)

20
presto-elasticsearch/src/main/java/io/prestosql/elasticsearch/ElasticsearchPlugin.java
View File

@ -17,9 +17,21 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
import static java.util.Objects.requireNonNull;
@ConnectorConfig(connectorLabel = "Elasticsearch: Allow access to Elasticsearch data from openLooKeng",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/elasticsearch.html",
configLink = "https://openlookeng.io/docs/docs/connector/elasticsearch.html#configuration")
public class ElasticsearchPlugin
implements Plugin
{
@ -41,4 +53,12 @@ public class ElasticsearchPlugin
{
return ImmutableList.of(connectorFactory);
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = ElasticsearchPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(ElasticsearchConfig.class.getDeclaredMethods()));
}
}

7
presto-hive/src/main/java/io/prestosql/plugin/hive/HiveConfig.java
View File

@ -26,6 +26,8 @@ import io.airlift.units.MinDataSize;
import io.airlift.units.MinDuration;
import io.prestosql.orc.OrcWriteValidation.OrcWriteValidationMode;
import io.prestosql.plugin.hive.s3.S3FileSystemType;
import io.prestosql.spi.function.Mandatory;
import io.prestosql.spi.queryeditorui.PropertyType;
import org.joda.time.DateTimeZone;
import javax.annotation.Nullable;
@ -551,6 +553,11 @@ public class HiveConfig
return resourceConfigFiles;
}
@Mandatory(name = "hive.config.resources",
description = "An optional comma-separated list of HDFS configuration files. These files must exist on the machines running openLooKeng. Only specify this if absolutely necessary to access HDFS. Ensure to upload these files.",
defaultValue = "core-site.xml,hdfs-site.xml",
readOnly = true,
type = PropertyType.FILES)
@Config("hive.config.resources")
public HiveConfig setResourceConfigFiles(String files)
{

23
presto-hive/src/main/java/io/prestosql/plugin/hive/HivePlugin.java
View File

@ -15,15 +15,28 @@ package io.prestosql.plugin.hive;
import com.google.common.collect.ImmutableList;
import io.prestosql.plugin.hive.metastore.HiveMetastore;
import io.prestosql.plugin.hive.metastore.thrift.StaticMetastoreConfig;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.util.Objects.requireNonNull;
@ConnectorConfig(connectorLabel = "Hive: Query data stored in a Hive data warehouse",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/hive.html",
configLink = "https://openlookeng.io/docs/docs/connector/hive.html#configuration")
public class HivePlugin
implements Plugin
{
@ -47,4 +60,14 @@ public class HivePlugin
{
return ImmutableList.of(new HiveConnectorFactory(name, HivePlugin.class.getClassLoader(), metastore));
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = HivePlugin.class.getAnnotation(ConnectorConfig.class);
ArrayList<Method> methods = new ArrayList<>();
methods.addAll(Arrays.asList(StaticMetastoreConfig.class.getDeclaredMethods()));
methods.addAll(Arrays.asList(HiveConfig.class.getDeclaredMethods()));
return ConnectorUtil.assembleConnectorProperties(connectorConfig, methods);
}
}

5
presto-hive/src/main/java/io/prestosql/plugin/hive/metastore/thrift/StaticMetastoreConfig.java
View File

@ -17,6 +17,7 @@ import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import io.prestosql.spi.function.Mandatory;
import javax.validation.constraints.NotNull;
@ -38,6 +39,10 @@ public class StaticMetastoreConfig
return metastoreUris;
}
@Mandatory(name = "hive.metastore.uri",
description = "The URI(s) of the Hive metastore to connect to using the Thrift protocol. If multiple URIs are provided, the first URI is used by default and the rest of the URIs are fallback metastores. This property is required. Example: thrift://192.0.2.3:9083 or thrift://192.0.2.3:9083,thrift://192.0.2.4:9083",
defaultValue = "thrift://host:port",
required = true)
@Config("hive.metastore.uri")
@ConfigDescription("Hive metastore URIs (comma separated)")
public StaticMetastoreConfig setMetastoreUris(String uris)

18
presto-jmx/src/main/java/io/prestosql/plugin/jmx/JmxPlugin.java
View File

@ -16,7 +16,17 @@ package io.prestosql.plugin.jmx;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "JMX: Query JMX information from all nodes in a openLooKeng cluster",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/jmx.html",
configLink = "https://openlookeng.io/docs/docs/connector/jmx.html#configuration")
public class JmxPlugin
implements Plugin
{
@ -25,4 +35,12 @@ public class JmxPlugin
{
return ImmutableList.of(new JmxConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = JmxPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(JmxConnectorConfig.class.getDeclaredMethods()));
}
}

12
presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaConnectorConfig.java
View File

@ -21,6 +21,7 @@ import io.airlift.units.DataSize.Unit;
import io.airlift.units.Duration;
import io.airlift.units.MinDuration;
import io.prestosql.spi.HostAddress;
import io.prestosql.spi.function.Mandatory;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;
@ -88,6 +89,10 @@ public class KafkaConnectorConfig
return tableNames;
}
@Mandatory(name = "kafka.table-names",
description = "List of all tables provided by the catalog",
defaultValue = "table1,table2",
required = true)
@Config("kafka.table-names")
public KafkaConnectorConfig setTableNames(String tableNames)
{
@ -101,6 +106,9 @@ public class KafkaConnectorConfig
return defaultSchema;
}
@Mandatory(name = "kafka.default-schema",
description = "Default schema name to use",
defaultValue = "default")
@Config("kafka.default-schema")
public KafkaConnectorConfig setDefaultSchema(String defaultSchema)
{
@ -114,6 +122,10 @@ public class KafkaConnectorConfig
return nodes;
}
@Mandatory(name = "kafka.nodes",
description = "List of nodes in the Kafka cluster ",
defaultValue = "host1:port,host2:port",
required = true)
@Config("kafka.nodes")
public KafkaConnectorConfig setNodes(String nodes)
{

18
presto-kafka/src/main/java/io/prestosql/plugin/kafka/KafkaPlugin.java
View File

@ -18,7 +18,11 @@ import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
@ -28,6 +32,12 @@ import static java.util.Objects.requireNonNull;
/**
* Presto plugin to use Apache Kafka as a data source.
*/
@ConnectorConfig(connectorLabel = "Kafka: Allow the use of Apache Kafka topics as tables in openLooKeng",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/kafka.html",
configLink = "https://openlookeng.io/docs/docs/connector/kafka.html#configuration")
public class KafkaPlugin
implements Plugin
{
@ -44,4 +54,12 @@ public class KafkaPlugin
{
return ImmutableList.of(new KafkaConnectorFactory(tableDescriptionSupplier));
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = KafkaPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(KafkaConnectorConfig.class.getDeclaredMethods()));
}
}

18
presto-local-file/src/main/java/io/prestosql/plugin/localfile/LocalFilePlugin.java
View File

@ -16,7 +16,17 @@ package io.prestosql.plugin.localfile;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "LocalFile: Query data stored on the local file system of each worker",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/localfile.html",
configLink = "https://openlookeng.io/docs/docs/connector/localfile.html#configuration")
public class LocalFilePlugin
implements Plugin
{
@ -25,4 +35,12 @@ public class LocalFilePlugin
{
return ImmutableList.of(new LocalFileConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = LocalFilePlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(LocalFileConfig.class.getDeclaredMethods()));
}
}

103
presto-main/etc/connector_properties.json.template
View File

@ -1,103 +0,0 @@
[
{
"connectorWithProperties": {
"connectorName": "tpch",
"connectorLabel": "TPC-H : TPC Benchmark™ H, data is generated on the fly",
"propertiesEnabled": false,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": []
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9a",
"docLink": "https://openlookeng.io/docs/docs/connector/tpch.html",
"configLink": "https://openlookeng.io/docs/docs/connector/tpch.html#configuration"
},
{
"connectorWithProperties": {
"connectorName": "tpcds",
"connectorLabel": "TPC-DS : TPC Benchmark™ DS, data is generated on the fly",
"propertiesEnabled": false,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": []
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9b",
"docLink": "https://openlookeng.io/docs/docs/connector/tpcds.html",
"configLink": "https://openlookeng.io/docs/docs/connector/tpcds.html#configuration"
},
{
"connectorWithProperties": {
"connectorName": "memory",
"connectorLabel": "Memory: All data and metadata is only stored in memory",
"propertiesEnabled": false,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": []
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9d",
"docLink": "https://openlookeng.io/docs/docs/connector/memory.html",
"configLink": "https://openlookeng.io/docs/docs/connector/memory.html#configuration"
},
{
"connectorWithProperties": {
"connectorName": "hive-hadoop2",
"connectorLabel": "Hive-Hadoop2: Data stored in Hive data warehouse / HDFS",
"propertiesEnabled": true,
"catalogConfigFilesEnabled": true,
"globalConfigFilesEnabled": true,
"properties": [
{
"name": "hive.metastore.uri",
"value": "thrift://localhost:9083",
"description": "The URI(s) of the Hive metastore to connect to using the Thrift protocol.\nIf multiple URIs are provided, the first URI is used by default and the rest of the URIs are fallback metastores.\nThis property is required.\nExample: thrift://192.0.2.3:9083 or thrift://192.0.2.3:9083,thrift://192.0.2.4:9083",
"required" : true,
"type" : "string"
},
{
"name": "hive.config.resources",
"value": "core-site.xml,hdfs-site.xml",
"description": "An optional comma-separated list of HDFS configuration files.\nThese files must exist on the machines running openLooKeng.\nOnly specify this if absolutely necessary to access HDFS.\n\nEnsure to upload these files.",
"readOnly": true,
"type": "files"
}
]
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9c",
"docLink": "https://openlookeng.io/docs/docs/connector/hive.html",
"configLink": "https://openlookeng.io/docs/docs/connector/hive.html#configuration"
},
{
"connectorWithProperties": {
"connectorName": "dc",
"connectorLabel": "DataCenter: Query data in remote OpenLooKeng data center",
"propertiesEnabled": true,
"catalogConfigFilesEnabled": false,
"globalConfigFilesEnabled": false,
"properties": [
{
"name": "connection-url",
"value": "http://localhost:8080",
"description": "The connection URL of remote OpenLooKeng data center",
"required" : true,
"type" : "string"
},
{
"name": "connection-user",
"value": "lk",
"description": "User to connect to remote data center",
"required" : true,
"readOnly": true,
"type": "string"
}
]
},
"user": "lk",
"uuid": "ff35a9cf-8cfc-4ff5-be2f-a6389c144f9e",
"docLink": "https://openlookeng.io/docs/docs/connector/datacenter.html",
"configLink": "https://openlookeng.io/docs/docs/connector/datacenter.html#configuration"
}
]

13
presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorConfig.java
View File

@ -31,7 +31,6 @@ public class QueryEditorConfig
private String coordinatorUri;
private String featuredQueriesPath = "etc/featured_queries.json";
private String userQueriesPath = "etc/user_queries.json";
private String connectorsListPath = "etc/connector_properties.json";
private int maxResultCount = 1000;
private DataSize maxResultSize = new DataSize(1, DataSize.Unit.GIGABYTE);
private Optional<String> sharedSecret = Optional.empty();
@ -106,23 +105,11 @@ public class QueryEditorConfig
return this;
}
@Config("hetu.queryeditor-ui.server.connector-properties-json")
public QueryEditorConfig setConnectorsListPath(String connectorsListPath)
{
this.connectorsListPath = connectorsListPath;
return this;
}
public String getUserQueriesPath()
{
return userQueriesPath;
}
public String getConnectorsListPath()
{
return connectorsListPath;
}
@NotNull
public Optional<String> getSharedSecret()
{

7
presto-main/src/main/java/io/prestosql/queryeditorui/QueryEditorUIModule.java
View File

@ -50,7 +50,6 @@ import io.prestosql.queryeditorui.resources.TablesResource;
import io.prestosql.queryeditorui.resources.UIExecuteResource;
import io.prestosql.queryeditorui.resources.UserResource;
import io.prestosql.queryeditorui.security.UiAuthenticator;
import io.prestosql.queryeditorui.store.connectors.ConnectorCache;
import io.prestosql.queryeditorui.store.files.ExpiringFileStore;
import io.prestosql.queryeditorui.store.history.JobHistoryStore;
import io.prestosql.queryeditorui.store.history.LocalJobHistoryStore;
@ -280,10 +279,4 @@ public class QueryEditorUIModule
{
return new InMemoryQueryStore(new File(queryEditorConfig.getFeaturedQueriesPath()), new File(queryEditorConfig.getUserQueriesPath()));
}
@Provides
public ConnectorCache provideConnectorCache(QueryEditorConfig queryEditorConfig) throws IOException
{
return new ConnectorCache(new File(queryEditorConfig.getConnectorsListPath()));
}
}

14
presto-main/src/main/java/io/prestosql/queryeditorui/protocol/Connector.java
View File

@ -16,6 +16,7 @@
package io.prestosql.queryeditorui.protocol;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.UUID;
@ -26,9 +27,6 @@ public class Connector
@JsonProperty
private ConnectorWithProperties connectorWithProperties;
@JsonProperty
private String user;
@JsonProperty
private UUID uuid;
@ -38,12 +36,6 @@ public class Connector
@JsonProperty
private String configLink;
@JsonProperty
public String getUser()
{
return user;
}
@JsonProperty
public UUID getUuid()
{
@ -72,13 +64,11 @@ public class Connector
{
}
public Connector(@JsonProperty("user") String userName,
@JsonProperty("uuid") UUID uuid,
public Connector(@JsonProperty("uuid") UUID uuid,
@JsonProperty("docLink") String docLink,
@JsonProperty("configLink") String configLink,
@JsonProperty("connectorWithProperties") ConnectorWithProperties connectorWithProperties)
{
this.user = requireNonNull(userName, "userName is null");
this.connectorWithProperties = requireNonNull(connectorWithProperties, "Properties is null");
this.uuid = requireNonNull(uuid, "uuid is null");
this.docLink = requireNonNull(docLink, "docLink is null");

17
presto-main/src/main/java/io/prestosql/queryeditorui/resources/ConnectorResource.java
View File

@ -14,9 +14,6 @@
*/
package io.prestosql.queryeditorui.resources;
import com.google.inject.Inject;
import io.prestosql.queryeditorui.protocol.Connector;
import io.prestosql.queryeditorui.security.UiAuthenticator;
import io.prestosql.queryeditorui.store.connectors.ConnectorCache;
import javax.servlet.http.HttpServletRequest;
@ -27,25 +24,13 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import java.util.List;
@Path("/api/connectors")
public class ConnectorResource
{
private final ConnectorCache connectorCache;
@Inject
public ConnectorResource(ConnectorCache connectorCache)
{
this.connectorCache = connectorCache;
}
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response getConnectors(@Context HttpServletRequest servletRequest)
{
String user = UiAuthenticator.getUser(servletRequest);
List<Connector> lk = connectorCache.getConnectors(user);
return Response.ok(lk).build();
return Response.ok(ConnectorCache.getConnectors()).build();
}
}

61
presto-main/src/main/java/io/prestosql/queryeditorui/store/connectors/ConnectorCache.java
View File

@ -12,60 +12,41 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.queryeditorui.store.connectors;
import io.airlift.json.JsonCodec;
import io.airlift.json.JsonCodecFactory;
import io.airlift.json.ObjectMapperProvider;
import io.prestosql.queryeditorui.protocol.Connector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.Optional;
import java.util.UUID;
import static java.util.Collections.unmodifiableList;
public class ConnectorCache
{
private static final Logger LOG = LoggerFactory.getLogger(ConnectorCache.class);
private ConnectorCache() {}
static final JsonCodec<List<Connector>> CONNECTORS_CODEC =
new JsonCodecFactory(new ObjectMapperProvider()).listJsonCodec(Connector.class);
private List<Connector> connectorsList = new ArrayList<>();
private static final List<Connector> CONNECTORS_CACHE = new ArrayList<>();
public ConnectorCache(File connectorsJsonPath) throws IOException
public static void addCatalogConfig(Plugin plugin, String connectorName)
{
loadConnectorsAndProperties(connectorsJsonPath);
Optional<ConnectorWithProperties> connectorWithProperties = plugin.getConnectorWithProperties();
if (connectorWithProperties.isPresent() && connectorName != null) {
connectorWithProperties.get().setConnectorName(connectorName);
Connector connector = new Connector(
UUID.randomUUID(),
connectorWithProperties.get().getDocLink(),
connectorWithProperties.get().getConfigLink(),
connectorWithProperties.get());
CONNECTORS_CACHE.add(connector);
}
}
public List<Connector> getConnectors(String user)
public static List<Connector> getConnectors()
{
return connectorsList.stream()
.filter(q -> q.getUser().equals(user))
.collect(Collectors.toList());
}
private synchronized void loadConnectorsAndProperties(File connectorsJsonPath) throws IOException
{
if (!connectorsJsonPath.exists()) {
return;
}
try (FileInputStream fin = new FileInputStream(connectorsJsonPath)) {
StringBuilder jsonString = new StringBuilder();
byte[] buffer = new byte[4096];
int n;
while ((n = fin.read(buffer)) > 0) {
jsonString.append(new String(buffer, 0, n));
}
connectorsList.addAll(CONNECTORS_CODEC.fromJson(jsonString.toString()));
}
catch (IOException e) {
LOG.error("Connectors properties file not found", e);
throw e;
}
return unmodifiableList(CONNECTORS_CACHE);
}
}

2
presto-main/src/main/java/io/prestosql/server/PluginManager.java
View File

@ -26,6 +26,7 @@ import io.prestosql.filesystem.FileSystemClientManager;
import io.prestosql.heuristicindex.HeuristicIndexerManager;
import io.prestosql.metadata.MetadataManager;
import io.prestosql.metastore.HetuMetaStoreManager;
import io.prestosql.queryeditorui.store.connectors.ConnectorCache;
import io.prestosql.security.AccessControlManager;
import io.prestosql.seedstore.SeedStoreManager;
import io.prestosql.server.security.PasswordAuthenticatorManager;
@ -260,6 +261,7 @@ public class PluginManager
for (ConnectorFactory connectorFactory : plugin.getConnectorFactories()) {
log.info("Registering connector %s", connectorFactory.getName());
connectorManager.addConnectorFactory(connectorFactory);
ConnectorCache.addCatalogConfig(plugin, connectorFactory.getName());
}
for (SessionPropertyConfigurationManagerFactory sessionConfigFactory : plugin.getSessionPropertyConfigurationManagerFactories()) {

5
presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryConfig.java
View File

@ -15,6 +15,7 @@ package io.prestosql.plugin.memory;
import io.airlift.configuration.Config;
import io.airlift.units.DataSize;
import io.prestosql.spi.function.Mandatory;
import javax.validation.constraints.NotNull;
@ -42,6 +43,10 @@ public class MemoryConfig
return maxDataPerNode;
}
@Mandatory(name = "memory.max-data-per-node",
description = "Define memory limit for pages stored in this connector per each node",
defaultValue = "128MB",
required = true)
@Config("memory.max-data-per-node")
public MemoryConfig setMaxDataPerNode(DataSize maxDataPerNode)
{

19
presto-memory/src/main/java/io/prestosql/plugin/memory/MemoryPlugin.java
View File

@ -16,12 +16,31 @@ package io.prestosql.plugin.memory;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "Memory : Stores all data and metadata in RAM on workers",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/memory.html",
configLink = "https://openlookeng.io/docs/docs/connector/memory.html#configuration")
public final class MemoryPlugin
implements Plugin
{
@Override
public Iterable<ConnectorFactory> getConnectorFactories()
{
return ImmutableList.of(new MemoryConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = MemoryPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(MemoryConfig.class.getDeclaredMethods()));
}
}

21
presto-mysql/src/main/java/io/prestosql/plugin/mysql/MySqlPlugin.java
View File

@ -13,8 +13,19 @@
*/
package io.prestosql.plugin.mysql;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "MySql : Query and create tables on an external MySql database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/mysql.html",
configLink = "https://openlookeng.io/docs/docs/connector/mysql.html#configuration")
public class MySqlPlugin
extends JdbcPlugin
{
@ -22,4 +33,14 @@ public class MySqlPlugin
{
super("mysql", new MySqlClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = MySqlPlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:mysql://host:port");
return connectorWithProperties;
}
}

21
presto-postgresql/src/main/java/io/prestosql/plugin/postgresql/PostgreSqlPlugin.java
View File

@ -13,8 +13,19 @@
*/
package io.prestosql.plugin.postgresql;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "PostgreSQL : Query and create tables on an external PostgreSQL database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/postgresql.html",
configLink = "https://openlookeng.io/docs/docs/connector/postgresql.html#configuration")
public class PostgreSqlPlugin
extends JdbcPlugin
{
@ -22,4 +33,14 @@ public class PostgreSqlPlugin
{
super("postgresql", new PostgreSqlClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = PostgreSqlPlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:postgresql://host:port/database");
return connectorWithProperties;
}
}

7
presto-spi/src/main/java/io/prestosql/spi/Plugin.java
View File

@ -19,6 +19,7 @@ import io.prestosql.spi.eventlistener.EventListenerFactory;
import io.prestosql.spi.filesystem.HetuFileSystemClientFactory;
import io.prestosql.spi.heuristicindex.IndexFactory;
import io.prestosql.spi.metastore.HetuMetaStoreFactory;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import io.prestosql.spi.resourcegroups.ResourceGroupConfigurationManagerFactory;
import io.prestosql.spi.security.PasswordAuthenticatorFactory;
import io.prestosql.spi.security.SystemAccessControlFactory;
@ -30,6 +31,7 @@ import io.prestosql.spi.type.ParametricType;
import io.prestosql.spi.type.Type;
import java.io.File;
import java.util.Optional;
import java.util.Set;
import static java.util.Collections.emptyList;
@ -122,6 +124,11 @@ public interface Plugin
return emptyList();
}
default Optional<ConnectorWithProperties> getConnectorWithProperties()
{
return Optional.empty();
}
default void setExternalFunctionsDir(File externalFuncsDir)
{}

58
presto-spi/src/main/java/io/prestosql/spi/function/ConnectorConfig.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.spi.function;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.TYPE)
public @interface ConnectorConfig
{
/**
* Connector label
*/
String connectorLabel() default "";
/**
* Whether to enable the connector properties configuration panel or not
*/
boolean propertiesEnabled() default false;
/**
* Whether to enable properties file upload for current connector configuration or not
*/
boolean catalogConfigFilesEnabled() default false;
/**
* Whether to enable properties file upload for system configuration (system configuration are shared by all connectors) or not
*/
boolean globalConfigFilesEnabled() default false;
/**
* Link to official introduction document of the connector
*/
String docLink() default "";
/**
* Link to official configuration document of the connector
*/
String configLink() default "";
}

60
presto-spi/src/main/java/io/prestosql/spi/function/Mandatory.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.spi.function;
import io.prestosql.spi.queryeditorui.PropertyType;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface Mandatory
{
/**
* Property name
*/
String name() default "";
/**
* Property value
*/
String defaultValue() default "";
/**
* Property description
*/
String description() default "";
/**
* Property required
*/
boolean required() default false;
/**
* If the read-only mode is enabled, the value modification is not supported
*/
boolean readOnly() default false;
/**
* Property type
*/
PropertyType type() default PropertyType.STRING;
}

74
presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorUtil.java Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.spi.queryeditorui;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.function.Mandatory;
import java.lang.reflect.Method;
import java.util.Optional;
public class ConnectorUtil
{
private ConnectorUtil()
{
}
public static Optional<ConnectorWithProperties> assembleConnectorProperties(ConnectorConfig connectorConfig, Iterable<Method> methods)
{
if (connectorConfig == null) {
return Optional.empty();
}
else {
ConnectorWithProperties connectorWithProperties = new ConnectorWithProperties();
connectorWithProperties.setConnectorLabel(Optional.of(connectorConfig.connectorLabel()));
connectorWithProperties.setPropertiesEnabled(connectorConfig.propertiesEnabled());
connectorWithProperties.setCatalogConfigFilesEnabled(connectorConfig.catalogConfigFilesEnabled());
connectorWithProperties.setGlobalConfigFilesEnabled(connectorConfig.globalConfigFilesEnabled());
connectorWithProperties.setDocLink(connectorConfig.docLink());
connectorWithProperties.setConfigLink(connectorConfig.configLink());
for (Method method : methods) {
ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties();
Mandatory mandatory = method.getAnnotation(Mandatory.class);
if (mandatory != null) {
properties.setName(mandatory.name());
properties.setDescription(mandatory.description());
properties.setValue(mandatory.defaultValue());
properties.setRequired(Optional.of(mandatory.required()));
properties.setReadOnly(Optional.of(mandatory.readOnly()));
properties.setType(Optional.of(mandatory.type().stringValue()));
connectorWithProperties.addProperties(properties);
}
}
return Optional.of(connectorWithProperties);
}
}
public static void addConnUrlProperty(Optional<ConnectorWithProperties> connectorWithProperties, String conUrl)
{
if (connectorWithProperties.isPresent()) {
ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties();
properties.setName("connection-url");
properties.setDescription("The connection URL of remote database");
properties.setValue(conUrl);
properties.setRequired(Optional.of(true));
properties.setReadOnly(Optional.of(false));
properties.setType(Optional.of(PropertyType.STRING.stringValue()));
connectorWithProperties.get().addProperties(properties);
}
}
}

107
presto-main/src/main/java/io/prestosql/queryeditorui/protocol/ConnectorWithProperties.java → presto-spi/src/main/java/io/prestosql/spi/queryeditorui/ConnectorWithProperties.java
View File

@ -13,11 +13,12 @@
* limitations under the License.
*/
package io.prestosql.queryeditorui.protocol;
package io.prestosql.spi.queryeditorui;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@ -33,13 +34,19 @@ public class ConnectorWithProperties
@JsonProperty
private boolean propertiesEnabled;
@JsonProperty
private boolean catalogConfigFilesEnabled;
@JsonProperty
private boolean globalConfigFilesEnabled;
@JsonProperty
private List<Properties> connectorProperties;
private List<Properties> connectorProperties = new ArrayList<>();
private String docLink;
private String configLink;
public ConnectorWithProperties()
{
@ -61,6 +68,56 @@ public class ConnectorWithProperties
this.connectorProperties = requireNonNull(connectorProperties, "Properties is null");
}
public void setConnectorName(String connectorName)
{
this.connectorName = requireNonNull(connectorName, "connectorName is null");
}
public void setConnectorLabel(Optional<String> connectorLabel)
{
this.connectorLabel = requireNonNull(connectorLabel, "connectorLabel is null");
}
public void setPropertiesEnabled(boolean propertiesEnabled)
{
this.propertiesEnabled = propertiesEnabled;
}
public void setCatalogConfigFilesEnabled(boolean catalogConfigFilesEnabled)
{
this.catalogConfigFilesEnabled = catalogConfigFilesEnabled;
}
public void setGlobalConfigFilesEnabled(boolean globalConfigFilesEnabled)
{
this.globalConfigFilesEnabled = globalConfigFilesEnabled;
}
public void addProperties(Properties properties)
{
this.connectorProperties.add(properties);
}
public void setDocLink(String docLink)
{
this.docLink = docLink;
}
public void setConfigLink(String configLink)
{
this.configLink = configLink;
}
public String getDocLink()
{
return docLink;
}
public String getConfigLink()
{
return configLink;
}
@JsonProperty
public String getConnectorName()
{
@ -102,12 +159,16 @@ public class ConnectorWithProperties
public static class Properties
{
private final String name;
private final String value;
private final String description;
private final Optional<String> type;
private final Optional<Boolean> required;
private final Optional<Boolean> readOnly;
private String name;
private String value;
private String description;
private Optional<String> type;
private Optional<Boolean> required;
private Optional<Boolean> readOnly;
public Properties()
{
}
@JsonCreator
public Properties(
@ -126,6 +187,36 @@ public class ConnectorWithProperties
this.readOnly = readOnly;
}
public void setName(String name)
{
this.name = name;
}
public void setValue(String value)
{
this.value = value;
}
public void setDescription(String description)
{
this.description = description;
}
public void setType(Optional<String> type)
{
this.type = type;
}
public void setRequired(Optional<Boolean> required)
{
this.required = required;
}
public void setReadOnly(Optional<Boolean> readOnly)
{
this.readOnly = readOnly;
}
@JsonProperty
public String getName()
{

34
presto-spi/src/main/java/io/prestosql/spi/queryeditorui/PropertyType.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.spi.queryeditorui;
public enum PropertyType
{
STRING("string"),
FILES("files");
String propertyType;
PropertyType(String propertyType)
{
this.propertyType = propertyType;
}
public String stringValue()
{
return propertyType;
}
}

21
presto-sqlserver/src/main/java/io/prestosql/plugin/sqlserver/SqlServerPlugin.java
View File

@ -13,8 +13,19 @@
*/
package io.prestosql.plugin.sqlserver;
import io.prestosql.plugin.jdbc.BaseJdbcConfig;
import io.prestosql.plugin.jdbc.JdbcPlugin;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Arrays;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "SQL Server : Query and create tables on an external SQL Server database",
propertiesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/sqlserver.html",
configLink = "https://openlookeng.io/docs/docs/connector/sqlserver.html#configuration")
public class SqlServerPlugin
extends JdbcPlugin
{
@ -22,4 +33,14 @@ public class SqlServerPlugin
{
super("sqlserver", new SqlServerClientModule());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = SqlServerPlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
Arrays.asList(BaseJdbcConfig.class.getDeclaredMethods()));
ConnectorUtil.addConnUrlProperty(connectorWithProperties, "jdbc:sqlserver://[serverName[\\instanceName][:portNumber]]");
return connectorWithProperties;
}
}

29
presto-thrift/src/main/java/io/prestosql/plugin/thrift/ThriftPlugin.java
View File

@ -17,8 +17,12 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Module;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.List;
import java.util.Optional;
import java.util.ServiceLoader;
import static com.google.common.base.Preconditions.checkArgument;
@ -26,6 +30,12 @@ import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.Iterables.getOnlyElement;
import static java.util.Objects.requireNonNull;
@ConnectorConfig(connectorLabel = "Thrift : Connect to Thrift servers to integrate with external storage systems",
propertiesEnabled = true,
catalogConfigFilesEnabled = true,
globalConfigFilesEnabled = true,
docLink = "https://openlookeng.io/docs/docs/connector/thrift.html",
configLink = "https://openlookeng.io/docs/docs/connector/thrift.html#configuration")
public class ThriftPlugin
implements Plugin
{
@ -62,4 +72,23 @@ public class ThriftPlugin
List<ThriftPluginInfo> list = ImmutableList.copyOf(loader);
return list.isEmpty() ? new ThriftPluginInfo() : getOnlyElement(list);
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = ThriftPlugin.class.getAnnotation(ConnectorConfig.class);
Optional<ConnectorWithProperties> connectorWithProperties = ConnectorUtil.assembleConnectorProperties(connectorConfig,
ImmutableList.of());
if (connectorWithProperties.isPresent()) {
ConnectorWithProperties.Properties properties = new ConnectorWithProperties.Properties();
properties.setName("presto.thrift.client.addresses");
properties.setDescription("Location of Thrift servers");
properties.setValue("host1:port,host2:port");
properties.setRequired(Optional.of(true));
properties.setReadOnly(Optional.of(false));
properties.setType(Optional.of("string"));
connectorWithProperties.get().addProperties(properties);
}
return connectorWithProperties;
}
}

15
presto-tpcds/src/main/java/io/prestosql/plugin/tpcds/TpcdsPlugin.java
View File

@ -16,7 +16,15 @@ package io.prestosql.plugin.tpcds;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "TPC-DS : TPC Benchmark™ DS, data is generated on the fly",
docLink = "https://openlookeng.io/docs/docs/connector/tpcds.html",
configLink = "https://openlookeng.io/docs/docs/connector/tpcds.html#configuration")
public class TpcdsPlugin
implements Plugin
{
@ -25,4 +33,11 @@ public class TpcdsPlugin
{
return ImmutableList.of(new TpcdsConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = TpcdsPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig, ImmutableList.of());
}
}

15
presto-tpch/src/main/java/io/prestosql/plugin/tpch/TpchPlugin.java
View File

@ -16,7 +16,15 @@ package io.prestosql.plugin.tpch;
import com.google.common.collect.ImmutableList;
import io.prestosql.spi.Plugin;
import io.prestosql.spi.connector.ConnectorFactory;
import io.prestosql.spi.function.ConnectorConfig;
import io.prestosql.spi.queryeditorui.ConnectorUtil;
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
import java.util.Optional;
@ConnectorConfig(connectorLabel = "TPC-H : TPC Benchmark™ H, data is generated on the fly",
docLink = "https://openlookeng.io/docs/docs/connector/tpch.html",
configLink = "https://openlookeng.io/docs/docs/connector/tpch.html#configuration")
public class TpchPlugin
implements Plugin
{
@ -25,4 +33,11 @@ public class TpchPlugin
{
return ImmutableList.of(new TpchConnectorFactory());
}
@Override
public Optional<ConnectorWithProperties> getConnectorWithProperties()
{
ConnectorConfig connectorConfig = TpchPlugin.class.getAnnotation(ConnectorConfig.class);
return ConnectorUtil.assembleConnectorProperties(connectorConfig, ImmutableList.of());
}
}