Compare commits
17 Commits
master
...
branch-1.6
| Author | SHA1 | Date |
|---|---|---|
 i-robot
|
ba9b020cf3 | |
zhousipei
|
34d392d6ee | |
|
Raghunandan
|
ae03d9d24e | |
|
Raghunandan
|
897ec4f198 | |
|
i-robot
|
93155f4a7b | |
|
i-robot
|
c8a4f0ee03 | |
|
wyy566
|
69d9cc5a34 | |
|
i-robot
|
d4b1f2c0ad | |
|
guojunfei
|
f437b03bbb | |
|
i-robot
|
9b42fbaf38 | |
|
zhousipei
|
1290643b6a | |
|
i-robot
|
543c34d1fe | |
|
DOU
|
3f9d32c649 | |
|
DOU
|
64334f4bf6 | |
|
DOU
|
78f06ed7c4 | |
|
Raghunandan
|
120b8dcd8e | |
|
Raghunandan
|
bbc79dea2f |
|
|
@ -22,7 +22,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-carbondata</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-clickhouse</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-common</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-cube</artifactId>
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-datacenter</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Audit Log
|
||||
|
||||
openLooKeng audit logging functionality is a custom event listener, which monitors the start and stop of openLooKeng cluster and the dynamic addition and deletion of nodes in the cluster; Listen to WebUi user login and exit events; Listen for query events and call when the query is created and completed (success or failure).
|
||||
openLooKeng audit logging functionality is a custom event listener that is invoked for query creation and query completion (success or failure)
|
||||
An audit log contains the following information:
|
||||
|
||||
1. time when an event occurs
|
||||
|
|
@ -24,17 +24,15 @@ To enable audit logging feature, the following configs must be present in `etc/e
|
|||
hetu.event.listener.type=AUDIT
|
||||
hetu.event.listener.listen.query.creation=true
|
||||
hetu.event.listener.listen.query.completion=true
|
||||
hetu.auditlog.logoutput=/var/log/
|
||||
hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
|
||||
```
|
||||
|
||||
The following is a detailed description of audit logging properties:
|
||||
Other audit logging properties include:
|
||||
|
||||
`hetu.event.listener.type`: property to define logging type for audit files. Allowed values are AUDIT and LOGGER.
|
||||
`hetu.event.listener.audit.file`: Optional property to define absolute file path for the audit file. Ensure the process running the openLooKeng server has write access to this directory.
|
||||
|
||||
`hetu.auditlog.logoutput`: property to define absolute file directory for audit files. Ensure the process running the openLooKeng server has write access to this directory.
|
||||
`hetu.event.listener.audit.filecount`: Optional property to define the number of files to use
|
||||
|
||||
`hetu.auditlog.logconversionpattern`: property to define the conversion pattern of audit files. Allowed values are yyyy-MM-dd.HH and yyyy-MM-dd.
|
||||
`hetu.event.listener.audit.limit`: Optional property to define the maximum number of bytes to write to any one file
|
||||
|
||||
Example configuration file:
|
||||
|
||||
|
|
@ -43,6 +41,7 @@ event-listener.name=hetu-listener
|
|||
hetu.event.listener.type=AUDIT
|
||||
hetu.event.listener.listen.query.creation=true
|
||||
hetu.event.listener.listen.query.completion=true
|
||||
hetu.auditlog.logoutput=/var/log/
|
||||
hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
|
||||
hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
|
||||
hetu.event.listener.audit.filecount=1
|
||||
hetu.event.listener.audit.limit=100000
|
||||
```
|
||||
|
|
|
|||
|
|
@ -118,20 +118,6 @@ This section describes the most important config properties that may be used to
|
|||
>
|
||||
> This is the amount of memory set aside as headroom/buffer in the JVM heap for allocations that are not tracked by openLooKeng.
|
||||
|
||||
### `query.suspend-query-enabled`
|
||||
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `false`
|
||||
>
|
||||
> Enables running query temporary suspension when system is in low resource situation.
|
||||
|
||||
### `query.max-suspended-queries`
|
||||
|
||||
> - **Type:** `integer`
|
||||
> - **Default value:** `10`
|
||||
>
|
||||
> Maximum number of queries to attempt suspension before starting of killing the queries. This property comes in effect only if `query.suspend-query-enabled` is configured `true`
|
||||
|
||||
## Spilling Properties
|
||||
|
||||
### `experimental.spill-enabled`
|
||||
|
|
@ -378,8 +364,33 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
|
|||
>
|
||||
> Maximum size of a response returned from an exchange request. The response will be placed in the exchange client buffer which is shared across all concurrent requests for the exchange.
|
||||
>
|
||||
>
|
||||
>
|
||||
> Increasing the value may improve network throughput if there is high latency. Decreasing the value may improve query performance for large clusters as it reduces skew due to the exchange client buffer holding responses for more tasks (rather than hold more data from fewer tasks).
|
||||
|
||||
### `exchange.max-error-duration`
|
||||
|
||||
> - **Type:** `duration`
|
||||
> - **Minimum value:** `1m`
|
||||
> - **Default value:** `7m`
|
||||
>
|
||||
> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a failure.
|
||||
|
||||
### `exchange.is-timeout-failure-detection-enabled`
|
||||
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `true`
|
||||
>
|
||||
> The failure detection mechanism in use. Default is timeout based failure detection. Otherwise, i.e. when this property is set to false, maximum retry based failure detection mechanism is enabled.
|
||||
>
|
||||
|
||||
### `exchange.max-retry-count`
|
||||
|
||||
> - **Type:** `integer`
|
||||
> - **Default value:** `100`
|
||||
>
|
||||
> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status. If the remote node status is failed as per the failure detector module, it is considered as a permanent failure. This parameter is the minimum count which is required to decide, not necessarily the exact count. Based on the cluster size, load on the cluster the exact count may vary slightly. This property is used only when exchange.is-timeout-failure-detection-enabled is set to false. This value needs to be at least 100 to take effect.
|
||||
|
||||
### `sink.max-buffer-size`
|
||||
|
||||
> - **Type:** `data size`
|
||||
|
|
@ -387,113 +398,6 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
|
|||
>
|
||||
> Output buffer size for task data that is waiting to be pulled by upstream tasks. If the task output is hash partitioned, then the buffer will be shared across all of the partitioned consumers. Increasing this value may improve network throughput for data transferred between stages if the network has high latency or if there are many nodes in the cluster.
|
||||
|
||||
## Failure Recovery handling Properties
|
||||
|
||||
### Failure Retry Policies
|
||||
|
||||
### `failure.recovery.retry.profile`
|
||||
|
||||
> - **Type:** `String`
|
||||
> - **Default value:** `default`
|
||||
>
|
||||
> This property defines the failure detection profile used to determine if failure has happened for a http client. The value `<profile-name>` set for this property has to correspond to `<profile-name>.properties` file in `etc/failure-retry-policy/`. In case no such profile is available, and this property is not set, "default" profile is used.
|
||||
> For example, `failure.recovery.retry.profile="test"` requires `test.properties` file to be present in `etc/failure-retry-policy`.
|
||||
> The file `test.properties` must contain `failure.recovery.retry.type` specified.
|
||||
|
||||
|
||||
### `failure.recovery.retry.type`
|
||||
|
||||
> - **Type:** `String`
|
||||
> - **Default value:** `timeout`
|
||||
>
|
||||
> The failure detection mechanism in use. Default is timeout based failure detection.
|
||||
>
|
||||
#### `timeout` based failure detection.
|
||||
> Using this mechanism, HTTP client failures are retried for a specific duration before considering it as a permanent failure.
|
||||
> Additional properties `max.error.duration` can be defined for this type of failure detection.
|
||||
>
|
||||
#### `max-retry` based failure detection.
|
||||
> Using this mechanism, HTTP client failures are retried for a specific number of times before considering it as a permanent failure.
|
||||
> Additional properties `max.retry.count` and `max.error.duration` can be defined for this type of failure detection.
|
||||
> Using this type of failure detection is configured to be used, `max.retry.count` times retry is performed before consulting the failure detector module. When the remote node is failed as per the failure detector module, HTTP client considers it a permanent failure. Otherwise, i.e. When remote worker node is alive but not sending response, retry happens for `max.error.duration` before considering it as permanent failure.
|
||||
|
||||
### `max.error.duration`
|
||||
> - **Type:** `duration`
|
||||
> - **Default value:** `300s`
|
||||
>
|
||||
> The maximum amount of time coordinator waits for inter-task related errors to be resolved before it's considered a permanent failure.
|
||||
|
||||
|
||||
### `max.retry.count`
|
||||
|
||||
> - **Type:** `integer`
|
||||
> - **Default value:** `100`
|
||||
>
|
||||
> The maximum number of retry for failed task performed by the coordinator before consulting the failure detector module about the remote node status.
|
||||
> This parameter is the minimum count before consulting the failure detection module. Hence, the actual number of failures may vary slightly based on the cluster size, and load on the cluster.
|
||||
> This property is used only for `max-retry` based failure detection profiles.
|
||||
> The minimum value for this parameter is 100.
|
||||
|
||||
### Gossip Protocol Configurations for Failure Detection
|
||||
|
||||
### `failure-detection-protocol`
|
||||
|
||||
>- **Type:** String
|
||||
>- **Default value:** `heartbeat`
|
||||
>
|
||||
> This property defines the type of failure detector in use. Default configuration is `heartbeat` failure detector.
|
||||
> Gossip protocol can be enabled by specifying this parameter in `config.properties` file, with the value `gossip`.
|
||||
> All nodes (i.e. coordinator as well as workers) in a cluster should have this property specified in their respective `etc/config.properties` file.
|
||||
|
||||
### `failure-detector.heartbeat-interval`
|
||||
|
||||
>- **Type:** Duration
|
||||
>- **Default value:** `500ms` (500 miliseconds)
|
||||
>
|
||||
> This is the interval of gossip between two nodes in the cluster.
|
||||
> In gossip protocol, two workers are expected to gossip with higher frequency than the coordinator and a worker.
|
||||
> In `config.properties` for the coordinator, this property can be set with a reasonably higher value, such as `5s` (5 seconds).
|
||||
> In workers, this property can be left to use the default value.
|
||||
>
|
||||
### `failure-detector.worker-gossip-probe-interval`
|
||||
>
|
||||
> - **Type:** Duration
|
||||
>- **Default value:** `5s` (5 seconds)
|
||||
>
|
||||
> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
|
||||
> This property specifies the interval of refreshing the monitoring tasks to trigger worker to worker gossip.
|
||||
> This property, if needed to be configured with any other value than the default, should be specified only for the worker nodes.
|
||||
> This parameter should have higher value than `failure-detector.heartbeat-interval`.
|
||||
>
|
||||
### `failure-detector.coordinator-gossip-probe-interval`
|
||||
>
|
||||
> - **Type:** Duration
|
||||
>- **Default value:** `5s` (5 seconds)
|
||||
>
|
||||
> Gossip protocol uses monitoring tasks (same as the heartbeat failure detector) to keep tab on the other nodes.
|
||||
> This property specifies the interval of refreshing the monitoring tasks to trigger coordinator to worker gossip.
|
||||
> This property, if needed to be configured with any other value than the default, should be specified only for the coordinator.
|
||||
> This parameter should have higher value than `failure-detector.heartbeat-interval` and `failure-detector.worker-gossip-probe-interval`.
|
||||
>
|
||||
### `failure-detector.coordinator-gossip-collate-interval`
|
||||
>
|
||||
> - **Type:** Duration
|
||||
>- **Default value:** `2s` (2 seconds)
|
||||
>
|
||||
> This property specifies the interval in which the coordinator collates all the gossips it obtained from all the workers.
|
||||
> This property has to be specified only for the coordinator.
|
||||
> This parameter should have higher value than `failure-detector.heartbeat-interval`.
|
||||
>
|
||||
### `failure-detector.gossip-group-size`
|
||||
>
|
||||
> - **Type:** Integer
|
||||
>- **Default value:** `Integer.MAX_VALUE`
|
||||
>
|
||||
> A worker should gossip with how many other workers in the cluster, is defined by this parameter.
|
||||
> Any value higher than the cluster-size (i.e. the number of workers) implies all-to-all gossip.
|
||||
> To keep the network overhead low, this value should be reasonably low for a big cluster (e.g. 10 for a cluster size of 100).
|
||||
> On each refresh of the worker-monitoring tasks at the coordinator, the coordinator defines the list of worker URIs of size `failure-detector.gossip-group-size` to trigger worker-to-worker gossip.
|
||||
|
||||
## Task Properties
|
||||
|
||||
### `task.concurrency`
|
||||
|
|
@ -952,21 +856,14 @@ helps with cache affinity scheduling.
|
|||
> Note:
|
||||
> For snapshot recovery `query.remote-task.max-error-duration` should be greater than `exchange.max-error-duration`.
|
||||
|
||||
## Query Recovery
|
||||
|
||||
### `recovery_enabled`
|
||||
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `false`
|
||||
>
|
||||
> This session property is used to enable or disable the recovery framework, which enables to restart/resume the query in case of failure.
|
||||
## Distributed Snapshot
|
||||
|
||||
### `snapshot_enabled`
|
||||
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `false`
|
||||
>
|
||||
> This session property is enabled to capture snapshots during query execution, when recovery framework is enabled. Without recovery framework enabled this flag has no significance
|
||||
> This session property is used to enable or disable the distributed snapshot functionality.
|
||||
|
||||
### `hetu.experimental.snapshot.profile`
|
||||
|
||||
|
|
@ -978,23 +875,23 @@ helps with cache affinity scheduling.
|
|||
>
|
||||
> This is an experimental property. In the future it may be allowed to store snapshots in non-file-system locations, e.g. in a connector.
|
||||
|
||||
### `hetu.recovery.maxRetries`
|
||||
### `hetu.snapshot.maxRetries`
|
||||
|
||||
> - **Type:** `int`
|
||||
> - **Default value:** `10`
|
||||
>
|
||||
> This property defines the maximum number of error recovery attempts for a query. When the limit is reached, the query fails.
|
||||
>
|
||||
> This can also be specified on a per-query basis using the `recovery_max_retries` session property.
|
||||
> This can also be specified on a per-query basis using the `snapshot_max_retries` session property.
|
||||
|
||||
### `hetu.recovery.retryTimeout`
|
||||
### `hetu.snapshot.retryTimeout`
|
||||
|
||||
> - **Type:** `duration`
|
||||
> - **Default value:** `10m` (10 minutes)
|
||||
>
|
||||
> This property defines the maximum amount of time for the system to wait until all tasks are successfully restored. If any task is not ready within this timeout, then the recovery attempt is considered a failure, and the query will try to resume from an earlier snapshot if available.
|
||||
>
|
||||
> This can also be specified on a per-query basis using the `recovery_retry_timeout` session property.
|
||||
> This can also be specified on a per-query basis using the `snapshot_retry_timeout` session property.
|
||||
|
||||
### `hetu.snapshot.useKryoSerialization`
|
||||
|
||||
|
|
@ -1003,14 +900,6 @@ helps with cache affinity scheduling.
|
|||
>
|
||||
> Enables Kryo based serialization for snapshot, instead of default java serializer.
|
||||
|
||||
### `experimental.eliminate-duplicate-spill-files`
|
||||
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `false`
|
||||
>
|
||||
> Enables elimination of duplicate spill files storage as part of snapshot capture.
|
||||
|
||||
|
||||
|
||||
## HTTP Client Configurations
|
||||
|
||||
|
|
@ -1033,12 +922,3 @@ helps with cache affinity scheduling.
|
|||
> After the configured time elapsed and no response received, then client connection consider that to be failure in submission of request.
|
||||
>
|
||||
> (Note: this parameter should be configured with higher time when in high load environment)
|
||||
|
||||
## Connector Properties configuration
|
||||
|
||||
### `case-insensitive-name-matching`
|
||||
>
|
||||
> - **Type:** `boolean`
|
||||
> - **Default value:** `false`
|
||||
>
|
||||
> Case-insensitive matching between database and collection names. The default is case sensitive.
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@ To achieve better performance while maintaining execution reliability, the *dist
|
|||
|
||||
As of release 1.2.0, openLooKeng supports recovery of tasks and worker node failures.
|
||||
|
||||
## Enable Recovery framework
|
||||
## Enable Distributed Snapshot
|
||||
|
||||
Recovery framework is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`recovery_enabled`](properties.md#recovery_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
|
||||
Distributed snapshot is most useful for long running queries. It is disabled by default, and must be enabled and disabled via a session property [`snapshot_enabled`](properties.md#snapshot_enabled). It is recommended that the feature is only enabled for complex queries that require high reliability.
|
||||
|
||||
## Requirements
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ When a query that does not meet the above requirements is submitted with distrib
|
|||
|
||||
## Detection
|
||||
|
||||
Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`Failure Recovery handling Properties`](properties.md#Failure Recovery handling Properties) configuration.
|
||||
Error recovery is triggered when communication between the coordinator and a remote task fails for an extended period of time, as controlled by the [`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration) configuration.
|
||||
|
||||
## Storage Considerations
|
||||
|
||||
|
|
@ -55,20 +55,18 @@ Each query execution may produce multiple snapshots. Contents of these snapshots
|
|||
|
||||
The ability to recover from an error and resume from a snapshot does not come for free. Capturing a snapshot, depending on complexity, takes time. Thus it is a trade-off between performance and reliability.
|
||||
|
||||
It is suggested to turn on snapshot capture when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
|
||||
It is suggested to only turn on distributed snapshot when necessary, i.e. for queries that run for a long time. For these types of workloads, the overhead of taking snapshots becomes negligible.
|
||||
|
||||
## Snapshot statistics
|
||||
|
||||
Snapshot capture and restore statistics are displayed in CLI along with query result when CLI is launched in debug mode
|
||||
|
||||
Snapshot capture statistics includes number of snapshots captured, size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query. These statistics are displayed for all snapshots and for last snapshot separately.
|
||||
Snapshot capture statistics covers size of snapshots captured, CPU Time taken for capturing the snapshots and Wall Time taken for capturing the snapshots during the query. These statistics are displayed for all snapshots and for last snapshot separately.
|
||||
|
||||
Snapshot restore statistics covers number of times restored from snapshots during query, Size of the snapshots loaded for restoring, CPU Time taken for restoring from snapshots and Wall Time taken for restoring from snapshots. Restore statistics are displayed only when there is restore(recovery) happened during the query.
|
||||
|
||||
Additionally, while query is in progress number of capturing snapshots and id of the restoring snapshot will be displayed. Refer below picture for more details
|
||||
|
||||
|
||||
|
||||
## Configurations
|
||||
|
||||
Configurations related to recovery framework feature can be found in [Properties Reference](properties.md#Query Recovery).
|
||||
Configurations related to distributed snapshot feature can be found in [Properties Reference](properties.md#distributed-snapshot).
|
||||
|
|
|
|||
|
|
@ -61,26 +61,4 @@ and statistics about the query is available by clicking the *JSON* link. These v
|
|||
> - **Type:** `duration`
|
||||
> - **Default value:** `1 DAYS`
|
||||
>
|
||||
> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
|
||||
|
||||
### `hetu.queryhistory.max-count`
|
||||
|
||||
> - **Type:** `int`
|
||||
> - **Default value:** `1000`
|
||||
>
|
||||
> The maximum number of query history stored by openLooKeng. This could be overridden via "hetu.queryhistory.max-count" of "etc/config.properties".
|
||||
|
||||
### `hetu.collectionsql.max-count`
|
||||
|
||||
> - **Type:** `int`
|
||||
> - **Default value:** `100`
|
||||
>
|
||||
> The Maximum number of SQL collected by each user. This could be overridden via "hetu.collectionsql.max-count" of "etc/config.properties".
|
||||
|
||||
## Remarks
|
||||
|
||||
The max length of the favorite SQL is 600 by default. You can modify it through the following steps:
|
||||
|
||||
1. Login MySQL database according to the JDBC configuration of `hetu-metastore.properties`
|
||||
|
||||
2. Select table hetu_favorite, execute script `alter table hetu_favorite modify query varchar(2000) not null;` to modify the max length of the favorite SQL.
|
||||
> UI session timeout is set to 1 day as default. This could be overridden via `hetu.queryeditor-ui.session-timeout` of `etc/config.properties`
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
# Hudi Connector
|
||||
|
||||
### Release Notes
|
||||
Currently Hudi only supports version 0.7.0.
|
||||
|
||||
### Hudi Introduction
|
||||
|
||||
Apache Hudi is a fast growing data lake storage system that helps organizations build and manage petabyte-scale data lakes. Hudi enables storing vast amounts of data on top of existing DFS compatible storage while also enabling stream processing in addition to typical batch-processing. This is made possible by providing two new primitives. Specifically,
|
||||
|
|
|
|||
|
|
@ -1,173 +0,0 @@
|
|||
Redis Connector
|
||||
====================
|
||||
Overview
|
||||
--------
|
||||
this connector allows the use of Redis key/value pair is presented as a single row in openLooKeng.
|
||||
|
||||
**Note**
|
||||
|
||||
*In Redis,key/value pair can only be mapped to string or hash value types.keys can be stored in a zset,then keys can split into multiple slice*
|
||||
|
||||
*Support Redis 2.8.0 or higher*
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
To configure the Redis connector, create a catalog properties file `etc/catalog/redis.properties` with the following contents, replacing the properties as appropriate:
|
||||
``` properties
|
||||
connector.name=redis
|
||||
redis.table-names=schema1.table1,schema1.table2
|
||||
redis.nodes=host1:port
|
||||
```
|
||||
### Multiple Redis Servers
|
||||
You can have as many catalogs as you need. If you have additional
|
||||
Redis servers, simply add another properties file to ``etc/catalog``
|
||||
with a different name, making sure it ends in ``.properties``.
|
||||
For example, if you name the property file `sales.properties`, openLooKeng will create a catalog named `sales` using the configured connector.
|
||||
|
||||
Configuration properties
|
||||
------------------------
|
||||
The following configuration properties are available:
|
||||
|
||||
| Property Name | Description |
|
||||
|:-----------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------|
|
||||
| `redis.table-names` | List of all tables provided by the catalog |
|
||||
| `redis.default-schema` | Default schema name for tables (default `default`) |
|
||||
| `redis.nodes` | List of nodes in the Redis server |
|
||||
| `redis.connect-timeout` | Timeout for connecting to the Redis server (ms) (default 2000) |
|
||||
| `redis.scan-count` | The number of keys obtained from each scan for string and hash value types (default 100) |
|
||||
| `redis.key-prefix-schema-table` | Redis keys have schema-name:table-name prefix (default false) |
|
||||
| `redis.key-delimiter` | Delimiter separating schema_name and table_name if redis.key-prefix-schema-table is used (default `:`) |
|
||||
| `redis.table-description-dir` | Directory containing table description files (default `etc/redis/`) |
|
||||
| `redis.hide-internal-columns` | Whether internal columns are shown in table metadata or not. (default true) |
|
||||
| `redis.database-index` | Redis database index (default 0) |
|
||||
| `redis.password` | Redis server password (default null) |
|
||||
| `redis.table-description-interval` | the interval of flush description files (ms) (default no flush,table description will be memoized without expiration) |
|
||||
|
||||
|
||||
Internal columns
|
||||
----------------
|
||||
|
||||
| Column name | Type | Description |
|
||||
|:-------------------| :------ |:-----------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `_key` | VARCHAR | Redis key. |
|
||||
| `_value` | VARCHAR | Redis value corresponding to the key |
|
||||
| `_key_length` | BIGINT | Number of bytes in the key. |
|
||||
| `_key_corrupt` | BOOLEAN | True if the decoder could not decode the key for this row. When true, data columns mapped from the key should be treated as invalid. |
|
||||
| `_value_corrupt` | BOOLEAN | True if the decoder could not decode the value for this row. When true, data columns mapped from the value should be treated as invalid. |
|
||||
|
||||
|
||||
Table Definition Files
|
||||
----------------------
|
||||
For openLooKeng, every key/value pair must be mapped into columns to allow queries against the data. It is like kafka conntector,so you can refer to kafka-tutorial
|
||||
|
||||
A table definition file consists of a JSON definition for a table. The name of the file can be arbitrary but must end in `.json`.
|
||||
|
||||
for example,there is a nation.json
|
||||
``` json
|
||||
{
|
||||
"tableName": "nation",
|
||||
"schemaName": "tpch",
|
||||
"key": {
|
||||
"dataFormat": "raw",
|
||||
"fields": [
|
||||
{
|
||||
"name": "redis_key",
|
||||
"type": "VARCHAR(64)",
|
||||
"hidden": "true"
|
||||
}
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"dataFormat": "json",
|
||||
"fields": [
|
||||
{
|
||||
"name": "nationkey",
|
||||
"mapping": "nationkey",
|
||||
"type": "BIGINT"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"mapping": "name",
|
||||
"type": "VARCHAR(25)"
|
||||
},
|
||||
{
|
||||
"name": "regionkey",
|
||||
"mapping": "regionkey",
|
||||
"type": "BIGINT"
|
||||
},
|
||||
{
|
||||
"name": "comment",
|
||||
"mapping": "comment",
|
||||
"type": "VARCHAR(152)"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
In redis,such data exists
|
||||
```shell
|
||||
127.0.0.1:6379> keys tpch:nation:*
|
||||
1) "tpch:nation:2"
|
||||
2) "tpch:nation:4"
|
||||
3) "tpch:nation:16"
|
||||
4) "tpch:nation:18"
|
||||
5) "tpch:nation:10"
|
||||
6) "tpch:nation:17"
|
||||
7) "tpch:nation:1"
|
||||
```
|
||||
```shell
|
||||
127.0.0.1:6379> get tpch:nation:1
|
||||
"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
|
||||
```
|
||||
Now we can use redis connector get data from redis,(redis_key don't show,because we set "hidden": "true" )
|
||||
```shell
|
||||
lk> select * from redis.tpch.nation;
|
||||
nationkey | name | regionkey | comment
|
||||
-----------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
|
||||
3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
|
||||
9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
|
||||
19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
|
||||
2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
|
||||
```
|
||||
**Note**
|
||||
|
||||
*if redis.key-prefix-schema-table is false (default is false),all keys in redis will be mapped to table's key,no matching occurs*
|
||||
|
||||
Please refer to the `kafka-tutorial` for the description of the ``dataFormat`` as well as various available decoders.
|
||||
|
||||
In addition to the above Kafka types, the Redis connector supports ``hash`` type for the ``value`` field which represent data stored in the Redis hash.
|
||||
Redis connector use `hgetall key` to get data.
|
||||
``` json
|
||||
{
|
||||
"tableName": ...,
|
||||
"schemaName": ...,
|
||||
"value": {
|
||||
"dataFormat": "hash",
|
||||
"fields": [
|
||||
...
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
the Redis connector supports ``zset`` type for the ``key`` field which represent key stored in the Redis zset.
|
||||
if and only if ``zset`` is used as key datafomart,the split is truly supported , because we can use `zrange zsetkey split.start split.end` to get keys of a split.
|
||||
``` json
|
||||
{
|
||||
"tableName": ...,
|
||||
"schemaName": ...,
|
||||
"key": {
|
||||
"dataFormat": "zset",
|
||||
"name": "zsetkey", //zadd zsetkey score member
|
||||
"fields": [
|
||||
...
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Redis Connector Limitations
|
||||
---------------------------
|
||||
only support read operation,don't support write operation.
|
||||
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
# Developer Guide
|
||||
|
||||
openLooKeng is based on Trino 316(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
|
||||
openLooKeng is based on Trino(formerly known as PrestoSQL), and has been forked from the Trino open source project. openLooKeng has additional optimizations, and enhanced features to allow in-situ analytics on any data, anywhere, including geographically remote data sources. This guide is intended for openLooKeng contributors and plugin developers.
|
||||
|
||||
|
|
|
|||
|
|
@ -40,10 +40,6 @@
|
|||
>
|
||||
> openLooKeng Bilibili channel: https://space.bilibili.com/627629884
|
||||
|
||||
8. Which version of Trino is openLooKeng developed on?
|
||||
|
||||
> Based on Trino 316 version development.
|
||||
|
||||
## Functions
|
||||
|
||||
1. What connectors does the openLooKeng support?
|
||||
|
|
|
|||
Binary file not shown.
|
Before Width: | Height: | Size: 138 KiB After Width: | Height: | Size: 104 KiB |
|
|
@ -84,7 +84,6 @@ headless: true
|
|||
- [JMX]({{< relref "./docs/connector/jmx.md" >}})
|
||||
- [Kafka]({{< relref "./docs/connector/kafka.md" >}})
|
||||
- [Kafka Connector Tutorial]({{< relref "./docs/connector/kafka-tutorial.md" >}})
|
||||
- [Redis] ({{< relref "./docs/connector/redis.md" >}})
|
||||
- [Local File]({{< relref "./docs/connector/localfile.md" >}})
|
||||
- [Memory]({{< relref "./docs/connector/memory.md" >}})
|
||||
- [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
|
||||
Built-in System Access Control
|
||||
==============================
|
||||
|
||||
|
|
@ -56,10 +57,7 @@ composed of the following fields:
|
|||
|
||||
- `user` (optional): regex to match against user name. Defaults to `.*`.
|
||||
- `catalog` (optional): regex to match against catalog name. Defaults to `.*`.
|
||||
- ``allow`` (required): string indicating whether a user has access to the catalog.
|
||||
This value can be ``all``, ``read-only`` or ``none``, and defaults to ``none``.
|
||||
Setting this value to ``read-only`` has the same behavior as the ``read-only``
|
||||
system access control plugin.
|
||||
- `allow` (required): boolean indicating whether a user has access to the catalog
|
||||
|
||||
|
||||
**Note**
|
||||
|
|
@ -67,9 +65,7 @@ composed of the following fields:
|
|||
*By default, all users have access to the `system` catalog. You can override this behavior by adding a rule.*
|
||||
|
||||
|
||||
For example, if you want to allow only the user ``admin`` to access the``mysql`` and the ``system`` catalog,
|
||||
allow all users to access the ``hive`` catalog, allow the user ``alice`` read-only access to the ``postgresql``
|
||||
catalog, and deny all other access, you can use the following rules:
|
||||
For example, if you want to allow only the user `admin` to access the `mysql` and the `system` catalog, allow all users to access the `hive` catalog, and deny all other access, you can use the following rules:
|
||||
|
||||
``` json
|
||||
{
|
||||
|
|
@ -77,20 +73,15 @@ catalog, and deny all other access, you can use the following rules:
|
|||
{
|
||||
"user": "admin",
|
||||
"catalog": "(mysql|system)",
|
||||
"allow": all
|
||||
"allow": true
|
||||
},
|
||||
{
|
||||
"catalog": "hive",
|
||||
"allow": all
|
||||
},
|
||||
{
|
||||
"user": "alice",
|
||||
"catalog": "postgresql",
|
||||
"allow": "read-only"
|
||||
"allow": true
|
||||
},
|
||||
{
|
||||
"catalog": "system",
|
||||
"allow": none
|
||||
"allow": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -204,7 +195,6 @@ If you want to allow users to use the extactly same name as their Kerberos prin
|
|||
```
|
||||
|
||||
### Node State Rules
|
||||
|
||||
These rules govern the node state info particular users can access. The user is granted access to update a node state based on the first matching rule read from top to bottom. If no rule matches, access is denied. Each rule is
|
||||
composed of the following fields:
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Overview
|
|||
|
||||
Apache Ranger delivers a comprehensive approach to security for a Hadoop cluster. It provides a centralized platform to define, administer and manage security policies consistently across Hadoop components. Check [Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index) for detail introduction and user guide.
|
||||
|
||||
[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is developed based on Ranger 2.1.0, which is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
|
||||
[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) is a ranger plugin for openLooKeng to enable, monitor and manage comprehensive data security.
|
||||
|
||||
Build Process
|
||||
-------------------------
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# 审计日志
|
||||
|
||||
openLooKeng审计日志记录功能是一个自定义事件监听器，监听openLooKeng集群启停与集群中节点的动态添加与删除事件；监听WebUi用户登录与退出事件；监听查询事件,在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
|
||||
openLooKeng审计日志记录功能是一个自定义事件监听器，在查询创建和完成（成功或失败）时调用。审计日志包含以下信息：
|
||||
|
||||
1. 事件发生时间
|
||||
2. 用户ID
|
||||
|
|
@ -23,17 +23,15 @@ openLooKeng审计日志记录功能是一个自定义事件监听器，监听ope
|
|||
hetu.event.listener.type=AUDIT
|
||||
hetu.event.listener.listen.query.creation=true
|
||||
hetu.event.listener.listen.query.completion=true
|
||||
hetu.auditlog.logoutput=/var/log/
|
||||
hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
|
||||
```
|
||||
|
||||
其他审计日志记录属性包括：
|
||||
|
||||
`hetu.event.listener.type`：用于定义审计日志的记录类型，允许的值为AUDIT和LOGGER。
|
||||
`hetu.event.listener.audit.file`：可选属性，用于定义审计文件的绝对文件路径。确保运行openLooKeng服务器的进程对该目录有写权限。
|
||||
|
||||
`hetu.auditlog.logoutput`：用于定义审计文件的绝对目录路径。确保运行openLooKeng服务器的进程对该目录有写权限。
|
||||
`hetu.event.listener.audit.filecount`：可选属性，用于定义要使用的文件数。
|
||||
|
||||
`hetu.auditlog.logconversionpattern`：用于定义审计日志的轮转模式。允许的值为yyyy-MM-dd.HH和yyyy-MM-dd。
|
||||
`hetu.event.listener.audit.limit`：可选属性，用于定义写入任一文件的最大字节数。
|
||||
|
||||
配置文件示例：
|
||||
|
||||
|
|
@ -45,6 +43,4 @@ hetu.event.listener.listen.query.completion=true
|
|||
hetu.event.listener.audit.file=/var/log/hetu/hetu-audit.log
|
||||
hetu.event.listener.audit.filecount=1
|
||||
hetu.event.listener.audit.limit=100000
|
||||
hetu.auditlog.logoutput=/var/log/
|
||||
hetu.auditlog.logconversionpattern=yyyy-MM-dd.HH
|
||||
```
|
||||
|
|
@ -115,20 +115,6 @@
|
|||
>
|
||||
> 此属性是在JVM堆中为openLooKeng不跟踪的分配留作裕量/缓冲区的内存量。
|
||||
|
||||
### `query.suspend-query-enabled`
|
||||
|
||||
> - **类型：** `boolean`
|
||||
> - **默认值：** `false`
|
||||
>
|
||||
> 系统资源不足时，临时挂起运行中的查询。
|
||||
|
||||
### `query.max-suspended-queries`
|
||||
|
||||
> - **类型：** `integer`
|
||||
> - **默认值：** `10`
|
||||
>
|
||||
> 终止查询之前，查询挂起尝试的最大次数。仅当`query.suspend-query-enabled`设置为`true`时，此属性才生效。
|
||||
|
||||
## 溢出属性
|
||||
|
||||
### `experimental.spill-enabled`
|
||||
|
|
@ -354,124 +340,35 @@
|
|||
>
|
||||
> 如果网络延迟较高，增大该值可以提高网络吞吐量。减小该值可以提高大型集群的查询性能，因为它减少了由于交换客户端缓冲区保存了较多任务（而不是保存较少任务中的较多数据）的响应而导致的倾斜。
|
||||
|
||||
### `exchange.max-error-duration`
|
||||
|
||||
> - **类型：** `duration`
|
||||
> - **最小值：** `1m`
|
||||
> - **默认值：** `7m`
|
||||
>
|
||||
> 交换错误最大缓冲时间，超过该时限则查询失败。
|
||||
|
||||
### `exchange.is-timeout-failure-detection-enabled`
|
||||
|
||||
> - **类型：** `boolean`
|
||||
> - **默认值：** `true`
|
||||
>
|
||||
> 正在使用的故障检测机制。默认值是基于超时的故障检测。但是，当该属性设置为`false`时，启用基于最大重试次数的故障检测机制。
|
||||
|
||||
### `exchange.max-retry-count`
|
||||
|
||||
> - **类型：** `integer`
|
||||
> - **默认值：** `10`
|
||||
>
|
||||
> Coordinator在将失败任务视为永久失败之前对其执行的最大重试次数。仅当`exchange.is-timeout-failure-detection-enabled`设置为`false`时，才使用此属性。
|
||||
|
||||
### `sink.max-buffer-size`
|
||||
|
||||
> - **类型：** `data size`
|
||||
> - **默认值：** `32MB`
|
||||
>
|
||||
>等待上游任务拉取的任务数据的输出缓冲区大小。如果任务输出是哈希分区的，则缓冲区将在所有分区的消费者之间共享。如果网络延迟高或集群中有许多节点，则增加此值可以提高阶段之间传输数据的网络吞吐量。
|
||||
|
||||
## 故障恢复处理属性
|
||||
|
||||
### 失败重试策略
|
||||
|
||||
### `failure.recovery.retry.profile`
|
||||
|
||||
> - **类型：** `string`
|
||||
> - **默认值：** `default`
|
||||
>
|
||||
> 此属性定义用于确定HTTP客户端上是否发生故障的故障检测配置文件。此属性的值`<profile-name>`必须对应`etc/failure-retry-policy/`路径中的`<profile-name>.properties`文件。如果没有此类配置文件可用，并且未设置此属性，则使用“default”配置文件。
|
||||
> 例如，`failure.recovery.retry.profile="test"`要求`test.properties`文件存在于`etc/failure-retry-policy`路径中。
|
||||
> `test.properties`文件必须包含指定的`failure.recovery.retry.type`。
|
||||
|
||||
|
||||
### `failure.recovery.retry.type`
|
||||
|
||||
> - **类型：** `string`
|
||||
> - **默认值：** `timeout`
|
||||
>
|
||||
> 此属性用来设置正在使用的故障检测机制。默认值是基于`timeout`的故障检测。
|
||||
|
||||
#### 基于`timeout`的故障检测
|
||||
|
||||
> 如果使用此机制，HTTP客户端故障将在指定时间段内重试，重试失败则被视为永久故障。
|
||||
>
|
||||
> 可以为此类故障检测定义`max.error.duration`属性。
|
||||
|
||||
#### 基于`max-retry`的故障检测
|
||||
|
||||
> 如果使用此机制，HTTP客户端故障将在被视为永久故障之前重试指定次数。
|
||||
> 可以为此类故障检测定义`max.retry.count`和`max.error.duration`属性。
|
||||
> 在这种类型的故障检测中，在查询故障检测模块之前，会执行`max.retry.count`次重试。当故障检测器模块检测到远程节点发生故障时，HTTP客户端将此故障视为永久故障。否则，例如，当远程工作节点处于活动状态但没有响应时，在`max.error.duration`指定的时间段内重试，重试失败则被视为永久故障。
|
||||
|
||||
### `max.error.duration`
|
||||
|
||||
> - **类型：** `duration`
|
||||
> - **默认值：** `300s`
|
||||
>
|
||||
> 被视为永久故障前，协调器等待解决任务间相关错误的最长时间。
|
||||
|
||||
### `max.retry.count`
|
||||
|
||||
> - **类型：** `integer`
|
||||
> - **默认值：** `100`
|
||||
>
|
||||
> 协调器在向故障检测器模块查询远程节点状态之前，对失败任务执行的最大重试次数。
|
||||
> 此属性指定查询失败检测模块之前的最小重试次数。因此，实际故障数量可能会因为集群大小和集群负载而略有不同。
|
||||
> 此属性仅用于基于`max-retry`的故障检测配置文件。
|
||||
> 最小值为100。
|
||||
|
||||
### 故障检测Gossip协议配置
|
||||
|
||||
### `failure-detection-protocol`
|
||||
|
||||
>- **类型：** `string`
|
||||
>- **默认值：** `heartbeat`
|
||||
>
|
||||
>此属性定义正在使用的故障检测器的类型。默认配置为`heartbeat`故障检测器。
|
||||
>在`config.properties`文件中，将此属性配置为`gossip`，可以启用Gossip协议。
|
||||
>集群中的所有节点（即协调器和工作节点）都应在其各自的`etc/config.properties`文件中指定此属性。
|
||||
|
||||
### `failure-detector.heartbeat-interval`
|
||||
|
||||
>- **类型：** `duration`
|
||||
>- **默认值：** `500ms` （500毫秒）
|
||||
>
|
||||
>集群中两个节点之间的消息散播间隔。
|
||||
>在Gossip协议中，两个工作节点间的消息散播频率高于协调器和一个工作节点间。
|
||||
>在协调器的`config.properties`文件中，可以为此属性配置一个较大的值，例如`5s`（5秒）。
|
||||
>在工作节点中，可以使用默认值。
|
||||
|
||||
### `failure-detector.worker-gossip-probe-interval`
|
||||
|
||||
>- **类型：** `duration`
|
||||
>- **默认值：** `5s`（5秒）
|
||||
>
|
||||
>Gossip协议使用监控任务（与`heartbeat`故障检测器相同）来监控其他节点。
|
||||
>此属性指定监控任务刷新间隔，以触发工作节点消息散播。
|
||||
>仅可以为工作节点指定默认值以外的任何其他值。
|
||||
>该属性的值必须大于`failure-detector.heartbeat-interval`的值。
|
||||
|
||||
### `failure-detector.coordinator-gossip-probe-interval`
|
||||
|
||||
>- **类型：** `duration`
|
||||
>- **默认值：** `5s`（5秒）
|
||||
>
|
||||
>Gossip协议使用监控任务（与heartbeat故障检测器相同）来监控其他节点。
|
||||
>此属性指定监控任务刷新间隔，以触发协调器参与工作节点消息散播。
|
||||
>仅可以为协调器指定默认值以外的任何其他值。
|
||||
>该属性的值必须大于`failure-detector.heartbeat-interval`和`failure-detector.worker-gossip-probe-interval`的值。
|
||||
|
||||
### `failure-detector.coordinator-gossip-collate-interval`
|
||||
|
||||
>- **类型：** `duration`
|
||||
>- **默认值：** `2s`（2秒）
|
||||
>
|
||||
>此属性指定协调器整理从所有工作节点获得的所有散播消息的间隔。
|
||||
>此属性只支持为协调器配置。
|
||||
>该属性的值必须大于`failure-detector.heartbeat-interval`的值。
|
||||
|
||||
### `failure-detector.gossip-group-size`
|
||||
|
||||
>- **类型：** `integer`
|
||||
>- **默认值：** `Integer.MAX_VALUE`
|
||||
>
|
||||
>此属性定义单个工作节点在集群中散播消息的工作节点数量。
|
||||
>任何大于集群大小（即工作节点数量）的值都意味着all-to-all消息散播。
|
||||
>要保持较低的网络开销，针对大型集群，请将此属性设置为一个较小的值（例如，100工作节点的集群设置为10）。
|
||||
>每次刷新协调器上的工作节点监视任务时，协调器都会定义工作节点URI列表，其大小由`failure-detector.gossip-group-size`指定，以触发worker-to-worker消息散播。
|
||||
>
|
||||
>
|
||||
> 上游任务等待拉取任务数据的输出缓冲区大小。如果任务输出是经过哈希分区的，那么缓冲区将在所有分区的使用者之间共享。如果网络延迟较高或集群中有多个节点，增加此值可以提高在阶段之间传输的数据的网络吞吐量。
|
||||
|
||||
## 任务属性
|
||||
|
||||
### `task.concurrency`
|
||||
|
|
@ -715,7 +612,7 @@
|
|||
|
||||
## 启发式索引属性
|
||||
|
||||
启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持Hive连接器的ORC存储格式的表。
|
||||
启发式索引是外部索引模块，可用于过滤连接器级别的行。 位图，Bloom和MinMaxIndex是openLooKeng提供的索引列表。 到目前为止，位图索引支持使用ORC存储格式的表支持蜂巢连接器。
|
||||
|
||||
### `hetu.heuristicindex.filter.enabled`
|
||||
|
||||
|
|
@ -898,25 +795,18 @@
|
|||
>
|
||||
> 远程任务错误最大缓冲时间，超过该时限则查询失败。
|
||||
|
||||
## 查询恢复
|
||||
|
||||
### `recovery_enabled`
|
||||
|
||||
> - **类型：** `boolean`
|
||||
> - **默认值：** `false`
|
||||
>
|
||||
> 此会话属性用于启用或禁用恢复框架，该框架在发生故障时启用或禁用查询重启/恢复。
|
||||
## 分布式快照
|
||||
|
||||
### `snapshot_enabled`
|
||||
|
||||
> - **类型：** `boolean`
|
||||
> - **默认值：** `false`
|
||||
> - 类型：`boolean`
|
||||
> - **默认值**：`false`
|
||||
>
|
||||
> 启用恢复框架时，启用此会话属性可以在查询执行期间捕获快照。如果未启用恢复框架，则此属性不生效。
|
||||
> 此会话属性用于启用或禁用分布式快照功能。
|
||||
|
||||
### `hetu.experimental.snapshot.profile`
|
||||
|
||||
> - **类型：**`string`
|
||||
> - 类型：`string`
|
||||
>
|
||||
> 此属性定义用于存储快照的[文件系统](../develop/filesystem.md)配置文件。对应的配置文件必须存在于`etc/filesystem`中。例如，如果将该属性设置为`hetu.experimental.snapshot.profile=snapshot-hdfs1`，则必须在`etc/filesystem`中创建描述此文件系统的配置文件`snapshot-hdfs1.properties`，其中包含的必要信息包括身份验证类型、配置和密钥表（如适用）。具体细节请参考[文件系统](../develop/filesystem.md)相关章节。
|
||||
>
|
||||
|
|
@ -924,23 +814,23 @@
|
|||
>
|
||||
> 作为实验性属性，或可以将快照存储在非文件系统位置，如连接器。
|
||||
|
||||
### `hetu.recovery.maxRetries`
|
||||
### `hetu.snapshot.maxRetries`
|
||||
|
||||
> - **类型：** `integer`
|
||||
> - **默认值：** `10`
|
||||
> - 类型：`int`
|
||||
> - **默认值**：`10`
|
||||
>
|
||||
> 此属性定义查询错误恢复尝试的最大次数。当达到限制时，查询失败。
|
||||
> 此属性定义查询的错误恢复尝试的最大次数。达到限制时，查询失败。
|
||||
>
|
||||
> 也可以使用`recovery_max_retries`会话属性为每个查询指定此属性。
|
||||
> 也可以使用`snapshot_max_retries`会话属性在每个查询基础上指定。
|
||||
|
||||
### `hetu.recovery.retryTimeout`
|
||||
### `hetu.snapshot.retryTimeout`
|
||||
|
||||
> - **类型：** `duration`
|
||||
> - **默认值：** `10m`（10分钟）
|
||||
>
|
||||
> 此属性定义系统等待所有任务成功恢复的最长时间。如果在此时间内有任何任务未就绪，则恢复尝试将被视为失败，查询将尝试从较早的快照恢复（如果可用）。
|
||||
> 此属性定义系统等待所有任务成功恢复的最大时长。如果在此超时时限内任何任务未就绪，则认为恢复失败，查询将尝试从较早快照恢复（如果可用）。
|
||||
>
|
||||
> 也可以使用`recovery_retry_timeout`会话属性为每个查询指定此属性。
|
||||
> 也可以使用`snapshot_retry_timeout`会话属性在每个查询基础上指定。
|
||||
|
||||
### `hetu.snapshot.useKryoSerialization`
|
||||
|
||||
|
|
@ -969,13 +859,4 @@
|
|||
> 此参数定义了http客户端接收响应的时间阈值。
|
||||
> 当超过所配置时间，客户端没有接收到任何响应，则视为客户端的请求提交失败。
|
||||
>
|
||||
> （注意： 建议在高负载环境下，该参数配置大一点。）
|
||||
|
||||
## 连接器属性配置
|
||||
|
||||
### `case-insensitive-name-matching`
|
||||
|
||||
> - **类型:** `boolean`
|
||||
> - **默认值:** `false`
|
||||
>
|
||||
> 不区分大小写匹配数据库和集合名称，默认区分大小写。
|
||||
> （注意： 建议在高负载环境下，该参数配置大一点。）
|
||||
|
|
@ -10,9 +10,9 @@
|
|||
|
||||
自版本1.2.0起，openLooKeng支持恢复任务和工作节点故障。
|
||||
|
||||
## 启用恢复框架
|
||||
|
||||
恢复框架对于长时间运行的查询最有用。默认禁用，可以使用会话属性[`recovery_enabled`](properties.md#recovery_enabled)启用和禁用恢复框架。建议仅对可靠性要求高的复杂查询启用该功能。
|
||||
## 启用分布式快照
|
||||
|
||||
分布式快照适用于长时间运行的查询任务。该功能默认为禁用状态，可以通过会话属性[`snapshot_enabled`](properties.md#snapshot_enabled)启用或禁用。建议仅在对可靠性要求高的复杂查询场景下启用该功能。
|
||||
|
||||
## 要求
|
||||
|
||||
|
|
@ -35,7 +35,7 @@
|
|||
|
||||
## 检测
|
||||
|
||||
当协调器与远程任务之间的通信长时间失败时，将触发错误恢复，由[`故障恢复处理属性`](properties.md#故障恢复处理属性)配置控制。
|
||||
协调节点与远程任务之间的通信长时间失败时，将触发错误恢复，由[`query.remote-task.max-error-duration`](properties.md#queryremote-taskmax-error-duration)配置控制。
|
||||
|
||||
## 存储注意事项
|
||||
|
||||
|
|
@ -53,20 +53,18 @@
|
|||
|
||||
从错误和快照中恢复需要成本。捕获快照需要时间，时间长短取决于复杂性。因此，需要在性能和可靠性之间进行权衡。
|
||||
|
||||
建议在必要时打开快照捕获，例如对于长时间运行的查询。对于这些类型的工作负载，拍摄快照的开销可以忽略不计。
|
||||
建议仅在必要时启用分布式快照，如运行时间较长的查询任务。对于这些类型的工作负载，捕获快照的开销可以忽略不计。
|
||||
|
||||
## 快照统计信息
|
||||
|
||||
在调试模式下启动CLI时，快照捕获信息和恢复信息将与查询结果一起显示在CLI中。
|
||||
|
||||
快照捕获统计信息包括捕获的快照数量、捕获的快照大小、捕获快照所需的CPU时间和在查询期间捕获快照所需的挂钟时间。所有快照和最后一个快照的统计信息会分别显示。
|
||||
快照捕获信息包括查询期间捕获的快照大小、捕获快照的CPU时间和捕获快照的挂钟时间。分别显示所有快照和最后一个快照的捕获信息。
|
||||
|
||||
快照恢复信息包括查询期间从快照恢复的次数、加载用于恢复的快照大小、从快照恢复所需的CPU时间和从快照恢复所需的挂钟时间。仅当查询期间发生恢复时，才会显示恢复信息。
|
||||
|
||||
此外，在查询正在进行时，将显示捕获的快照数量和恢复的快照的ID。更多详细信息，见下图。
|
||||
|
||||
|
||||
|
||||
|
||||
## 配置
|
||||
|
||||
与恢复框架功能相关的配置，请参见[属性参考](properties.md#查询恢复)。
|
||||
与分布式快照功能相关的配置可参见[属性参考](properties.md#分布式快照)。
|
||||
|
|
@ -57,8 +57,6 @@ openLooKeng将溢出路径视为独立的磁盘（参见[JBOD](https://en.wikipe
|
|||
|
||||
通过这种机制，联接操作符使用的峰值内存可以降低到最大构建表分区的大小。假设没有数据倾斜，这个值将是整个构建表大小的`1 / task.concurrency`倍。
|
||||
|
||||
注意：spill-to-disk不支持交叉连接。
|
||||
|
||||
### 聚合
|
||||
|
||||
聚合函数对一组值执行操作并返回一个值。如果要聚合的组数量很大，可能需要大量内存。当启用溢出到磁盘时，如果没有足够的内存，则中间累积的聚合结果将写入磁盘。结果被重新加载回来，并以较低的内存占用量合并。
|
||||
|
|
|
|||
|
|
@ -58,27 +58,4 @@ openLooKeng提供了一个用于监视和管理查询的Web界面。Web界面可
|
|||
> - **类型：** `duration`
|
||||
> - **默认值：** `1 DAYS`
|
||||
>
|
||||
> UI会话超时默认设置为1天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.session-timeout`属性修改。
|
||||
|
||||
### `hetu.queryhistory.max-count`
|
||||
|
||||
> - **Type:** `int`
|
||||
> - **Default value:** `1000`
|
||||
>
|
||||
> openLooKeng储存的历史查询记录最大数量。可以通过配置`etc/config.properties`文件的`hetu.queryhistory.max-count`属性修改。
|
||||
|
||||
### `hetu.collectionsql.max-count`
|
||||
|
||||
> - **Type:** `int`
|
||||
> - **Default value:** `100`
|
||||
>
|
||||
> 每位用户收藏sql语句条数上限.可以通过配置`etc/config.properties`文件的`hetu.collectionsql.max-count`属性修改。
|
||||
|
||||
|
||||
## 备注
|
||||
|
||||
收藏sql语句的最大长度默认为600，可通过如下步骤对其进行修改：
|
||||
|
||||
1. 根据hetu-metastore.properties文件中jdbc配置信息登录mysql数据库。
|
||||
|
||||
2. 选中hetu_favorite表，使用命令`alter table hetu_favorite modify query varchar(2000) not null;`修改收藏语句最大长度为2000。
|
||||
> UI会话超时默认设置为1天。可以通过配置`etc/config.properties`文件中的`hetu.queryeditor-ui.session-timeout`属性修改。
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
# Hudi连接器
|
||||
|
||||
### 版本说明
|
||||
目前Hudi只支持0.7.0版本。
|
||||
|
||||
### Hudi介绍
|
||||
|
||||
Apache Hudi是一个快速迭代的数据湖存储系统，可以帮助企业构建和管理PB级数据湖。它提供在DFS上存储超大规模数据集，同时使得流式处理如果批处理一样，该实现主要是通过如下两个原语实现。
|
||||
|
|
|
|||
|
|
@ -1,171 +0,0 @@
|
|||
Redis 连接器
|
||||
====================
|
||||
概述
|
||||
--------
|
||||
此连接器允许在openlookeng中，将redis中一个kv键值对映射成表的一行数据
|
||||
|
||||
**说明**
|
||||
|
||||
*kv键值对只能在Reids中映射为string或hash类型。keys可以存储为一个zset,然后keys可以被分割成多个片*
|
||||
|
||||
*支持 Redis 2.8.0 或更高版本*
|
||||
|
||||
|
||||
配置
|
||||
-------------
|
||||
要配置Redis连接器，创建具有以下内容的目录属性文件etc/catalog/redis.properties，并适当替换以下属性：
|
||||
``` properties
|
||||
connector.name=redis
|
||||
redis.table-names=schema1.table1,schema1.table2
|
||||
redis.nodes=host1:port
|
||||
```
|
||||
### 多个 Redis Servers
|
||||
可以根据需要创建任意多的目录，因此，如果有额外的redis server，只需添加另一个不同的名称的属性文件到etc/catalog中（确保它以.properties结尾）。例如，如果将属性文件命名为sales.properties，openLooKeng将使用配置的连接器创建一个名为sales的目录。
|
||||
|
||||
配置属性
|
||||
------------------------
|
||||
配置属性包括：
|
||||
|
||||
| 属性名称 | 说明 |
|
||||
|:-----------------------------------------------------------|:----------------------------------------------------------------------------|
|
||||
| `redis.table-names` | catalog 提供的所有表的列表 |
|
||||
| `redis.default-schema` | 表的默认schema名 (默认`default`) |
|
||||
| `redis.nodes` | Redis server的节点列表 |
|
||||
| `redis.connect-timeout` | 连接Redis server的超时时间 (ms) (默认 2000) |
|
||||
| `redis.scan-count` | 每轮scan获得的key的数量 (默认 100) |
|
||||
| `redis.key-prefix-schema-table` | Redis keys 是否有 schema-name:table-name的前缀 (默认 false) |
|
||||
| `redis.key-delimiter` | 如果`redis.key-prefix-schema-table`被启用，那么schema-name和table-name的分隔符为 (默认 `:`) |
|
||||
| `redis.table-description-dir` | 存放表定义json文件的相对地址 (默认 `etc/redis/`) |
|
||||
| `redis.hide-internal-columns` | 内部列是否在元数据中隐藏(默认 true) |
|
||||
| `redis.database-index` | Redis database 的索引 (默认 0) |
|
||||
| `redis.password` | Redis server 密码 (默认 null) |
|
||||
| `redis.table-description-interval` | flush表定义文件的时间间隔（默认不会flush,意味着Plugin被加载后,表定义一直留存在内存中,不再主动读取json文件） |
|
||||
|
||||
内部列
|
||||
----------------
|
||||
|
||||
| 列名 | 类型 | 说明 |
|
||||
|:-------------------| :------ |:----------------------------------------------------|
|
||||
| `_key` | VARCHAR | Redis key. |
|
||||
| `_value` | VARCHAR | 与key相对应的值 |
|
||||
| `_key_length` | BIGINT | key 的字节大小 |
|
||||
| `_key_corrupt` | BOOLEAN | 如果解码器无法解码此行的key，则为true。当为 true 时，从键映射的数据列应被视为无效。 |
|
||||
| `_value_corrupt` | BOOLEAN | 如果解码器无法解码此行的value，则为true。当为 true 时，从该值映射的数据列应被视为无效。 |
|
||||
|
||||
|
||||
|
||||
表定义文件
|
||||
----------------------
|
||||
对于openLooKeng，每个kv键值对 必须被映射到列中，以便允许对数据查询。这很像kafka connector,所以你可以参考 kafka连接器教程
|
||||
|
||||
|
||||
表定义文件由一个表的JSON定义组成。文件名可以任意，但必须以.json结尾。
|
||||
|
||||
以nation.json为例
|
||||
``` json
|
||||
{
|
||||
"tableName": "nation",
|
||||
"schemaName": "tpch",
|
||||
"key": {
|
||||
"dataFormat": "raw",
|
||||
"fields": [
|
||||
{
|
||||
"name": "redis_key",
|
||||
"type": "VARCHAR(64)",
|
||||
"hidden": "true"
|
||||
}
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"dataFormat": "json",
|
||||
"fields": [
|
||||
{
|
||||
"name": "nationkey",
|
||||
"mapping": "nationkey",
|
||||
"type": "BIGINT"
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"mapping": "name",
|
||||
"type": "VARCHAR(25)"
|
||||
},
|
||||
{
|
||||
"name": "regionkey",
|
||||
"mapping": "regionkey",
|
||||
"type": "BIGINT"
|
||||
},
|
||||
{
|
||||
"name": "comment",
|
||||
"mapping": "comment",
|
||||
"type": "VARCHAR(152)"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
在redis,相应的有这样的数据
|
||||
```shell
|
||||
127.0.0.1:6379> keys tpch:nation:*
|
||||
1) "tpch:nation:2"
|
||||
2) "tpch:nation:4"
|
||||
3) "tpch:nation:16"
|
||||
4) "tpch:nation:18"
|
||||
5) "tpch:nation:10"
|
||||
6) "tpch:nation:17"
|
||||
7) "tpch:nation:1"
|
||||
```
|
||||
```shell
|
||||
127.0.0.1:6379> get tpch:nation:1
|
||||
"{\"nationkey\":1,\"name\":\"ARGENTINA\",\"regionkey\":1,\"comment\":\"al foxes promise slyly according to the regular accounts. bold requests alon\"}"
|
||||
```
|
||||
我们可以使用redis connector从redis中获取数据，（redis_key没有显示，这是因为我们设置了"hidden": "true" ）
|
||||
```shell
|
||||
lk> select * from redis.tpch.nation;
|
||||
nationkey | name | regionkey | comment
|
||||
-----------+----------------+-----------+--------------------------------------------------------------------------------------------------------------------
|
||||
3 | CANADA | 1 | eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold
|
||||
9 | INDONESIA | 2 | slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull
|
||||
19 | ROMANIA | 3 | ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account
|
||||
2 | BRAZIL | 1 | y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special
|
||||
```
|
||||
**说明**
|
||||
|
||||
*如果属性 `redis.key-prefix-schema-table` 是false (默认是false),那么当前所有key的都会被视作的nation表的key,不会发生匹配过滤*
|
||||
|
||||
有关各种可用解码器的描述，请参考kafka连接器教程
|
||||
|
||||
除了kafka支持的类型，Redis connector对value字段支持hash类型
|
||||
``` json
|
||||
{
|
||||
"tableName": ...,
|
||||
"schemaName": ...,
|
||||
"value": {
|
||||
"dataFormat": "hash",
|
||||
"fields": [
|
||||
...
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Redis connector 支持``zset`` 作为``key``在Redis中存储类型。
|
||||
当且仅当``zset`` 作为key的存储格式时，split切片功能才能被真正支持，因为我们可以使用`zrange zsetkey split.start split.end`来得到一个切片的keys
|
||||
``` json
|
||||
{
|
||||
"tableName": ...,
|
||||
"schemaName": ...,
|
||||
"key": {
|
||||
"dataFormat": "zset",
|
||||
"name": "zsetkey", //zadd zsetkey score member
|
||||
"fields": [
|
||||
...
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Redis 连接器的局限性
|
||||
---------------------------
|
||||
只支持读操作，不支持写操作.
|
||||
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
|
||||
# 开发者指南
|
||||
|
||||
openLooKeng基于Trino 316版本(之前称为PrestoSQL)，是Trino开源项目的一个分支。与Trino开源项目相比，openLooKeng提供了额外的优化和增强功能，可以在任何位置对任何数据（包括远程数据源）进行现场分析。本指南适用于openLooKeng参与者和插件开发人员。
|
||||
openLooKeng基于Trino(之前称为PrestoSQL)，是Trino开源项目的一个分支。与Trino开源项目相比，openLooKeng提供了额外的优化和增强功能，可以在任何位置对任何数据（包括远程数据源）进行现场分析。本指南适用于openLooKeng参与者和插件开发人员。
|
||||
|
|
|
|||
|
|
@ -35,10 +35,6 @@
|
|||
> openLooKeng微信群：请添加openLooKeng小助手微信号：openLooKengoss，小助手会拉您入群
|
||||
> openLooKeng B站： https://space.bilibili.com/627629884
|
||||
|
||||
8. openLooKeng基于Trino哪个版本进行开发？
|
||||
|
||||
> 基于Trino 316版本进行开发。
|
||||
|
||||
## 功能
|
||||
|
||||
1. openLooKeng目前支持哪些连接器?
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 104 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 64 KiB |
|
|
@ -84,7 +84,6 @@ headless: true
|
|||
- [JMX]({{< relref "./docs/connector/jmx.md" >}})
|
||||
- [Kafka]({{< relref "./docs/connector/kafka.md" >}})
|
||||
- [Kafka连接器教程]({{< relref "./docs/connector/kafka-tutorial.md" >}})
|
||||
- [Redis] ({{< relref "./docs/connector/redis.md" >}})
|
||||
- [本地文件]({{< relref "./docs/connector/localfile.md" >}})
|
||||
- [内存]({{< relref "./docs/connector/memory.md" >}})
|
||||
- [MongoDB]({{< relref "./docs/connector/mongodb.md" >}})
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ security.refresh-period=1s
|
|||
|
||||
- `user`（可选）：用于匹配用户名的正则表达式。默认为`.*`。
|
||||
- `catalog`（可选）:用于匹配目录名的正则表达式。默认为`.*`。
|
||||
- `allow`（必选）: 字符串参数，表示用户是否有访问目录的权限。这个值可以是all、read-only或none，默认为none。将此值设置为read-only，其行为与只读的系统访问控制插件相同。
|
||||
- `allow`（必选）: 布尔类型参数，表示用户是否有访问目录的权限
|
||||
|
||||
|
||||
**注意**
|
||||
|
|
@ -74,20 +74,15 @@ security.refresh-period=1s
|
|||
{
|
||||
"user": "admin",
|
||||
"catalog": "(mysql|system)",
|
||||
"allow": all
|
||||
"allow": true
|
||||
},
|
||||
{
|
||||
"catalog": "hive",
|
||||
"allow": all
|
||||
},
|
||||
{
|
||||
"user": "alice",
|
||||
"catalog": "postgresql",
|
||||
"allow": "read-only"
|
||||
"allow": true
|
||||
},
|
||||
{
|
||||
"catalog": "system",
|
||||
"allow": none
|
||||
"allow": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Ranger 访问控制
|
|||
|
||||
Apache Ranger 为 Hadoop 集群提供了一种全面的安全框架，以跨组件的、一致性的方式进行定义、授权、管理安全策略。Ranger 详细介绍和用户指导可以参考[Apache Ranger Wiki](https://cwiki.apache.org/confluence/display/RANGER/Index )。
|
||||
|
||||
[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) 基于Ranger 2.1.0版本进行开发，是为 openLooKeng 开发的 Ranger 插件，用于全面的数据安全监控和权限管理。
|
||||
[openlookeng-ranger-plugin](https://gitee.com/openlookeng/openlookeng-ranger-plugin) 是为 openLooKeng 开发的 Ranger 插件，用于全面的数据安全监控和权限管理。
|
||||
|
||||
编译过程
|
||||
-------------------------
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-function-namespace-managers</artifactId>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-hana</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-hazelcast</artifactId>
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
<jline.version>3.21.0</jline.version>
|
||||
<hazelcast.version>5.1</hazelcast.version>
|
||||
<aspectjweaver.version>1.9.4</aspectjweaver.version>
|
||||
<spring-context.version>5.3.20</spring-context.version>
|
||||
<spring-context.version>5.2.19.RELEASE</spring-context.version>
|
||||
<cache-api.version>1.1.1</cache-api.version>
|
||||
<jmockit.version>1.38</jmockit.version>
|
||||
</properties>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-hbase</artifactId>
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
<dep.log4j.version>2.17.2</dep.log4j.version>
|
||||
<dep.log4j.version>2.17.1</dep.log4j.version>
|
||||
<version.annotations>13.0</version.annotations>
|
||||
<version.validation-api>2.0.1.Final</version.validation-api>
|
||||
<version.commons-lang3>3.10</version.commons-lang3>
|
||||
|
|
@ -249,7 +249,7 @@
|
|||
<dependency>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<version>9.4.46.v20220331</version>
|
||||
<version>9.4.43.v20210629</version>
|
||||
<scope>runtime</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -197,7 +197,7 @@ public class HiveTypeTranslator
|
|||
if (charLength <= HiveChar.MAX_CHAR_LENGTH) {
|
||||
return getCharTypeInfo(charLength);
|
||||
}
|
||||
throw new PrestoException(NOT_SUPPORTED, String.format(Locale.ROOT, "Unsupported Hive type: %s." +
|
||||
throw new PrestoException(NOT_SUPPORTED, String.format("Unsupported Hive type: %s." +
|
||||
" Supported CHAR types: CHAR(<=%d).",
|
||||
type, HiveChar.MAX_CHAR_LENGTH));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-kylin</artifactId>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -42,10 +42,6 @@ public class HetuEventListenerConfig
|
|||
|
||||
private int auditFileCount = 1;
|
||||
|
||||
private String auditLogFile;
|
||||
|
||||
private String auditLogConversionPattern;
|
||||
|
||||
public BaseEventListener.Type getType()
|
||||
{
|
||||
return type;
|
||||
|
|
@ -243,42 +239,4 @@ public class HetuEventListenerConfig
|
|||
this.auditFileLimit = auditFileLimit;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* set auditLogfile
|
||||
*
|
||||
* @param auditLogFile auditlogfile from properties file
|
||||
* @return config object
|
||||
*/
|
||||
@Config("hetu.auditlog.logoutput")
|
||||
@ConfigDescription("AuditLog event output file path")
|
||||
public HetuEventListenerConfig setAuditLogFile(String auditLogFile)
|
||||
{
|
||||
this.auditLogFile = auditLogFile;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* set auditLogConversionPattern
|
||||
*
|
||||
* @param auditLogConversionPattern auditLogConversionPattern from properties file
|
||||
* @return config object
|
||||
*/
|
||||
@Config("hetu.auditlog.logconversionpattern")
|
||||
@ConfigDescription("define the ConversionPattern of AuditLog event")
|
||||
public HetuEventListenerConfig setAuditLogConversionPattern(String auditLogConversionPattern)
|
||||
{
|
||||
this.auditLogConversionPattern = auditLogConversionPattern;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getAuditLogFile()
|
||||
{
|
||||
return auditLogFile;
|
||||
}
|
||||
|
||||
public String getAuditLogConversionPattern()
|
||||
{
|
||||
return auditLogConversionPattern;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ package io.hetu.core.eventlistener;
|
|||
|
||||
import io.airlift.configuration.ConfigurationFactory;
|
||||
import io.hetu.core.eventlistener.listeners.BaseEventListener;
|
||||
import io.hetu.core.eventlistener.util.HetuLogUtil;
|
||||
import io.prestosql.spi.eventlistener.EventListener;
|
||||
import io.prestosql.spi.eventlistener.EventListenerFactory;
|
||||
|
||||
|
|
@ -42,7 +41,6 @@ public class HetuEventListenerFactory
|
|||
{
|
||||
ConfigurationFactory configurationFactory = new ConfigurationFactory(properties);
|
||||
HetuEventListenerConfig config = configurationFactory.build(HetuEventListenerConfig.class);
|
||||
HetuLogUtil.create(config);
|
||||
return BaseEventListener.create(config);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ package io.hetu.core.eventlistener.listeners;
|
|||
|
||||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.eventlistener.HetuEventListenerConfig;
|
||||
import io.hetu.core.eventlistener.util.HetuLogUtil;
|
||||
import io.hetu.core.eventlistener.util.ListenerErrorCode;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.eventlistener.QueryCompletedEvent;
|
||||
|
|
@ -67,16 +66,6 @@ class AuditEventLogger
|
|||
protected void onQueryCreatedEvent(QueryCreatedEvent queryCreatedEvent)
|
||||
{
|
||||
QueryContext queryContext = queryCreatedEvent.getContext();
|
||||
java.util.logging.Logger log = HetuLogUtil.getLoggerByName(queryContext.getUser(), "INFO", HetuLogUtil.AuditType.Sql);
|
||||
|
||||
log.info(queryCreatedEvent.getCreateTime().atZone(ZoneId.systemDefault()) +
|
||||
" UserName=" + queryContext.getUser() +
|
||||
" UserIp=" + queryContext.getRemoteClientAddress().orElse(null) +
|
||||
" queryId=" + queryCreatedEvent.getMetadata().getQueryId() +
|
||||
" operation= " + queryCreatedEvent.getClass().getSimpleName() +
|
||||
" stmt={" + queryCreatedEvent.getMetadata().getQuery() +
|
||||
"} status=" + queryCreatedEvent.getMetadata().getQueryState() + " " +
|
||||
queryCreatedEvent.getClass().getCanonicalName());
|
||||
logger.info(queryCreatedEvent.getCreateTime().atZone(ZoneId.systemDefault()) +
|
||||
" UserName=" + queryContext.getUser() +
|
||||
" UserIp=" + queryContext.getRemoteClientAddress().orElse(null) +
|
||||
|
|
@ -91,18 +80,6 @@ class AuditEventLogger
|
|||
protected void onQueryCompletedEvent(QueryCompletedEvent queryCompletedEvent)
|
||||
{
|
||||
QueryContext queryContext = queryCompletedEvent.getContext();
|
||||
java.util.logging.Logger log = HetuLogUtil.getLoggerByName(queryContext.getUser(), "INFO", HetuLogUtil.AuditType.Sql);
|
||||
log.info(queryCompletedEvent.getCreateTime().atZone(ZoneId.systemDefault()) +
|
||||
" UserName=" + queryContext.getUser() +
|
||||
" UserIp=" + queryContext.getRemoteClientAddress().orElse(null) +
|
||||
" queryId=" + queryCompletedEvent.getMetadata().getQueryId() +
|
||||
" operation= " + queryCompletedEvent.getClass().getSimpleName() +
|
||||
" WrittenRows= " + queryCompletedEvent.getStatistics().getWrittenRows() +
|
||||
" TotalRows= " + queryCompletedEvent.getStatistics().getTotalRows() +
|
||||
" OutputRows= " + queryCompletedEvent.getStatistics().getOutputRows() +
|
||||
" stmt={" + queryCompletedEvent.getMetadata().getQuery() +
|
||||
"} status=" + queryCompletedEvent.getMetadata().getQueryState() +
|
||||
" " + queryCompletedEvent.getClass().getCanonicalName());
|
||||
logger.info(queryCompletedEvent.getCreateTime().atZone(ZoneId.systemDefault()) +
|
||||
" UserName=" + queryContext.getUser() +
|
||||
" UserIp=" + queryContext.getRemoteClientAddress().orElse(null) +
|
||||
|
|
|
|||
|
|
@ -15,18 +15,11 @@
|
|||
package io.hetu.core.eventlistener.listeners;
|
||||
|
||||
import io.hetu.core.eventlistener.HetuEventListenerConfig;
|
||||
import io.hetu.core.eventlistener.util.HetuLogUtil;
|
||||
import io.prestosql.spi.eventlistener.AuditLogEvent;
|
||||
import io.prestosql.spi.eventlistener.EventListener;
|
||||
import io.prestosql.spi.eventlistener.QueryCompletedEvent;
|
||||
import io.prestosql.spi.eventlistener.QueryCreatedEvent;
|
||||
import io.prestosql.spi.eventlistener.SplitCompletedEvent;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.ZoneId;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public abstract class BaseEventListener
|
||||
|
|
@ -67,6 +60,7 @@ public abstract class BaseEventListener
|
|||
public static EventListener create(HetuEventListenerConfig config)
|
||||
{
|
||||
Type type = config.getType();
|
||||
|
||||
if (type == Type.AUDIT) {
|
||||
return new AuditEventLogger(config);
|
||||
}
|
||||
|
|
@ -110,30 +104,6 @@ public abstract class BaseEventListener
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public final void auditLogged(AuditLogEvent auditLogEvent)
|
||||
{
|
||||
Logger log = HetuLogUtil.getLoggerByName(auditLogEvent.getUser(), auditLogEvent.getLevel(), HetuLogUtil.AuditType.valueOf(auditLogEvent.getType()));
|
||||
ZonedDateTime zonedDateTime = LocalDateTime.now().atZone(ZoneId.systemDefault());
|
||||
String logRecord = zonedDateTime + " " + auditLogEvent;
|
||||
switch (auditLogEvent.getLevel()) {
|
||||
case "INFO":
|
||||
log.info(logRecord);
|
||||
break;
|
||||
case "ERROR":
|
||||
log.severe(logRecord);
|
||||
break;
|
||||
case "WARN":
|
||||
log.warning(logRecord);
|
||||
break;
|
||||
case "DEBUG":
|
||||
log.fine(logRecord);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* call back for query created
|
||||
*
|
||||
|
|
|
|||
|
|
@ -17,11 +17,9 @@ package io.hetu.core.eventlistener.listeners;
|
|||
import io.airlift.log.Logger;
|
||||
import io.hetu.core.eventlistener.HetuEventListenerConfig;
|
||||
import io.hetu.core.eventlistener.util.EventUtility;
|
||||
import io.hetu.core.eventlistener.util.HetuLogUtil;
|
||||
import io.hetu.core.eventlistener.util.ListenerErrorCode;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.eventlistener.QueryCompletedEvent;
|
||||
import io.prestosql.spi.eventlistener.QueryContext;
|
||||
import io.prestosql.spi.eventlistener.QueryCreatedEvent;
|
||||
import io.prestosql.spi.eventlistener.SplitCompletedEvent;
|
||||
|
||||
|
|
@ -67,18 +65,12 @@ class QueryEventLogger
|
|||
@Override
|
||||
protected void onQueryCreatedEvent(QueryCreatedEvent queryCreatedEvent)
|
||||
{
|
||||
QueryContext queryContext = queryCreatedEvent.getContext();
|
||||
java.util.logging.Logger log = HetuLogUtil.getLoggerByName(queryContext.getUser(), "INFO", HetuLogUtil.AuditType.Sql);
|
||||
log.info(EventUtility.toString(queryCreatedEvent));
|
||||
logger.info(EventUtility.toString(queryCreatedEvent));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onQueryCompletedEvent(QueryCompletedEvent queryCompletedEvent)
|
||||
{
|
||||
QueryContext queryContext = queryCompletedEvent.getContext();
|
||||
java.util.logging.Logger log = HetuLogUtil.getLoggerByName(queryContext.getUser(), "INFO", HetuLogUtil.AuditType.Sql);
|
||||
log.info(EventUtility.toString(queryCompletedEvent));
|
||||
logger.info(EventUtility.toString(queryCompletedEvent));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,129 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.eventlistener.util;
|
||||
|
||||
import io.hetu.core.eventlistener.HetuEventListenerConfig;
|
||||
import io.prestosql.spi.PrestoException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.logging.FileHandler;
|
||||
import java.util.logging.Formatter;
|
||||
import java.util.logging.Handler;
|
||||
import java.util.logging.LogRecord;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
public class HetuLogUtil
|
||||
{
|
||||
private static String logConversionPattern;
|
||||
|
||||
private static String logOutput;
|
||||
|
||||
private static ConcurrentHashMap<String, Logger> loggerConcurrentHashMap;
|
||||
|
||||
private HetuLogUtil(){}
|
||||
|
||||
public static void create(HetuEventListenerConfig config)
|
||||
{
|
||||
logConversionPattern = config.getAuditLogConversionPattern();
|
||||
logOutput = config.getAuditLogFile();
|
||||
loggerConcurrentHashMap = new ConcurrentHashMap<>();
|
||||
createFile(config.getAuditLogFile());
|
||||
}
|
||||
|
||||
private static String getCurrentDate(String logConversionPattern)
|
||||
{
|
||||
String hetuLogConversionPattern = logConversionPattern;
|
||||
if (hetuLogConversionPattern == null) {
|
||||
hetuLogConversionPattern = "yyyy-MM-dd.HH";
|
||||
}
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
DateTimeFormatter format = DateTimeFormatter.ofPattern(hetuLogConversionPattern);
|
||||
return now.format(format);
|
||||
}
|
||||
|
||||
public static Logger getLoggerByName(String username, String level, AuditType type)
|
||||
{
|
||||
String fileName = logOutput + "/" + type + "/" + level + "#" + username + "#" + getCurrentDate(logConversionPattern) + ".log";
|
||||
java.util.logging.Logger logger = java.util.logging.Logger.getLogger(fileName);
|
||||
if (logOutput == null) {
|
||||
return logger;
|
||||
}
|
||||
try {
|
||||
if (loggerConcurrentHashMap.containsKey(fileName)) {
|
||||
return loggerConcurrentHashMap.get(fileName);
|
||||
}
|
||||
synchronized (HetuLogUtil.class) {
|
||||
if (loggerConcurrentHashMap.containsKey(fileName)) {
|
||||
return loggerConcurrentHashMap.get(fileName);
|
||||
}
|
||||
FileHandler fileHandler = new FileHandler(fileName, 0, 1, true);
|
||||
fileHandler.setFormatter(new Formatter(){
|
||||
@Override
|
||||
public String format(LogRecord logRecord)
|
||||
{
|
||||
return logRecord.getMessage() + "\n";
|
||||
}
|
||||
});
|
||||
logger.addHandler(fileHandler);
|
||||
logger.setUseParentHandlers(false);
|
||||
loggerConcurrentHashMap.put(fileName, logger);
|
||||
clearUnusedLogger();
|
||||
}
|
||||
return loggerConcurrentHashMap.get(fileName);
|
||||
}
|
||||
catch (IOException ex) {
|
||||
throw new PrestoException(ListenerErrorCode.LOCAL_FILE_FILESYSTEM_ERROR,
|
||||
"failed to create logger writing to " + fileName, ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static void clearUnusedLogger()
|
||||
{
|
||||
String currentTime = getCurrentDate(logConversionPattern);
|
||||
for (String key : loggerConcurrentHashMap.keySet()) {
|
||||
String loggerTime = key.substring(key.lastIndexOf("#") + 1, key.length() - 4);
|
||||
if (loggerTime.compareTo(currentTime) < 0) {
|
||||
Logger deadLogger = loggerConcurrentHashMap.get(key);
|
||||
for (Handler h : deadLogger.getHandlers()) {
|
||||
h.close();
|
||||
}
|
||||
loggerConcurrentHashMap.remove(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void createFile(String auditLogFile)
|
||||
{
|
||||
if (auditLogFile == null) {
|
||||
return;
|
||||
}
|
||||
File logFileSql = new File(auditLogFile + "/Sql");
|
||||
File logFileWebUi = new File(auditLogFile + "/WebUi");
|
||||
File logFileCluster = new File(auditLogFile + "/Cluster");
|
||||
logFileSql.mkdir();
|
||||
logFileWebUi.mkdir();
|
||||
logFileCluster.mkdir();
|
||||
}
|
||||
|
||||
public enum AuditType{
|
||||
WebUi,
|
||||
Sql,
|
||||
Cluster
|
||||
}
|
||||
}
|
||||
|
|
@ -38,9 +38,7 @@ public class TestHetuEventListenerConfig
|
|||
.setLogFileCount(1)
|
||||
.setAuditFile(null)
|
||||
.setAuditFileLimit(0)
|
||||
.setAuditFileCount(1)
|
||||
.setAuditLogFile(null)
|
||||
.setAuditLogConversionPattern(null));
|
||||
.setAuditFileCount(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -57,8 +55,6 @@ public class TestHetuEventListenerConfig
|
|||
.put("hetu.event.listener.audit.filecount", "10")
|
||||
.put("hetu.event.listener.audit.limit", "1024")
|
||||
.put("hetu.event.listener.audit.file", "/var/hetu-audit.log")
|
||||
.put("hetu.auditlog.logoutput", "/var/log")
|
||||
.put("hetu.auditlog.logconversionpattern", "yyyy-MM-dd.HH")
|
||||
.build();
|
||||
|
||||
HetuEventListenerConfig expected = new HetuEventListenerConfig().setType(BaseEventListener.Type.LOGGER)
|
||||
|
|
@ -70,9 +66,7 @@ public class TestHetuEventListenerConfig
|
|||
.setLogFileLimit(1024)
|
||||
.setAuditFile("/var/hetu-audit.log")
|
||||
.setAuditFileLimit(1024)
|
||||
.setAuditFileCount(10)
|
||||
.setAuditLogFile("/var/log")
|
||||
.setAuditLogConversionPattern("yyyy-MM-dd.HH");
|
||||
.setAuditFileCount(10);
|
||||
|
||||
assertFullMapping(properties, expected);
|
||||
}
|
||||
|
|
@ -91,8 +85,6 @@ public class TestHetuEventListenerConfig
|
|||
.put("hetu.event.listener.audit.filecount", "10")
|
||||
.put("hetu.event.listener.audit.limit", "1024")
|
||||
.put("hetu.event.listener.audit.file", "/var/hetu-audit.log")
|
||||
.put("hetu.auditlog.logoutput", "/var/log")
|
||||
.put("hetu.auditlog.logconversionpattern", "yyyy-MM-dd.HH")
|
||||
.build();
|
||||
|
||||
HetuEventListenerConfig expected = new HetuEventListenerConfig().setType(BaseEventListener.Type.AUDIT)
|
||||
|
|
@ -104,9 +96,7 @@ public class TestHetuEventListenerConfig
|
|||
.setLogFileLimit(1024)
|
||||
.setAuditFile("/var/hetu-audit.log")
|
||||
.setAuditFileLimit(1024)
|
||||
.setAuditFileCount(10)
|
||||
.setAuditLogFile("/var/log")
|
||||
.setAuditLogConversionPattern("yyyy-MM-dd.HH");
|
||||
.setAuditFileCount(10);
|
||||
|
||||
assertFullMapping(properties, expected);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -15,15 +15,11 @@
|
|||
package io.hetu.core.metastore;
|
||||
|
||||
import io.airlift.log.Logger;
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.favorite.FavoriteResult;
|
||||
import io.prestosql.spi.metastore.HetuCache;
|
||||
import io.prestosql.spi.metastore.HetuMetastore;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryResult;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
|
@ -310,60 +306,6 @@ public class HetuMetastoreCache
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertQueryHistory(QueryHistoryEntity queryHistoryEntity, String jsonString)
|
||||
{
|
||||
delegate.insertQueryHistory(queryHistoryEntity, jsonString);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteQueryHistoryBatch()
|
||||
{
|
||||
delegate.deleteQueryHistoryBatch();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAllQueryHistoryNum()
|
||||
{
|
||||
return delegate.getAllQueryHistoryNum();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryDetail(String queryId)
|
||||
{
|
||||
return delegate.getQueryDetail(queryId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryHistoryResult getQueryHistory(int startNum, int pageSize,
|
||||
String user, String startTime, String endTime,
|
||||
String queryId, String query, String resourceGroup,
|
||||
String resource, List<String> state, List<String> failed,
|
||||
String sort, String sortOrder)
|
||||
{
|
||||
return delegate.getQueryHistory(startNum, pageSize,
|
||||
user, startTime, endTime, queryId, query,
|
||||
resourceGroup, resource, state, failed, sort, sortOrder);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
delegate.insertFavorite(favoriteEntity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
delegate.deleteFavorite(favoriteEntity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FavoriteResult getFavorite(int startNum, int pageSize, String user)
|
||||
{
|
||||
return delegate.getFavorite(startNum, pageSize, user);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<TableEntity> getTable(String catalogName, String databaseName, String tableName)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,14 +14,10 @@
|
|||
*/
|
||||
package io.hetu.core.metastore;
|
||||
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.favorite.FavoriteResult;
|
||||
import io.prestosql.spi.metastore.HetuMetastore;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryResult;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
|
|
@ -164,58 +160,4 @@ public class HetuMetastoreNone
|
|||
{
|
||||
delegate.alterTableParameter(catalogName, databaseName, tableName, key, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertQueryHistory(QueryHistoryEntity queryHistoryEntity, String jsonString)
|
||||
{
|
||||
delegate.insertQueryHistory(queryHistoryEntity, jsonString);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteQueryHistoryBatch()
|
||||
{
|
||||
delegate.deleteQueryHistoryBatch();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAllQueryHistoryNum()
|
||||
{
|
||||
return delegate.getAllQueryHistoryNum();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryDetail(String queryId)
|
||||
{
|
||||
return delegate.getQueryDetail(queryId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryHistoryResult getQueryHistory(int startNum, int pageSize,
|
||||
String user, String startTime, String endTime,
|
||||
String queryId, String query, String resourceGroup,
|
||||
String resource, List<String> state, List<String> failed,
|
||||
String sort, String sortOrder)
|
||||
{
|
||||
return delegate.getQueryHistory(startNum, pageSize,
|
||||
user, startTime, endTime, queryId, query,
|
||||
resourceGroup, resource, state, failed, sort, sortOrder);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
delegate.insertFavorite(favoriteEntity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
delegate.deleteFavorite(favoriteEntity);
|
||||
}
|
||||
|
||||
@Override
|
||||
public FavoriteResult getFavorite(int startNum, int pageSize, String user)
|
||||
{
|
||||
return delegate.getFavorite(startNum, pageSize, user);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,16 +30,12 @@ import io.prestosql.spi.connector.SchemaNotFoundException;
|
|||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.connector.TableAlreadyExistsException;
|
||||
import io.prestosql.spi.connector.TableNotFoundException;
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.favorite.FavoriteResult;
|
||||
import io.prestosql.spi.filesystem.FileBasedLock;
|
||||
import io.prestosql.spi.filesystem.HetuFileSystemClient;
|
||||
import io.prestosql.spi.metastore.HetuMetastore;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryResult;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
||||
|
|
@ -800,52 +796,4 @@ public class HetuFsMetastore
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertQueryHistory(QueryHistoryEntity queryHistoryEntity, String jsonString)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteQueryHistoryBatch()
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAllQueryHistoryNum()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryDetail(String queryId)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryHistoryResult getQueryHistory(int startNum, int pageSize,
|
||||
String user, String startTime, String endTime,
|
||||
String queryId, String query, String resourceGroup,
|
||||
String resource, List<String> state, List<String> failed,
|
||||
String sort, String sortOrder)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public FavoriteResult getFavorite(int startNum, int pageSize, String user)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.metastore.jdbc;
|
||||
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import org.jdbi.v3.core.result.LinkedHashMapRowReducer;
|
||||
import org.jdbi.v3.core.result.RowView;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class FavoriteEntityReducer
|
||||
implements LinkedHashMapRowReducer<Long, FavoriteEntity>
|
||||
{
|
||||
@Override
|
||||
public void accumulate(Map<Long, FavoriteEntity> map, RowView rowView)
|
||||
{
|
||||
map.computeIfAbsent(rowView.getColumn("id", Long.class),
|
||||
id -> rowView.getRow(FavoriteEntity.class));
|
||||
}
|
||||
}
|
||||
|
|
@ -20,15 +20,11 @@ import io.prestosql.spi.connector.CatalogNotFoundException;
|
|||
import io.prestosql.spi.connector.SchemaNotFoundException;
|
||||
import io.prestosql.spi.connector.SchemaTableName;
|
||||
import io.prestosql.spi.connector.TableNotFoundException;
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.favorite.FavoriteResult;
|
||||
import io.prestosql.spi.metastore.HetuMetastore;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.ColumnEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryResult;
|
||||
import org.jdbi.v3.core.Jdbi;
|
||||
import org.jdbi.v3.sqlobject.SqlObjectPlugin;
|
||||
|
||||
|
|
@ -436,92 +432,6 @@ public class JdbcHetuMetastore
|
|||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertQueryHistory(QueryHistoryEntity queryHistoryEntity, String jsonString)
|
||||
{
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
transactionDao.insertQueryHistory(queryHistoryEntity, jsonString);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteQueryHistoryBatch()
|
||||
{
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
transactionDao.deleteQueryHistoryBatch();
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAllQueryHistoryNum()
|
||||
{
|
||||
long[] ans = new long[1];
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
ans[0] = transactionDao.getAllQueryHistoryNum();
|
||||
});
|
||||
return ans[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryDetail(String queryId)
|
||||
{
|
||||
String[] jsonString = new String[1];
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
jsonString[0] = transactionDao.getQueryDetail(queryId);
|
||||
});
|
||||
return jsonString[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryHistoryResult getQueryHistory(int startNum, int pageSize,
|
||||
String user, String startTime, String endTime,
|
||||
String queryId, String query, String resourceGroup,
|
||||
String resource, List<String> state, List<String> failed,
|
||||
String sort, String sortOrder)
|
||||
{
|
||||
QueryHistoryResult queryHistoryResult = new QueryHistoryResult();
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
queryHistoryResult.setTotal(transactionDao.getQueryHistoryCount(user, startTime, endTime, queryId, query, resourceGroup, resource, state, failed));
|
||||
queryHistoryResult.setQueries(transactionDao.getQueryHistory(startNum, pageSize, user, startTime, endTime, queryId, query, resourceGroup, resource, state, failed, sort, sortOrder));
|
||||
});
|
||||
return queryHistoryResult;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
transactionDao.insertFavorite(favoriteEntity);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
transactionDao.deleteFavorite(favoriteEntity);
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public FavoriteResult getFavorite(int startNum, int pageSize, String user)
|
||||
{
|
||||
FavoriteResult favoriteResult = new FavoriteResult();
|
||||
runTransactionWithLock(jdbi, handle -> {
|
||||
JdbcMetadataDao transactionDao = handle.attach(JdbcMetadataDao.class);
|
||||
favoriteResult.setQueries(transactionDao.getFavorite(startNum, pageSize, user));
|
||||
favoriteResult.setTotal(transactionDao.getFavoriteCount(user));
|
||||
});
|
||||
return favoriteResult;
|
||||
}
|
||||
|
||||
private void getTableColumns(List<Map.Entry<Long, TableEntity>> tableEntries,
|
||||
ImmutableList.Builder<TableEntity> tables, JdbcMetadataDao transactionDao)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,17 +14,13 @@
|
|||
*/
|
||||
package io.hetu.core.metastore.jdbc;
|
||||
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.ColumnEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import org.jdbi.v3.sqlobject.config.RegisterBeanMapper;
|
||||
import org.jdbi.v3.sqlobject.customizer.Bind;
|
||||
import org.jdbi.v3.sqlobject.customizer.BindBean;
|
||||
import org.jdbi.v3.sqlobject.customizer.BindList;
|
||||
import org.jdbi.v3.sqlobject.customizer.Define;
|
||||
import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys;
|
||||
import org.jdbi.v3.sqlobject.statement.SqlBatch;
|
||||
import org.jdbi.v3.sqlobject.statement.SqlQuery;
|
||||
|
|
@ -45,8 +41,6 @@ import java.util.Optional;
|
|||
@RegisterBeanMapper(value = TableEntity.class, prefix = "t")
|
||||
@RegisterBeanMapper(value = ColumnEntity.class, prefix = "c")
|
||||
@RegisterBeanMapper(value = PropertyEntity.class, prefix = "p")
|
||||
@RegisterBeanMapper(value = QueryHistoryEntity.class)
|
||||
@RegisterBeanMapper(value = FavoriteEntity.class)
|
||||
|
||||
public interface JdbcMetadataDao
|
||||
{
|
||||
|
|
@ -473,142 +467,4 @@ public interface JdbcMetadataDao
|
|||
*/
|
||||
@SqlQuery("SELECT id FROM hetu_tab_lock WHERE resource=1")
|
||||
Long getLockId();
|
||||
|
||||
/**
|
||||
* insert queryHistoryEntity
|
||||
*
|
||||
* @param queryHistoryEntity queryHistory Entity
|
||||
*/
|
||||
@SqlUpdate("INSERT IGNORE INTO hetu_query_history \n" +
|
||||
"(user,source,queryId,resource,query,state,failed,\n" +
|
||||
"createTime,elapsedTime,cpuTime,executionTime,\n" +
|
||||
"catalog,schemata,currentMemory,cumulativeUserMemory,\n" +
|
||||
"jsonString,completedDrivers,runningDrivers,\n" +
|
||||
"queuedDrivers,totalCpuTime,totalMemoryReservation,\n" +
|
||||
"peakTotalMemoryReservation) \n" +
|
||||
"VALUES (:user,:source,:queryId,:resource,:query,:state,:failed,\n" +
|
||||
":createTime,:elapsedTime,:cpuTime,:executionTime,\n" +
|
||||
":catalog,:schemata,:currentMemory,:cumulativeUserMemory,\n" +
|
||||
":jsonString,:completedDrivers,:runningDrivers,\n" +
|
||||
":queuedDrivers,:totalCpuTime,:totalMemoryReservation,\n" +
|
||||
":peakTotalMemoryReservation)")
|
||||
void insertQueryHistory(@BindBean QueryHistoryEntity queryHistoryEntity, @Bind("jsonString") String jsonString);
|
||||
|
||||
/**
|
||||
* delete the oldest 100 queryHistoryEntities
|
||||
*
|
||||
*/
|
||||
@SqlUpdate("DELETE FROM hetu_query_history \n" +
|
||||
"WHERE 1=1 \n" +
|
||||
"ORDER BY id \n" +
|
||||
"LIMIT 100")
|
||||
void deleteQueryHistoryBatch();
|
||||
|
||||
/**
|
||||
* get QueryDetail
|
||||
*
|
||||
* @param queryId
|
||||
* @return QueryDetail
|
||||
*/
|
||||
@SqlQuery("SELECT jsonString FROM hetu_query_history WHERE queryId = :queryId")
|
||||
String getQueryDetail(@Bind("queryId") String queryId);
|
||||
|
||||
/**
|
||||
* get all QueryHistory
|
||||
*
|
||||
* @param
|
||||
* @return QueryHistoryEntities
|
||||
*/
|
||||
@SqlQuery("SELECT id,user,source,queryId,resource,query,state,failed, " +
|
||||
"createTime,elapsedTime,cpuTime,executionTime," +
|
||||
"catalog,schemata,currentMemory,cumulativeUserMemory," +
|
||||
"completedDrivers,runningDrivers,queuedDrivers,totalCpuTime," +
|
||||
"totalMemoryReservation,peakTotalMemoryReservation " +
|
||||
"FROM hetu_query_history WHERE\n" +
|
||||
"(user like concat(\'%\',:user,\'%\')) AND" +
|
||||
"(:startTime='' or createTime >= :startTime) AND" +
|
||||
"(:endTime='' or createTime <= :endTime) AND" +
|
||||
"(queryId like concat(\'%\',:queryId)) AND" +
|
||||
"(query like concat(\'%\',:query,\'%\')) AND" +
|
||||
"(:resourceGroup='' or resource = :resourceGroup) AND" +
|
||||
"(:resource='' or source = :resource) AND" +
|
||||
"(state in (<state>) ) AND" +
|
||||
"(failed in (<failed>) )" +
|
||||
"ORDER BY <sort> <sortOrder> LIMIT :startNum , :pageSize")
|
||||
@UseRowReducer(QueryHistoryEntityReducer.class)
|
||||
List<QueryHistoryEntity> getQueryHistory(@Bind("startNum") int startNum, @Bind("pageSize") int pageSize,
|
||||
@Bind("user") String user, @Bind("startTime") String startTime, @Bind("endTime") String endTime,
|
||||
@Bind("queryId") String queryId, @Bind("query") String query, @Bind("resourceGroup") String resourceGroup,
|
||||
@Bind("resource") String resource, @BindList("state") List<String> state, @BindList("failed") List<String> failed,
|
||||
@Define("sort") String sort, @Define("sortOrder") String sortOrder);
|
||||
|
||||
/**
|
||||
* get QueryHistory counts
|
||||
*
|
||||
* @param
|
||||
* @return counts
|
||||
*/
|
||||
@SqlQuery("SELECT count(*) FROM hetu_query_history WHERE \n" +
|
||||
"(user like concat(\'%\',:user,\'%\')) AND" +
|
||||
"(:startTime='' or createTime >= :startTime) AND" +
|
||||
"(:endTime='' or createTime <= :endTime) AND" +
|
||||
"(queryId like concat(\'%\',:queryId)) AND" +
|
||||
"(query like concat(\'%\',:query,\'%\')) AND" +
|
||||
"(:resourceGroup='' or resource = :resourceGroup) AND" +
|
||||
"(:resource='' or source = :resource) AND" +
|
||||
"(state in (<state>)) AND" +
|
||||
"(failed in (<failed>))")
|
||||
long getQueryHistoryCount(@Bind("user") String user, @Bind("startTime") String startTime, @Bind("endTime") String endTime,
|
||||
@Bind("queryId") String queryId, @Bind("query") String query, @Bind("resourceGroup") String resourceGroup,
|
||||
@Bind("resource") String resource, @BindList("state") List<String> state, @BindList("failed") List<String> failed);
|
||||
|
||||
/**
|
||||
* get all QueryHistory counts
|
||||
*
|
||||
* @param
|
||||
* @return counts
|
||||
*/
|
||||
@SqlQuery("SELECT count(1) FROM hetu_query_history")
|
||||
long getAllQueryHistoryNum();
|
||||
|
||||
/**
|
||||
* insert favoriteEntity
|
||||
*
|
||||
* @param favoriteEntity favorite Entity
|
||||
*/
|
||||
@SqlUpdate("INSERT IGNORE INTO hetu_favorite \n" +
|
||||
"(creationTime, user, query, catalog, schemata)\n" +
|
||||
"VALUES(DEFAULT, :user, :query, :catalog, :schemata)")
|
||||
void insertFavorite(@BindBean FavoriteEntity favoriteEntity);
|
||||
|
||||
/**
|
||||
* delete favoriteEntity
|
||||
*
|
||||
* @param favoriteEntity
|
||||
*/
|
||||
@SqlUpdate("DELETE FROM hetu_favorite WHERE \n" +
|
||||
" user = :user and query = :query and catalog = :catalog and schemata = :schemata")
|
||||
void deleteFavorite(@BindBean FavoriteEntity favoriteEntity);
|
||||
|
||||
/**
|
||||
* get all favoriteEntity
|
||||
*
|
||||
* @param user
|
||||
* @return favoriteEntities
|
||||
*/
|
||||
@SqlQuery("SELECT id, user, query, catalog, schemata \n" +
|
||||
"FROM hetu_favorite WHERE user = :user \n" +
|
||||
"ORDER BY creationTime desc \n" +
|
||||
"LIMIT :startNum, :pageSize")
|
||||
@UseRowReducer(FavoriteEntityReducer.class)
|
||||
List<FavoriteEntity> getFavorite(@Bind("startNum")int startNum, @Bind("pageSize")int pageSize, @Bind("user")String user);
|
||||
|
||||
/**
|
||||
* get all favoriteEntity
|
||||
*
|
||||
* @param user
|
||||
* @return favoriteEntities
|
||||
*/
|
||||
@SqlQuery("SELECT count(*) FROM hetu_favorite WHERE user = :user")
|
||||
long getFavoriteCount(@Bind("user")String user);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -178,9 +178,5 @@ public class JdbcMetadataUtil
|
|||
tableDao.createTableColumnParameters();
|
||||
// hetu_tab_lock table
|
||||
tableDao.createTableLock();
|
||||
//hetu_query_history table
|
||||
tableDao.createTableQueryHistory();
|
||||
//hetu_favorite table
|
||||
tableDao.createTableFavorite();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -144,51 +144,4 @@ public interface MetadataTableDao
|
|||
+ "UNIQUE (resource)\n"
|
||||
+ ")")
|
||||
void createTableLock();
|
||||
|
||||
/**
|
||||
* create table of query_history
|
||||
*/
|
||||
@SqlUpdate("CREATE TABLE IF NOT EXISTS hetu_query_history (\n" +
|
||||
" id BIGINT NOT NULL AUTO_INCREMENT,\n" +
|
||||
" user varchar(20) NOT NULL,\n" +
|
||||
" source varchar(30) NOT NULL,\n" +
|
||||
" queryId varchar(50) NOT NULL,\n" +
|
||||
" resource varchar(30) NOT NULL,\n" +
|
||||
" query varchar(10000) NOT NULL,\n" +
|
||||
" state varchar(30) NOT NULL,\n" +
|
||||
" failed varchar(30) DEFAULT NULL,\n" +
|
||||
" createTime varchar(30) DEFAULT NULL,\n" +
|
||||
" elapsedTime varchar(30) NOT NULL,\n" +
|
||||
" cpuTime varchar(30) NOT NULL,\n" +
|
||||
" executionTime varchar(30) NOT NULL,\n" +
|
||||
" catalog varchar(30) NOT NULL,\n" +
|
||||
" schemata varchar(30) NOT NULL,\n" +
|
||||
" currentMemory varchar(30) NOT NULL,\n" +
|
||||
" cumulativeUserMemory double NOT NULL,\n" +
|
||||
" jsonString mediumtext NOT NULL,\n" +
|
||||
" completedDrivers int(11) NOT NULL,\n" +
|
||||
" runningDrivers int(11) NOT NULL,\n" +
|
||||
" queuedDrivers int(11) NOT NULL,\n" +
|
||||
" totalCpuTime varchar(50) NOT NULL,\n" +
|
||||
" totalMemoryReservation varchar(50) NOT NULL,\n" +
|
||||
" peakTotalMemoryReservation varchar(50) NOT NULL,\n" +
|
||||
" PRIMARY KEY (id),\n" +
|
||||
" UNIQUE KEY query_history_queryId_uindex (queryId)\n" +
|
||||
")")
|
||||
void createTableQueryHistory();
|
||||
|
||||
/**
|
||||
* create table of hatu_favorite
|
||||
*/
|
||||
@SqlUpdate("CREATE TABLE IF NOT EXISTS hetu_favorite (\n" +
|
||||
" creationTime datetime DEFAULT CURRENT_TIMESTAMP,\n" +
|
||||
" user varchar(20) NOT NULL,\n" +
|
||||
" query varchar(600) NOT NULL,\n" +
|
||||
" catalog varchar(30) NOT NULL,\n" +
|
||||
" schemata varchar(30) NOT NULL,\n" +
|
||||
" id BIGINT NOT NULL AUTO_INCREMENT,\n" +
|
||||
" PRIMARY KEY (id),\n" +
|
||||
" UNIQUE KEY favorite_uni (user,query,catalog,schemata)\n" +
|
||||
")")
|
||||
void createTableFavorite();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package io.hetu.core.metastore.jdbc;
|
||||
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import org.jdbi.v3.core.result.LinkedHashMapRowReducer;
|
||||
import org.jdbi.v3.core.result.RowView;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class QueryHistoryEntityReducer
|
||||
implements LinkedHashMapRowReducer<Long, QueryHistoryEntity>
|
||||
{
|
||||
@Override
|
||||
public void accumulate(Map<Long, QueryHistoryEntity> map, RowView rowView)
|
||||
{
|
||||
map.computeIfAbsent(rowView.getColumn("id", Long.class),
|
||||
id -> rowView.getRow(QueryHistoryEntity.class));
|
||||
}
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ import com.mongodb.MongoCredential;
|
|||
import com.mongodb.ServerAddress;
|
||||
import io.airlift.configuration.Config;
|
||||
import io.airlift.configuration.DefunctConfig;
|
||||
import io.prestosql.spi.function.Mandatory;
|
||||
|
||||
import javax.validation.constraints.Min;
|
||||
import javax.validation.constraints.NotNull;
|
||||
|
|
@ -91,10 +90,6 @@ public class MongoClientConfig
|
|||
return seeds;
|
||||
}
|
||||
|
||||
@Mandatory(name = "mongodb.seeds",
|
||||
description = "List of mongodb servers",
|
||||
defaultValue = "ip:port",
|
||||
required = true)
|
||||
@Config("mongodb.seeds")
|
||||
public MongoClientConfig setSeeds(String commaSeparatedList)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -17,24 +17,12 @@ import com.google.common.collect.ImmutableList;
|
|||
import com.google.common.collect.ImmutableSet;
|
||||
import io.prestosql.spi.Plugin;
|
||||
import io.prestosql.spi.connector.ConnectorFactory;
|
||||
import io.prestosql.spi.function.ConnectorConfig;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorUtil;
|
||||
import io.prestosql.spi.queryeditorui.ConnectorWithProperties;
|
||||
import io.prestosql.spi.type.Type;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import static io.hetu.core.plugin.mongodb.ObjectIdType.OBJECT_ID;
|
||||
|
||||
/**
|
||||
* hetu plugin to use mongodb as a data source.
|
||||
*/
|
||||
@ConnectorConfig(connectorLabel = "MongoDB: Allow access to MongoDB data from openLooKeng",
|
||||
propertiesEnabled = true,
|
||||
docLink = "https://openlookeng.io/docs/docs/connector/mongodb.html",
|
||||
configLink = "https://openlookeng.io/docs/docs/connector/mongodb.html#configuration")
|
||||
public class MongoPlugin
|
||||
implements Plugin
|
||||
{
|
||||
|
|
@ -55,12 +43,4 @@ public class MongoPlugin
|
|||
{
|
||||
return ImmutableList.of(new MongoConnectorFactory("mongodb"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<ConnectorWithProperties> getConnectorWithProperties()
|
||||
{
|
||||
ConnectorConfig connectorConfig = MongoPlugin.class.getAnnotation(ConnectorConfig.class);
|
||||
return ConnectorUtil.assembleConnectorProperties(connectorConfig,
|
||||
Arrays.asList(MongoClientConfig.class.getDeclaredMethods()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
@ -157,7 +157,7 @@
|
|||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-postgresql</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
<scope>compile</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
|
|
@ -176,7 +176,7 @@
|
|||
<dependency>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-postgresql</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
<classifier>tests</classifier>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-oracle</artifactId>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-server-rpm</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,76 +0,0 @@
|
|||
{
|
||||
"requires": true,
|
||||
"lockfileVersion": 1,
|
||||
"dependencies": {
|
||||
"@babel/runtime": {
|
||||
"version": "7.16.3",
|
||||
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.16.3.tgz",
|
||||
"integrity": "sha512-WBwekcqacdY2e9AF/Q7WLFUWmdJGJTkbjqTjoMDgXkVZ3ZRUvOPsLb5KdwISoQVsbP+DQzVZW4Zhci0DvpbNTQ==",
|
||||
"requires": {
|
||||
"regenerator-runtime": "^0.13.4"
|
||||
}
|
||||
},
|
||||
"@juggle/resize-observer": {
|
||||
"version": "3.3.1",
|
||||
"resolved": "https://registry.npmjs.org/@juggle/resize-observer/-/resize-observer-3.3.1.tgz",
|
||||
"integrity": "sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw=="
|
||||
},
|
||||
"@rsuite/icon-font": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@rsuite/icon-font/-/icon-font-4.0.0.tgz",
|
||||
"integrity": "sha512-rZTgpTH3H3HLczCA2rnkWfoMKm0ZXoRzsrkVujfP/FfslnKUMvO6w56pa8pCvhWGpNEPUsLS2ULnFGpTEcup/Q=="
|
||||
},
|
||||
"@rsuite/icons": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@rsuite/icons/-/icons-1.0.2.tgz",
|
||||
"integrity": "sha512-Y7vJNDQpJnFlyYSUXQ2iQ9Meg7+ZKcrIenhpYDdM3c7vYDE/L7pml+hrK28jk6QfV/QkVv5B504D+l7aM6AAJQ==",
|
||||
"requires": {
|
||||
"@rsuite/icon-font": "^4.0.0",
|
||||
"classnames": "^2.2.5",
|
||||
"insert-css": "^2.0.0",
|
||||
"lodash": "^4.17.20"
|
||||
}
|
||||
},
|
||||
"classnames": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.1.tgz",
|
||||
"integrity": "sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA=="
|
||||
},
|
||||
"dom-lib": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/dom-lib/-/dom-lib-3.0.0.tgz",
|
||||
"integrity": "sha512-3US7OyceYve90NSU4MhuE0lK4a7aB1hbb14h45v3CPV/ZolLoEOeuPrdS/A0L8kLnwgVCDiKwlPy/Ajq0u9vfA==",
|
||||
"requires": {
|
||||
"@babel/runtime": "^7.16.0"
|
||||
}
|
||||
},
|
||||
"insert-css": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/insert-css/-/insert-css-2.0.0.tgz",
|
||||
"integrity": "sha1-610Ql7dUL0x56jBg067gfQU4gPQ="
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.21",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
|
||||
"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
|
||||
},
|
||||
"regenerator-runtime": {
|
||||
"version": "0.13.9",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz",
|
||||
"integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA=="
|
||||
},
|
||||
"rsuite-table": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/rsuite-table/-/rsuite-table-5.2.0.tgz",
|
||||
"integrity": "sha512-MiJtfZYSMXqaUdEFQ0Q5A5mNjiB0upZumoqMyd0WGsJDC6Ff8tbnMEDBbcMcQ6GcL2MSJoxi1vz3kIo/mZ1Wew==",
|
||||
"requires": {
|
||||
"@babel/runtime": "^7.12.5",
|
||||
"@juggle/resize-observer": "^3.3.1",
|
||||
"@rsuite/icons": "^1.0.0",
|
||||
"classnames": "^2.2.5",
|
||||
"dom-lib": "^3.0.0",
|
||||
"lodash": "^4.17.21"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-server</artifactId>
|
||||
|
|
|
|||
|
|
@ -184,18 +184,17 @@
|
|||
</artifact>
|
||||
</artifactSet>
|
||||
|
||||
|
||||
<artifactSet to="plugin/redis">
|
||||
<artifact id="${project.groupId}:presto-redis:zip:${project.version}">
|
||||
<unpack />
|
||||
</artifact>
|
||||
</artifactSet>
|
||||
|
||||
<artifactSet to="plugin/accumulo">
|
||||
<artifact id="${project.groupId}:presto-accumulo:zip:${project.version}">
|
||||
<unpack />
|
||||
</artifact>
|
||||
</artifactSet>-->
|
||||
<artifactSet to="plugin/redis">
|
||||
<artifact id="${project.groupId}:presto-redis:zip:${project.version}">
|
||||
<unpack />
|
||||
</artifact>
|
||||
</artifactSet>
|
||||
|
||||
<artifactSet to="plugin/mongodb">
|
||||
<artifact id="${project.groupId}:hetu-mongodb:zip:${project.version}">
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-sql-migration-tool</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-startree</artifactId>
|
||||
|
|
|
|||
|
|
@ -26,14 +26,10 @@ import io.hetu.core.cube.startree.tree.StarTreeMetadataBuilder;
|
|||
import io.hetu.core.spi.cube.CubeMetadata;
|
||||
import io.hetu.core.spi.cube.CubeStatus;
|
||||
import io.hetu.core.spi.cube.io.CubeMetaStore;
|
||||
import io.prestosql.spi.favorite.FavoriteEntity;
|
||||
import io.prestosql.spi.favorite.FavoriteResult;
|
||||
import io.prestosql.spi.metastore.HetuMetastore;
|
||||
import io.prestosql.spi.metastore.model.CatalogEntity;
|
||||
import io.prestosql.spi.metastore.model.DatabaseEntity;
|
||||
import io.prestosql.spi.metastore.model.TableEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryEntity;
|
||||
import io.prestosql.spi.queryhistory.QueryHistoryResult;
|
||||
import io.prestosql.sql.analyzer.FeaturesConfig;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
|
@ -309,53 +305,5 @@ public class TestStarTreeMetaStore
|
|||
|
||||
@Override
|
||||
public void alterTableParameter(String catalogName, String databaseName, String tableName, String key, String value) {}
|
||||
|
||||
@Override
|
||||
public void insertQueryHistory(QueryHistoryEntity queryHistoryEntity, String jsonString)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteQueryHistoryBatch()
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getAllQueryHistoryNum()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getQueryDetail(String queryId)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryHistoryResult getQueryHistory(int startNum, int pageSize,
|
||||
String user, String startTime, String endTime,
|
||||
String queryId, String query, String resourceGroup,
|
||||
String resource, List<String> state, List<String> failed,
|
||||
String sort, String sortOrder)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void insertFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteFavorite(FavoriteEntity favoriteEntity)
|
||||
{
|
||||
}
|
||||
|
||||
@Override
|
||||
public FavoriteResult getFavorite(int startNum, int pageSize, String user)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
|
|
|
|||
|
|
@ -184,11 +184,6 @@ public class PagesSerdeUtil
|
|||
return new PageReader(serde, sliceInput);
|
||||
}
|
||||
|
||||
public static Iterator<Page> readPages(GenericPagesSerde serde, SliceInput sliceInput, int spillPrefetchReadPages, Predicate<InputStream> eof)
|
||||
{
|
||||
return new PageReader(serde, sliceInput, spillPrefetchReadPages, eof);
|
||||
}
|
||||
|
||||
public static Iterator<Page> readPagesDirect(GenericPagesSerde serde, InputStream input, Predicate<InputStream> eof, int spillPrefetchReadPages)
|
||||
{
|
||||
return new PageReaderDirect(serde, input, eof, spillPrefetchReadPages);
|
||||
|
|
@ -200,44 +195,30 @@ public class PagesSerdeUtil
|
|||
private final GenericPagesSerde serde;
|
||||
private final SliceInput input;
|
||||
private final int spillPrefetchReadPages;
|
||||
Predicate<InputStream> eof;
|
||||
LinkedList<Page> prefetchedPages;
|
||||
|
||||
PageReader(GenericPagesSerde serde, SliceInput input)
|
||||
{
|
||||
this(serde, input, DEFAULT_NUM_PAGES_PREFETCH, null);
|
||||
this(serde, input, DEFAULT_NUM_PAGES_PREFETCH);
|
||||
}
|
||||
|
||||
PageReader(GenericPagesSerde serde, SliceInput sliceInput, int spillPrefetchReadPages)
|
||||
{
|
||||
this(serde, sliceInput, spillPrefetchReadPages, null);
|
||||
}
|
||||
|
||||
PageReader(GenericPagesSerde serde, SliceInput input, int spillPrefetchReadPages, Predicate<InputStream> eof)
|
||||
PageReader(GenericPagesSerde serde, SliceInput input, int spillPrefetchReadPages)
|
||||
{
|
||||
this.serde = requireNonNull(serde, "serde is null");
|
||||
this.input = requireNonNull(input, "input is null");
|
||||
checkArgument(spillPrefetchReadPages > 0, "spillPrefetchReadPages cannot be less then 1");
|
||||
this.prefetchedPages = new LinkedList<>();
|
||||
this.spillPrefetchReadPages = spillPrefetchReadPages;
|
||||
this.eof = eof;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Page computeNext()
|
||||
{
|
||||
if (eof != null) {
|
||||
if (eof.test(input) && prefetchedPages.size() == 0) {
|
||||
return endOfData();
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (!input.isReadable() && prefetchedPages.size() == 0) {
|
||||
return endOfData();
|
||||
}
|
||||
if (!input.isReadable() && prefetchedPages.size() == 0) {
|
||||
return endOfData();
|
||||
}
|
||||
if (prefetchedPages.size() == 0) {
|
||||
for (int i = 0; i < spillPrefetchReadPages && (eof != null ? !eof.test(input) : input.isReadable()); i++) {
|
||||
for (int i = 0; i < spillPrefetchReadPages && input.isReadable(); i++) {
|
||||
prefetchedPages.add(serde.deserialize(readSerializedPage(input)));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hetu-vdm</artifactId>
|
||||
|
|
|
|||
11
pom.xml
11
pom.xml
|
|
@ -10,7 +10,7 @@
|
|||
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
<packaging>pom</packaging>
|
||||
|
||||
<name>presto-root</name>
|
||||
|
|
@ -59,7 +59,7 @@
|
|||
<dep.prestosql.version>316</dep.prestosql.version>
|
||||
<dep.jackson.version>2.11.4</dep.jackson.version>
|
||||
<dep.hetu.version>${project.version}</dep.hetu.version>
|
||||
<dep.jetty.version>9.4.46.v20220331</dep.jetty.version>
|
||||
<dep.jetty.version>9.4.43.v20210629</dep.jetty.version>
|
||||
<dep.bcprov.version>1.66</dep.bcprov.version>
|
||||
<dep.hudi.version>0.7.0</dep.hudi.version>
|
||||
<dep.kryo.version>5.0.3</dep.kryo.version>
|
||||
|
|
@ -86,7 +86,6 @@
|
|||
<module>presto-record-decoder</module>
|
||||
<module>presto-expressions</module>
|
||||
<module>presto-kafka</module>
|
||||
<module>presto-redis</module>
|
||||
<module>presto-memory</module>
|
||||
<module>presto-orc</module>
|
||||
<module>presto-parquet</module>
|
||||
|
|
@ -912,7 +911,7 @@
|
|||
<dependency>
|
||||
<groupId>mysql</groupId>
|
||||
<artifactId>mysql-connector-java</artifactId>
|
||||
<version>5.1.49</version>
|
||||
<version>5.1.47</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
|
|
@ -1871,4 +1870,8 @@
|
|||
</build>
|
||||
</profile>
|
||||
</profiles>
|
||||
|
||||
<scm>
|
||||
<tag>branch-1.6</tag>
|
||||
</scm>
|
||||
</project>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-array</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-atop</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-base-jdbc</artifactId>
|
||||
|
|
|
|||
|
|
@ -773,28 +773,6 @@ public class BaseJdbcClient
|
|||
return TableStatistics.empty();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
execute(session, "CREATE SCHEMA " + quoted(schemaName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
execute(session, "DROP SCHEMA " + quoted(schemaName));
|
||||
}
|
||||
|
||||
protected void execute(ConnectorSession session, String query)
|
||||
{
|
||||
try (Connection connection = connectionFactory.openConnection(JdbcIdentity.from(session))) {
|
||||
execute(connection, query);
|
||||
}
|
||||
catch (SQLException e) {
|
||||
throw new PrestoException(JDBC_ERROR, e);
|
||||
}
|
||||
}
|
||||
|
||||
protected void execute(Connection connection, String query)
|
||||
throws SQLException
|
||||
{
|
||||
|
|
|
|||
|
|
@ -189,18 +189,6 @@ public abstract class ForwardingJdbcClient
|
|||
return getDelegate().getTableStatistics(session, handle, tupleDomain);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
getDelegate().createSchema(session, schemaName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
getDelegate().dropSchema(session, schemaName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean supportsLimit()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -121,10 +121,6 @@ public interface JdbcClient
|
|||
|
||||
TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, TupleDomain<ColumnHandle> tupleDomain);
|
||||
|
||||
void createSchema(ConnectorSession session, String schemaName);
|
||||
|
||||
void dropSchema(ConnectorSession session, String schemaName);
|
||||
|
||||
/**
|
||||
* Hetu requires catalog names to create dynamic catalogs.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -310,18 +310,6 @@ public class JdbcMetadata
|
|||
return jdbcClient.getTableStatistics(session, handle, constraint.getSummary());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createSchema(ConnectorSession session, String schemaName, Map<String, Object> properties)
|
||||
{
|
||||
jdbcClient.createSchema(session, schemaName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
jdbcClient.dropSchema(session, schemaName);
|
||||
}
|
||||
|
||||
// added by Hetu so extending class can access the client
|
||||
public JdbcClient getJdbcClient()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -82,18 +82,6 @@ public class TransactionScopeCachingJdbcClient
|
|||
super.dropColumn(identity, handle, column);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
getDelegate().createSchema(session, schemaName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
getDelegate().dropSchema(session, schemaName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnHandle getDeleteRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -241,18 +241,6 @@ public class StatisticsAwareJdbcClient
|
|||
return stats.getTableStatistics.wrap(() -> getDelegate().getTableStatistics(session, handle, tupleDomain));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
stats.createSchema.wrap(() -> getDelegate().createSchema(session, schemaName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void dropSchema(ConnectorSession session, String schemaName)
|
||||
{
|
||||
stats.dropSchema.wrap(() -> getDelegate().dropSchema(session, schemaName));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnHandle getDeleteRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle)
|
||||
{
|
||||
|
|
@ -360,8 +348,6 @@ public class StatisticsAwareJdbcClient
|
|||
private final JdbcApiStats buildInsertSql = new JdbcApiStats();
|
||||
private final JdbcApiStats getPreparedStatement = new JdbcApiStats();
|
||||
private final JdbcApiStats getTableStatistics = new JdbcApiStats();
|
||||
private final JdbcApiStats createSchema = new JdbcApiStats();
|
||||
private final JdbcApiStats dropSchema = new JdbcApiStats();
|
||||
private final JdbcApiStats addColumn = new JdbcApiStats();
|
||||
private final JdbcApiStats dropColumn = new JdbcApiStats();
|
||||
private final JdbcApiStats renameColumn = new JdbcApiStats();
|
||||
|
|
@ -527,20 +513,6 @@ public class StatisticsAwareJdbcClient
|
|||
return getTableStatistics;
|
||||
}
|
||||
|
||||
@Managed
|
||||
@Nested
|
||||
public JdbcApiStats getCreateSchema()
|
||||
{
|
||||
return createSchema;
|
||||
}
|
||||
|
||||
@Managed
|
||||
@Nested
|
||||
public JdbcApiStats getDropSchema()
|
||||
{
|
||||
return dropSchema;
|
||||
}
|
||||
|
||||
@Managed
|
||||
@Nested
|
||||
public JdbcApiStats getAddColumn()
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-benchmark-driver</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-benchmark</artifactId>
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ import static io.prestosql.spi.function.FunctionKind.SCALAR;
|
|||
import static io.prestosql.spi.type.BigintType.BIGINT;
|
||||
import static io.prestosql.sql.relational.SqlToRowExpressionTranslator.translate;
|
||||
import static io.prestosql.testing.TestingPagesSerdeFactory.TESTING_SERDE_FACTORY;
|
||||
import static io.prestosql.testing.TestingRecoveryUtils.NOOP_RECOVERY_UTILS;
|
||||
import static io.prestosql.testing.TestingSession.testSessionBuilder;
|
||||
import static io.prestosql.testing.TestingSnapshotUtils.NOOP_SNAPSHOT_UTILS;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
import static java.util.concurrent.TimeUnit.NANOSECONDS;
|
||||
|
|
@ -298,7 +298,7 @@ public abstract class AbstractOperatorBenchmark
|
|||
localQueryRunner.getScheduler(),
|
||||
new DataSize(256, MEGABYTE),
|
||||
spillSpaceTracker,
|
||||
NOOP_RECOVERY_UTILS)
|
||||
NOOP_SNAPSHOT_UTILS)
|
||||
.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()),
|
||||
setSession,
|
||||
false,
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ public class OrderByBenchmark
|
|||
false,
|
||||
Optional.empty(),
|
||||
new OrderingCompiler(),
|
||||
false, false);
|
||||
false);
|
||||
|
||||
return ImmutableList.of(tableScanOperator, limitOperator, orderByOperator);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,8 +45,8 @@ import java.util.OptionalInt;
|
|||
|
||||
import static io.airlift.units.DataSize.Unit.GIGABYTE;
|
||||
import static io.prestosql.testing.TestingPagesSerdeFactory.TESTING_SERDE_FACTORY;
|
||||
import static io.prestosql.testing.TestingRecoveryUtils.NOOP_RECOVERY_UTILS;
|
||||
import static io.prestosql.testing.TestingSession.testSessionBuilder;
|
||||
import static io.prestosql.testing.TestingSnapshotUtils.NOOP_SNAPSHOT_UTILS;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
public class MemoryLocalQueryRunner
|
||||
|
|
@ -88,7 +88,7 @@ public class MemoryLocalQueryRunner
|
|||
localQueryRunner.getScheduler(),
|
||||
new DataSize(4, GIGABYTE),
|
||||
spillSpaceTracker,
|
||||
NOOP_RECOVERY_UTILS);
|
||||
NOOP_SNAPSHOT_UTILS);
|
||||
|
||||
TaskContext taskContext = queryContext
|
||||
.addTaskContext(new TaskStateMachine(new TaskId("query", 0, 0), localQueryRunner.getExecutor()),
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-benchto-benchmarks</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-cli</artifactId>
|
||||
|
|
|
|||
|
|
@ -218,7 +218,33 @@ Spilled: 20GB
|
|||
readTime.getValue(SECONDS));
|
||||
reprintLine(summary);
|
||||
}
|
||||
printSnapshotStats(stats, true);
|
||||
|
||||
// Snapshot Capture stats All: 100MB/22s/18s, Last: 40MB/10s/7s
|
||||
SnapshotStats snapshotStats = stats.getSnapshotStats();
|
||||
// snapshotStats should be null in case snapshot feature is disabled
|
||||
if (snapshotStats != null) {
|
||||
Duration allCaptureCPUTime = millis(snapshotStats.getTotalCaptureCpuTime());
|
||||
Duration allCaptureWallTime = millis(snapshotStats.getTotalCaptureWallTime());
|
||||
Duration lastCaptureCPUTime = millis(snapshotStats.getLastCaptureCpuTime());
|
||||
Duration lastCaptureWallTime = millis(snapshotStats.getLastCaptureWallTime());
|
||||
String allSnapshotsSize = FormatUtils.formatDataSize(bytes(snapshotStats.getAllCaptureSize()), true);
|
||||
String lastSnapshotSize = FormatUtils.formatDataSize(bytes(snapshotStats.getLastCaptureSize()), true);
|
||||
String captureSummary = String.format("Snapshot Capture: All: %s/%.1fs/%.1fs, Last: %s/%.1fs/%.1fs",
|
||||
allSnapshotsSize, allCaptureCPUTime.getValue(SECONDS), allCaptureWallTime.getValue(SECONDS),
|
||||
lastSnapshotSize, lastCaptureCPUTime.getValue(SECONDS), lastCaptureWallTime.getValue(SECONDS));
|
||||
reprintLine(captureSummary);
|
||||
|
||||
// Snapshot restore stats: 1/100MB/22s/18s
|
||||
long restoreCount = snapshotStats.getSuccessRestoreCount();
|
||||
if (restoreCount > 0) {
|
||||
Duration allRestoreCPUTime = millis(snapshotStats.getTotalRestoreCpuTime());
|
||||
Duration allRestoreWallTime = millis(snapshotStats.getTotalRestoreWallTime());
|
||||
String allRestoreSize = FormatUtils.formatDataSize(bytes(snapshotStats.getTotalRestoreSize()), true);
|
||||
String restoreSummary = String.format(Locale.ROOT, "Snapshot Restore: %d/%s/%.1fs/%.1fs", restoreCount,
|
||||
allRestoreSize, allRestoreCPUTime.getValue(SECONDS), allRestoreWallTime.getValue(SECONDS));
|
||||
reprintLine(restoreSummary);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 0:32 [2.12GB, 15M rows] [67MB/s, 463K rows/s]
|
||||
|
|
@ -235,54 +261,6 @@ Spilled: 20GB
|
|||
out.println();
|
||||
}
|
||||
|
||||
private void printSnapshotStats(StatementStats stats, boolean isFinal)
|
||||
{
|
||||
SnapshotStats snapshotStats = stats.getSnapshotStats();
|
||||
// snapshotStats should be null in case snapshot feature is disabled
|
||||
if (snapshotStats != null) {
|
||||
List<Long> capturedList = snapshotStats.getCapturedSnapshotList();
|
||||
int capturedListSize = capturedList != null ? capturedList.size() : 0;
|
||||
long restoreCount = snapshotStats.getSuccessRestoreCount();
|
||||
String snapshotSummary = null;
|
||||
|
||||
// Display captured snapshot stats if we have any captured snapshots already
|
||||
if (capturedListSize > 0) {
|
||||
Duration allCaptureCPUTime = millis(snapshotStats.getTotalCaptureCpuTime());
|
||||
Duration allCaptureWallTime = millis(snapshotStats.getTotalCaptureWallTime());
|
||||
Duration lastCaptureCPUTime = millis(snapshotStats.getLastCaptureCpuTime());
|
||||
Duration lastCaptureWallTime = millis(snapshotStats.getLastCaptureWallTime());
|
||||
String allSnapshotsSize = FormatUtils.formatDataSize(bytes(snapshotStats.getAllCaptureSize()), true);
|
||||
String lastSnapshotSize = FormatUtils.formatDataSize(bytes(snapshotStats.getLastCaptureSize()), true);
|
||||
|
||||
snapshotSummary = String.format(Locale.ROOT, "Snapshots Captured: %d [All: %s/%.1fs/%.1fs, Last: %s/%.1fs/%.1fs]",
|
||||
capturedListSize,
|
||||
allSnapshotsSize, allCaptureCPUTime.getValue(SECONDS), allCaptureWallTime.getValue(SECONDS),
|
||||
lastSnapshotSize, lastCaptureCPUTime.getValue(SECONDS), lastCaptureWallTime.getValue(SECONDS));
|
||||
reprintLine(snapshotSummary);
|
||||
}
|
||||
|
||||
// Display restored snapshot stats if we have any captured snapshots already
|
||||
if (restoreCount > 0) {
|
||||
Duration allRestoreCPUTime = millis(snapshotStats.getTotalRestoreCpuTime());
|
||||
Duration allRestoreWallTime = millis(snapshotStats.getTotalRestoreWallTime());
|
||||
String allRestoreSize = FormatUtils.formatDataSize(bytes(snapshotStats.getTotalRestoreSize()), true);
|
||||
snapshotSummary = String.format(Locale.ROOT, "Snapshots Restored: %d/%s/%.1fs/%.1fs",
|
||||
restoreCount, allRestoreSize, allRestoreCPUTime.getValue(SECONDS), allRestoreWallTime.getValue(SECONDS));
|
||||
reprintLine(snapshotSummary);
|
||||
}
|
||||
|
||||
List<Long> inProgressCaptureList = snapshotStats.getCapturingSnapshotIds();
|
||||
long inProgressRestoreSnapshot = snapshotStats.getRestoringSnapshotId();
|
||||
// Snapshots in progress: Capture: 4,5, Restore: 3
|
||||
if (!isFinal && ((inProgressCaptureList != null && inProgressCaptureList.size() > 0) || inProgressRestoreSnapshot != 0)) {
|
||||
String restoreId = inProgressRestoreSnapshot == 0 ? "None" : String.valueOf(inProgressRestoreSnapshot);
|
||||
int captureCount = (inProgressCaptureList != null) ? inProgressCaptureList.size() : 0;
|
||||
String inProgressMsg = String.format(Locale.ROOT, "Snapshots in progress: Capture count: %d Restoring id: %s", captureCount, restoreId);
|
||||
reprintLine(inProgressMsg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void printQueryInfo(QueryStatusInfo results, WarningsPrinter warningsPrinter)
|
||||
{
|
||||
StatementStats stats = results.getStats();
|
||||
|
|
@ -369,7 +347,6 @@ Spilled: 20GB
|
|||
readTime.getValue(SECONDS));
|
||||
reprintLine(summary);
|
||||
}
|
||||
printSnapshotStats(stats, false);
|
||||
}
|
||||
|
||||
verify(terminalWidth >= 75); // otherwise handled above
|
||||
|
|
@ -410,37 +387,21 @@ Spilled: 20GB
|
|||
|
||||
// blank line
|
||||
reprintLine("");
|
||||
String stagesHeader = null;
|
||||
if (!debug || stats.getSnapshotStats() == null) {
|
||||
// STAGE S ROWS RPS BYTES BPS QUEUED RUN DONE
|
||||
stagesHeader = format("%10s%1s %5s %6s %5s %7s %6s %5s %5s",
|
||||
"STAGE",
|
||||
"S",
|
||||
"ROWS",
|
||||
"ROWS/s",
|
||||
"BYTES",
|
||||
"BYTES/s",
|
||||
"QUEUED",
|
||||
"RUN",
|
||||
"DONE");
|
||||
}
|
||||
else {
|
||||
// STAGE SS ROWS RPS BYTES BPS QUEUED RUN DONE SNAPSHOT
|
||||
stagesHeader = format("%10s%2s %5s %6s %5s %7s %6s %5s %5s %8s",
|
||||
"STAGE",
|
||||
"SS",
|
||||
"ROWS",
|
||||
"ROWS/s",
|
||||
"BYTES",
|
||||
"BYTES/s",
|
||||
"QUEUED",
|
||||
"RUN",
|
||||
"DONE",
|
||||
"SNAPSHOT");
|
||||
}
|
||||
|
||||
// STAGE S ROWS RPS BYTES BPS QUEUED RUN DONE
|
||||
String stagesHeader = format("%10s%1s %5s %6s %5s %7s %6s %5s %5s",
|
||||
"STAGE",
|
||||
"S",
|
||||
"ROWS",
|
||||
"ROWS/s",
|
||||
"BYTES",
|
||||
"BYTES/s",
|
||||
"QUEUED",
|
||||
"RUN",
|
||||
"DONE");
|
||||
reprintLine(stagesHeader);
|
||||
|
||||
printStageTree(stats.getRootStage(), "", new AtomicInteger(), stats.getSnapshotStats());
|
||||
printStageTree(stats.getRootStage(), "", new AtomicInteger());
|
||||
}
|
||||
else {
|
||||
// Query 31 [S] i[2.7M 67.3MB 62.7MBps] o[35 6.1KB 1KBps] splits[252/16/380]
|
||||
|
|
@ -464,10 +425,17 @@ Spilled: 20GB
|
|||
warningsPrinter.print(results.getWarnings(), false, false);
|
||||
}
|
||||
|
||||
private void printStageTree(StageStats stage, String indent, AtomicInteger stageNumberCounter, SnapshotStats snapshotStats)
|
||||
private void printStageTree(StageStats stage, String indent, AtomicInteger stageNumberCounter)
|
||||
{
|
||||
Duration elapsedTime = nanosSince(start);
|
||||
|
||||
// STAGE S ROWS ROWS/s BYTES BYTES/s QUEUED RUN DONE
|
||||
// 0......Q 26M 9077M 9993G 9077M 9077M 9077M 9077M
|
||||
// 2....R 17K 627M 673M 627M 627M 627M 627M
|
||||
// 3..C 999 627M 673M 627M 627M 627M 627M
|
||||
// 4....R 26M 627M 673T 627M 627M 627M 627M
|
||||
// 5..F 29T 627M 673M 627M 627M 627M 627M
|
||||
|
||||
String id = String.valueOf(stageNumberCounter.getAndIncrement());
|
||||
String name = indent + id;
|
||||
name += Strings.repeat(".", max(0, 10 - name.length()));
|
||||
|
|
@ -483,55 +451,23 @@ Spilled: 20GB
|
|||
rowsPerSecond = FormatUtils.formatCountRate(stage.getProcessedRows(), elapsedTime, false);
|
||||
}
|
||||
|
||||
String stageSummary = null;
|
||||
if (!debug || snapshotStats == null) {
|
||||
// STAGE S ROWS ROWS/s BYTES BYTES/s QUEUED RUN DONE
|
||||
// 0......Q 26M 9077M 9993G 9077M 9077M 9077M 9077M
|
||||
// 2....R 17K 627M 673M 627M 627M 627M 627M
|
||||
// 3..C 999 627M 673M 627M 627M 627M 627M
|
||||
// 4....R 26M 627M 673T 627M 627M 627M 627M
|
||||
// 5..F 29T 627M 673M 627M 627M 627M 627M
|
||||
stageSummary = String.format("%10s%1s %5s %6s %5s %7s %6s %5s %5s",
|
||||
name,
|
||||
stageStateCharacter(stage.getState()),
|
||||
String stageSummary = String.format("%10s%1s %5s %6s %5s %7s %6s %5s %5s",
|
||||
name,
|
||||
stageStateCharacter(stage.getState()),
|
||||
|
||||
FormatUtils.formatCount(stage.getProcessedRows()),
|
||||
rowsPerSecond,
|
||||
FormatUtils.formatCount(stage.getProcessedRows()),
|
||||
rowsPerSecond,
|
||||
|
||||
FormatUtils.formatDataSize(bytes(stage.getProcessedBytes()), false),
|
||||
bytesPerSecond,
|
||||
FormatUtils.formatDataSize(bytes(stage.getProcessedBytes()), false),
|
||||
bytesPerSecond,
|
||||
|
||||
stage.getQueuedSplits(),
|
||||
stage.getRunningSplits(),
|
||||
stage.getCompletedSplits());
|
||||
}
|
||||
else {
|
||||
// STAGE SS ROWS ROWS/s BYTES BYTES/s QUEUED RUN DONE SNAPSHOT
|
||||
// 0.......Q 26M 9077M 9993G 9077M 9077M 9077M 9077M 1
|
||||
// 2.....R 17K 627M 673M 627M 627M 627M 627M 4
|
||||
// 3...C 999 627M 673M 627M 627M 627M 627M 3
|
||||
// 4....#R 26M 627M 673T 627M 627M 627M 627M 12
|
||||
// 5..#F 29T 627M 673M 627M 627M 627M 627M 11
|
||||
stageSummary = String.format("%10s%1s%1s %5s %6s %5s %7s %6s %5s %5s %8d",
|
||||
name,
|
||||
stage.isRestoring() ? "#" : ".",
|
||||
stageStateCharacter(stage.getState()),
|
||||
|
||||
FormatUtils.formatCount(stage.getProcessedRows()),
|
||||
rowsPerSecond,
|
||||
|
||||
FormatUtils.formatDataSize(bytes(stage.getProcessedBytes()), false),
|
||||
bytesPerSecond,
|
||||
|
||||
stage.getQueuedSplits(),
|
||||
stage.getRunningSplits(),
|
||||
stage.getCompletedSplits(),
|
||||
stage.getSnapshotId());
|
||||
}
|
||||
stage.getQueuedSplits(),
|
||||
stage.getRunningSplits(),
|
||||
stage.getCompletedSplits());
|
||||
reprintLine(stageSummary);
|
||||
|
||||
for (StageStats subStage : stage.getSubStages()) {
|
||||
printStageTree(subStage, indent + " ", stageNumberCounter, snapshotStats);
|
||||
printStageTree(subStage, indent + " ", stageNumberCounter);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-client</artifactId>
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
|||
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.google.common.base.MoreObjects.toStringHelper;
|
||||
|
||||
@Immutable
|
||||
|
|
@ -40,10 +38,8 @@ public class SnapshotStats
|
|||
private final long allCaptureSize;
|
||||
// Size (bytes) of last successful snapshot
|
||||
private final long lastCaptureSize;
|
||||
// List of snapshots from which restore happened
|
||||
private final List<Long> restoredSnapshotList;
|
||||
// List of successfully captured snapshots
|
||||
private final List<Long> capturedSnapshotList;
|
||||
// Snapshot id of last restore
|
||||
private final long lastRestoreSnapshotId;
|
||||
// Number of successful restores in current query
|
||||
private final long successRestoreCount;
|
||||
// Total Wall time (ms) for all restores happened during query
|
||||
|
|
@ -52,10 +48,6 @@ public class SnapshotStats
|
|||
private final long totalRestoreSize;
|
||||
// Total Cpu time (ms) for loading state during restore
|
||||
private final long totalRestoreCpuTime;
|
||||
// Snapshot Id from which restore in progress
|
||||
private final long restoringSnapshotId;
|
||||
// Snapshot Ids from which capture in progress (Multiple captures possible at a time)
|
||||
private final List<Long> capturingSnapshotIds;
|
||||
|
||||
@JsonCreator
|
||||
public SnapshotStats(
|
||||
|
|
@ -66,14 +58,11 @@ public class SnapshotStats
|
|||
@JsonProperty("lastCaptureWallTime") long lastCaptureWallTime,
|
||||
@JsonProperty("allCaptureSize") long allCaptureSize,
|
||||
@JsonProperty("lastCaptureSize") long lastCaptureSize,
|
||||
@JsonProperty("restoredSnapshotList") List<Long> restoredSnapshotList,
|
||||
@JsonProperty("capturedSnapshotList") List<Long> capturedSnapshotList,
|
||||
@JsonProperty("lastRestoreSnapshotId") long lastRestoreSnapshotId,
|
||||
@JsonProperty("successRestoreCount") long successRestoreCount,
|
||||
@JsonProperty("totalRestoreWallTime") long totalRestoreWallTime,
|
||||
@JsonProperty("totalRestoreSize") long totalRestoreSize,
|
||||
@JsonProperty("totalRestoreCpuTime") long totalRestoreCpuTime,
|
||||
@JsonProperty("restoringSnapshotId") long restoringSnapshotId,
|
||||
@JsonProperty("capturingSnapshotIds") List<Long> capturingSnapshotIds)
|
||||
@JsonProperty("totalRestoreCpuTime") long totalRestoreCpuTime)
|
||||
{
|
||||
this.lastCaptureSnapshotId = lastCaptureSnapshotId;
|
||||
this.totalCaptureCpuTime = totalCaptureCpuTime;
|
||||
|
|
@ -82,14 +71,11 @@ public class SnapshotStats
|
|||
this.lastCaptureWallTime = lastCaptureWallTime;
|
||||
this.allCaptureSize = allCaptureSize;
|
||||
this.lastCaptureSize = lastCaptureSize;
|
||||
this.restoredSnapshotList = restoredSnapshotList;
|
||||
this.capturedSnapshotList = capturedSnapshotList;
|
||||
this.lastRestoreSnapshotId = lastRestoreSnapshotId;
|
||||
this.successRestoreCount = successRestoreCount;
|
||||
this.totalRestoreWallTime = totalRestoreWallTime;
|
||||
this.totalRestoreSize = totalRestoreSize;
|
||||
this.totalRestoreCpuTime = totalRestoreCpuTime;
|
||||
this.restoringSnapshotId = restoringSnapshotId;
|
||||
this.capturingSnapshotIds = capturingSnapshotIds;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -135,27 +121,9 @@ public class SnapshotStats
|
|||
}
|
||||
|
||||
@JsonProperty
|
||||
public List<Long> getRestoredSnapshotList()
|
||||
public long getLastRestoreSnapshotId()
|
||||
{
|
||||
return restoredSnapshotList;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public List<Long> getCapturedSnapshotList()
|
||||
{
|
||||
return capturedSnapshotList;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public List<Long> getCapturingSnapshotIds()
|
||||
{
|
||||
return capturingSnapshotIds;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getRestoringSnapshotId()
|
||||
{
|
||||
return restoringSnapshotId;
|
||||
return lastRestoreSnapshotId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -193,14 +161,11 @@ public class SnapshotStats
|
|||
.add("lastCaptureWallTime", lastCaptureWallTime)
|
||||
.add("allCaptureSize", allCaptureSize)
|
||||
.add("lastCaptureSize", lastCaptureSize)
|
||||
.add("restoredSnapshotList", restoredSnapshotList)
|
||||
.add("capturedSnapshotList", capturedSnapshotList)
|
||||
.add("lastRestoreSnapshotId", lastRestoreSnapshotId)
|
||||
.add("successRestoreCount", successRestoreCount)
|
||||
.add("totalRestoreWallTime", totalRestoreWallTime)
|
||||
.add("totalRestoreSize", totalRestoreSize)
|
||||
.add("totalRestoreCpuTime", totalRestoreCpuTime)
|
||||
.add("restoringSnapshotId", restoringSnapshotId)
|
||||
.add("capturingSnapshotIds", capturingSnapshotIds)
|
||||
.toString();
|
||||
}
|
||||
|
||||
|
|
@ -218,14 +183,11 @@ public class SnapshotStats
|
|||
private long lastCaptureWallTime;
|
||||
private long allCaptureSize;
|
||||
private long lastCaptureSize;
|
||||
private List<Long> restoredSnapshotList;
|
||||
private List<Long> capturedSnapshotList;
|
||||
private long lastRestoreSnapshotId;
|
||||
private long successRestoreCount;
|
||||
private long totalRestoreWallTime;
|
||||
private long totalRestoreSize;
|
||||
private long totalRestoreCpuTime;
|
||||
private long restoringSnapshotId;
|
||||
private List<Long> capturingSnapshotIds;
|
||||
|
||||
private Builder() {}
|
||||
|
||||
|
|
@ -271,21 +233,9 @@ public class SnapshotStats
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder setRestoringSnapshotId(long restoringSnapshotId)
|
||||
public Builder setLastRestoreSnapshotId(long lastRestoreSnapshotId)
|
||||
{
|
||||
this.restoringSnapshotId = restoringSnapshotId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setRestoredSnapshotList(List<Long> restoredSnapshotList)
|
||||
{
|
||||
this.restoredSnapshotList = restoredSnapshotList;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setCapturedSnapshotList(List<Long> capturedSnapshotList)
|
||||
{
|
||||
this.capturedSnapshotList = capturedSnapshotList;
|
||||
this.lastRestoreSnapshotId = lastRestoreSnapshotId;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -313,12 +263,6 @@ public class SnapshotStats
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder setCapturingSnapshotIds(List<Long> capturingSnapshotIds)
|
||||
{
|
||||
this.capturingSnapshotIds = capturingSnapshotIds;
|
||||
return this;
|
||||
}
|
||||
|
||||
public SnapshotStats build()
|
||||
{
|
||||
return new SnapshotStats(
|
||||
|
|
@ -329,14 +273,11 @@ public class SnapshotStats
|
|||
lastCaptureWallTime,
|
||||
allCaptureSize,
|
||||
lastCaptureSize,
|
||||
restoredSnapshotList,
|
||||
capturedSnapshotList,
|
||||
lastRestoreSnapshotId,
|
||||
successRestoreCount,
|
||||
totalRestoreWallTime,
|
||||
totalRestoreSize,
|
||||
totalRestoreCpuTime,
|
||||
restoringSnapshotId,
|
||||
capturingSnapshotIds);
|
||||
totalRestoreCpuTime);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,6 @@ public class StageStats
|
|||
{
|
||||
private final String stageId;
|
||||
private final String state;
|
||||
private final boolean restoring;
|
||||
private final long snapshotId;
|
||||
private final boolean done;
|
||||
private final int nodes;
|
||||
private final int totalSplits;
|
||||
|
|
@ -47,8 +45,6 @@ public class StageStats
|
|||
public StageStats(
|
||||
@JsonProperty("stageId") String stageId,
|
||||
@JsonProperty("state") String state,
|
||||
@JsonProperty("restoring") boolean restoring,
|
||||
@JsonProperty("snapshotId") long snapshotId,
|
||||
@JsonProperty("done") boolean done,
|
||||
@JsonProperty("nodes") int nodes,
|
||||
@JsonProperty("totalSplits") int totalSplits,
|
||||
|
|
@ -63,8 +59,6 @@ public class StageStats
|
|||
{
|
||||
this.stageId = stageId;
|
||||
this.state = requireNonNull(state, "state is null");
|
||||
this.restoring = restoring;
|
||||
this.snapshotId = snapshotId;
|
||||
this.done = done;
|
||||
this.nodes = nodes;
|
||||
this.totalSplits = totalSplits;
|
||||
|
|
@ -90,18 +84,6 @@ public class StageStats
|
|||
return state;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isRestoring()
|
||||
{
|
||||
return restoring;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getSnapshotId()
|
||||
{
|
||||
return snapshotId;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isDone()
|
||||
{
|
||||
|
|
@ -173,8 +155,6 @@ public class StageStats
|
|||
{
|
||||
return toStringHelper(this)
|
||||
.add("state", state)
|
||||
.add("restoring", restoring)
|
||||
.add("snapshotId", snapshotId)
|
||||
.add("done", done)
|
||||
.add("nodes", nodes)
|
||||
.add("totalSplits", totalSplits)
|
||||
|
|
@ -198,8 +178,6 @@ public class StageStats
|
|||
{
|
||||
private String stageId;
|
||||
private String state;
|
||||
private boolean restoring;
|
||||
private long snapshotId;
|
||||
private boolean done;
|
||||
private int nodes;
|
||||
private int totalSplits;
|
||||
|
|
@ -226,18 +204,6 @@ public class StageStats
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder setRestoring(boolean restoring)
|
||||
{
|
||||
this.restoring = restoring;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setSnapshotId(long snapshotId)
|
||||
{
|
||||
this.snapshotId = snapshotId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setDone(boolean done)
|
||||
{
|
||||
this.done = done;
|
||||
|
|
@ -309,8 +275,6 @@ public class StageStats
|
|||
return new StageStats(
|
||||
stageId,
|
||||
state,
|
||||
restoring,
|
||||
snapshotId,
|
||||
done,
|
||||
nodes,
|
||||
totalSplits,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-elasticsearch</artifactId>
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
|
||||
<properties>
|
||||
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
|
||||
<dep.log4j.version>2.17.2</dep.log4j.version>
|
||||
<dep.log4j.version>2.17.1</dep.log4j.version>
|
||||
<dep.elasticsearch.version>6.0.0</dep.elasticsearch.version>
|
||||
<httpcore-nio.version>4.4.5</httpcore-nio.version>
|
||||
</properties>
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-example-http</artifactId>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<parent>
|
||||
<groupId>io.hetu.core</groupId>
|
||||
<artifactId>presto-root</artifactId>
|
||||
<version>1.7.0-SNAPSHOT</version>
|
||||
<version>1.6.2-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>presto-expressions</artifactId>
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue