Fix for star tree cube incorrect results
1. Block INSERT INTO CUBE if source table has been updated. 2. Block duplicate data insert into the cube 3. Insert entire data from source table into cube 4. Updated hetu docs
This commit is contained in:
parent
2513e706e8
commit
8ec9bdffdf
|
|
@ -26,22 +26,55 @@ import java.util.stream.Collectors;
|
|||
public interface CubeMetadata
|
||||
extends Serializable
|
||||
{
|
||||
String getCubeTableName();
|
||||
/**
|
||||
* Returns name of the cube
|
||||
*/
|
||||
String getCubeName();
|
||||
|
||||
String getOriginalTableName();
|
||||
/**
|
||||
* Returns the name of the source table
|
||||
*/
|
||||
String getSourceTableName();
|
||||
|
||||
long getLastUpdated();
|
||||
/**
|
||||
* Returns the last updated time of the cube
|
||||
*/
|
||||
long getLastUpdatedTime();
|
||||
|
||||
/**
|
||||
* Returns the last updated time of the source table
|
||||
*/
|
||||
long getSourceTableLastUpdatedTime();
|
||||
|
||||
/**
|
||||
* Return the names of the dimension columns
|
||||
*/
|
||||
List<String> getDimensions();
|
||||
|
||||
/**
|
||||
* Return the names of the aggregation columns
|
||||
*/
|
||||
List<String> getAggregations();
|
||||
|
||||
List<String> getAggregationsAsString();
|
||||
|
||||
/**
|
||||
* Return the group by columns
|
||||
*/
|
||||
Set<String> getGroup();
|
||||
|
||||
/**
|
||||
* Checks if metadata matches the CubeStatement
|
||||
* @param statement cube statement
|
||||
* @return true - if metadata matches CubeStatement
|
||||
* false - otherwise
|
||||
*/
|
||||
boolean matches(CubeStatement statement);
|
||||
|
||||
/**
|
||||
* Filters all metadata that matches the cube statement
|
||||
* @param metadataList metadata list
|
||||
* @param statement cube statement
|
||||
* @return all metadata that is matching the cube statement
|
||||
*/
|
||||
static List<CubeMetadata> filter(List<CubeMetadata> metadataList, CubeStatement statement)
|
||||
{
|
||||
return metadataList.stream()
|
||||
|
|
@ -49,19 +82,36 @@ public interface CubeMetadata
|
|||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves the cube column matching the given aggregation signature
|
||||
* @return name of the aggregation column if found
|
||||
*/
|
||||
Optional<String> getColumn(AggregationSignature aggSignature);
|
||||
|
||||
Optional<String> getAggregationFunction(String starTableColumn);
|
||||
|
||||
Optional<String> getAggregationColumn(String aggFunction, String originalColumn, boolean distinct);
|
||||
/**
|
||||
* Return the aggregation function associated with given cube column
|
||||
* @return name of the aggregation function
|
||||
*/
|
||||
Optional<String> getAggregationFunction(String column);
|
||||
|
||||
/**
|
||||
* Get the aggregation information of the given cube column
|
||||
* @param column name of the cube column
|
||||
*/
|
||||
Optional<AggregationSignature> getAggregationSignature(String column);
|
||||
|
||||
/**
|
||||
* Return all aggregation column information
|
||||
*/
|
||||
List<AggregationSignature> getAggregationSignatures();
|
||||
|
||||
/**
|
||||
* Return cube predicate string
|
||||
*/
|
||||
String getPredicateString();
|
||||
|
||||
String getGroupString();
|
||||
|
||||
/**
|
||||
* Return the status of the cube
|
||||
*/
|
||||
CubeStatus getCubeStatus();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,9 @@ public interface CubeMetadataBuilder
|
|||
|
||||
void setCubeStatus(CubeStatus cubeStatus);
|
||||
|
||||
CubeMetadata build();
|
||||
void setTableLastUpdatedTime(long tableLastUpdatedTime);
|
||||
|
||||
CubeMetadata build(long createdTime);
|
||||
void setCubeLastUpdatedTime(long cubeLastUpdatedTime);
|
||||
|
||||
CubeMetadata build();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,16 +21,16 @@ import com.fasterxml.jackson.annotation.JsonProperty;
|
|||
import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.AVG;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.COUNT;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.MAX;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.MIN;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.SUM;
|
||||
|
||||
public class AggregationSignature
|
||||
implements Serializable, Comparable<AggregationSignature>
|
||||
{
|
||||
public static final String AVG_FUNCTION_NAME = "avg";
|
||||
public static final String COUNT_FUNCTION_NAME = "count";
|
||||
public static final String SUM_FUNCTION_NAME = "sum";
|
||||
public static final String MIN_FUNCTION_NAME = "min";
|
||||
public static final String MAX_FUNCTION_NAME = "max";
|
||||
|
||||
private static final AggregationSignature COUNT_SIGNATURE = new AggregationSignature(COUNT_FUNCTION_NAME, "*", false);
|
||||
private static final AggregationSignature COUNT_SIGNATURE = new AggregationSignature(COUNT.getName(), "*", false);
|
||||
|
||||
private String function;
|
||||
private String dimension;
|
||||
|
|
@ -54,27 +54,27 @@ public class AggregationSignature
|
|||
|
||||
public static AggregationSignature count(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(COUNT_FUNCTION_NAME, dimension, distinct);
|
||||
return new AggregationSignature(COUNT.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature sum(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(SUM_FUNCTION_NAME, dimension, distinct);
|
||||
return new AggregationSignature(SUM.toString(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature avg(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(AVG_FUNCTION_NAME, dimension, distinct);
|
||||
return new AggregationSignature(AVG.toString(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature min(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(MIN_FUNCTION_NAME, dimension, distinct);
|
||||
return new AggregationSignature(MIN.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
public static AggregationSignature max(String dimension, boolean distinct)
|
||||
{
|
||||
return new AggregationSignature(MAX_FUNCTION_NAME, dimension, distinct);
|
||||
return new AggregationSignature(MAX.getName(), dimension, distinct);
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
|
|||
|
|
@ -36,10 +36,10 @@ public interface CubeMetaStore
|
|||
/**
|
||||
* Create a new Metadata builder
|
||||
* @param cubeName Name of the cube
|
||||
* @param originalTableName Name of the original table
|
||||
* @param sourceTableName Name of the table from which cube was created
|
||||
* @return a metadata builder
|
||||
*/
|
||||
CubeMetadataBuilder getBuilder(String cubeName, String originalTableName);
|
||||
CubeMetadataBuilder getBuilder(String cubeName, String sourceTableName);
|
||||
|
||||
/**
|
||||
* Create new metadata builder from the existing metadata
|
||||
|
|
|
|||
|
|
@ -128,6 +128,7 @@ headless: true
|
|||
- [CALL]({{< relref "./docs/sql/call.md" >}})
|
||||
- [COMMENT]({{< relref "./docs/sql/comment.md" >}})
|
||||
- [COMMIT]({{< relref "./docs/sql/commit.md" >}})
|
||||
- [CREATE CUBE]({{< relref "./docs/sql/create-cube.md" >}})
|
||||
- [CREATE ROLE]({{< relref "./docs/sql/create-role.md" >}})
|
||||
- [CREATE SCHEMA]({{< relref "./docs/sql/create-schema.md" >}})
|
||||
- [CREATE TABLE]({{< relref "./docs/sql/create-table.md" >}})
|
||||
|
|
@ -139,6 +140,7 @@ headless: true
|
|||
- [DESCRIBE INPUT]({{< relref "./docs/sql/describe-input.md" >}})
|
||||
- [DESCRIBE OUTPUT]({{< relref "./docs/sql/describe-output.md" >}})
|
||||
- [DROP CACHE]({{< relref "./docs/sql/drop-cache.md" >}})
|
||||
- [DROP CUBE]({{< relref "./docs/sql/drop-cube.md" >}})
|
||||
- [DROP ROLE]({{< relref "./docs/sql/drop-role.md" >}})
|
||||
- [DROP SCHEMA]({{< relref "./docs/sql/drop-schema.md" >}})
|
||||
- [DROP TABLE]({{< relref "./docs/sql/drop-table.md" >}})
|
||||
|
|
@ -150,6 +152,8 @@ headless: true
|
|||
- [GRANT ROLES]({{< relref "./docs/sql/grant-roles.md" >}})
|
||||
- [INSERT]({{< relref "./docs/sql/insert.md" >}})
|
||||
- [INSERT OVERWRITE]({{< relref "./docs/sql/insert-overwrite.md" >}})
|
||||
- [INSERT CUBE]({{< relref "./docs/sql/insert-cube.md" >}})
|
||||
- [INSERT OVERWRITE CUBE]({{< relref "./docs/sql/insert-overwrite-cube.md" >}})
|
||||
- [JMX]({{< relref "./docs/sql/jmx.md" >}})
|
||||
- [PREPARE]({{< relref "./docs/sql/prepare.md" >}})
|
||||
- [RESET SESSION]({{< relref "./docs/sql/reset-session.md" >}})
|
||||
|
|
@ -164,6 +168,7 @@ headless: true
|
|||
- [SHOW COLUMNS]({{< relref "./docs/sql/show-columns.md" >}})
|
||||
- [SHOW CREATE TABLE]({{< relref "./docs/sql/show-create-table.md" >}})
|
||||
- [SHOW CREATE VIEW]({{< relref "./docs/sql/show-create-view.md" >}})
|
||||
- [SHOW CUBES]({{< relref "./docs/sql/show-cubes.md" >}})
|
||||
- [SHOW FUNCTIONS]({{< relref "./docs/sql/show-functions.md" >}})
|
||||
- [SHOW EXTERNAL FUNCTION]({{< relref "./docs/sql/show-external-function.md" >}})
|
||||
- [SHOW GRANTS]({{< relref "./docs/sql/show-grants.md" >}})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
CREATE CUBE
|
||||
============
|
||||
|
||||
Synopsis
|
||||
--------
|
||||
|
||||
``` sql
|
||||
CREATE CUBE [ IF NOT EXISTS ]
|
||||
cube_name ON table_name WITH (
|
||||
AGGREGATIONS = ( expression [, ...] ), GROUP = ( column_name [, ...] )
|
||||
[, ( property_name = expression [, ...] ) ]
|
||||
)
|
||||
```
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
Create a new, empty star-tree cube with the specified group and aggregations. Use `insert-into-cube` to insert data.
|
||||
|
||||
The optional `IF NOT EXISTS` clause causes the error to be suppressed if the table already exists.
|
||||
|
||||
The optional `property_name` section can be used to set properties on the newly created cube. To list all available table properties, run the following query:
|
||||
|
||||
SELECT * FROM system.metadata.table_properties
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Create a new cube `orders_cube` on `orders`:
|
||||
|
||||
CREATE CUBE orders_cube ON orders WITH (
|
||||
AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
|
||||
GROUP = ( orderstatus, orderdate ),
|
||||
format = 'ORC'
|
||||
)
|
||||
|
||||
Create a new partitioned cube `orders_cube`:
|
||||
|
||||
CREATE CUBE orders_cube ON orders WITH (
|
||||
AGGREGATIONS = ( SUM(totalprice), AVG(totalprice) ),
|
||||
GROUP = ( orderstatus, orderdate ),
|
||||
format = 'ORC',
|
||||
partitioned_by = ARRAY['orderdate']
|
||||
)
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
|
||||
- Supported aggregate functions:
|
||||
COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
|
||||
- Only one group is supported per Cube.
|
||||
- Different connector might support different data type, and different table/column properties.
|
||||
- Can currently only create cubes in Hive connector, but the cubes can be created on a table from another connector.
|
||||
|
||||
See Also
|
||||
--------
|
||||
[INSERT INTO CUBE](./insert-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
|
||||
DROP CUBE
|
||||
==========
|
||||
|
||||
Synopsis
|
||||
--------
|
||||
|
||||
``` sql
|
||||
DROP CUBE [ IF EXISTS ] cube_name
|
||||
```
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
Drop an existing cube.
|
||||
|
||||
The optional `IF EXISTS` clause causes the error to be suppressed if the cube does not exist.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Drop the cube `orders_cube`:
|
||||
|
||||
DROP CUBE orders_cube
|
||||
|
||||
Drop the cube `orders_cube` if it exists:
|
||||
|
||||
DROP CUBE IF EXISTS orders_cube
|
||||
|
||||
See Also
|
||||
--------
|
||||
|
||||
[CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [INSERT INTO CUBE](./insert-cube.md)
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
INSERT INTO CUBE
|
||||
======
|
||||
|
||||
Synopsis
|
||||
--------
|
||||
|
||||
``` sql
|
||||
INSERT INTO CUBE cube_name [WHERE condition]
|
||||
```
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
Insert data into a star-tree cube. Predicate information is optional. If predicate provided, only data matching
|
||||
the given predicate are processed from the source table and inserted into the cube. Otherwise, entire
|
||||
data from the source table is processed and inserted into Cube.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Insert data based on condition into the `orders_cube` cube:
|
||||
|
||||
INSERT INTO CUBE orders_cube WHERE orderdate > date '1999-01-01';
|
||||
INSERT INTO CUBE order_all_cube;
|
||||
|
||||
See Also
|
||||
--------
|
||||
|
||||
[INSERT OVERWRITE CUBE](./insert-overwrite-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
INSERT INTO CUBE
|
||||
======
|
||||
|
||||
Synopsis
|
||||
--------
|
||||
|
||||
``` sql
|
||||
INSERT OVERWRITE CUBE cube_name [WHERE condition]
|
||||
```
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
Similar to INSERT INTO CUBE statement but with this statement the existing data is overwritten. Predicates
|
||||
are optional.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Insert data based on condition into the `orders_cube` cube:
|
||||
|
||||
INSERT OVERWRITE CUBE orders_cube WHERE orderdate > date '1999-01-01';
|
||||
INSERT OVERWRITE CUBE orders_cube;
|
||||
|
||||
See Also
|
||||
--------
|
||||
|
||||
[INSERT INTO CUBE](./insert-cube.md), [CREATE CUBE](./create-cube.md), [SHOW CUBES](./show-cubes.md), [DROP CUBE](./drop-cube.md)
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
# Star-Tree
|
||||
|
||||
Star tree cubing is a pre-aggregation technique to achieve low latency runtime for iceberg queries. Star tree cubing aimed to reduce
|
||||
latency for iceberg queries. An iceberg query computes an aggregate function over an attribute ( or set of attributes) in order to
|
||||
find aggregate values above a specified threshold.
|
||||
|
||||
## Supported functions
|
||||
COUNT, COUNT DISTINCT, MIN, MAX, SUM, AVG
|
||||
|
||||
## Enabling and Disabling Star-tree
|
||||
To enable:
|
||||
```sql
|
||||
SET SESSION enable_star_tree_index=true;
|
||||
```
|
||||
To disable:
|
||||
```sql
|
||||
SET SESSION enable_star_tree_index=false;
|
||||
```
|
||||
|
||||
## Configuration Properties
|
||||
| Property Name | Default Value | Required| Description|
|
||||
|---------------------------------------------------|---------------------|---------|--------------|
|
||||
| optimizer.enable-star-tree-index | false | No | Enables star-tree index|
|
||||
| cube.metadata-cache-size | 5 | No | The maximum number of metadata for star-trees that could be loaded into cache before eviction happens|
|
||||
| cube.metadata-cache-ttl | 1h | No | The maximum time to live of star-trees that are be loaded into cache before eviction happens |
|
||||
|
||||
## Examples
|
||||
|
||||
Creating a star-tree cube:
|
||||
```sql
|
||||
CREATE CUBE nation_cube
|
||||
ON nation
|
||||
WITH (AGGREGATIONS=(count(*), count(distinct regionkey), avg(nationkey), max(regionkey)),
|
||||
GROUP=(nationkey),
|
||||
format='orc', partitioned_by=ARRAY['nationkey']);
|
||||
```
|
||||
Next, to add data to the cube:
|
||||
```sql
|
||||
INSERT INTO CUBE nation_cube WHERE nationkey > 5;
|
||||
```
|
||||
To use the new cube, just query the original table using aggregations that were included in the cube:
|
||||
```sql
|
||||
SELECT count(*) FROM nation WHERE nationkey > 5 GROUP BY nationkey;
|
||||
SELECT nationkey, avg(nationkey), max(regionkey) WHERE nationkey > 5 GROUP BY nationkey;
|
||||
```
|
||||
|
||||
## Optimizer Changes
|
||||
|
||||
The star tree aggregation rule is an Iterative optimizer that optimizes the logical plan by replacing the original aggregation sub-tree
|
||||
and original table scan with pre-aggregation table scan.
|
||||
|
||||
|
||||
## Dependencies
|
||||
|
||||
Star Tree index relies on Hetu metastore to store the cube related metadata.
|
||||
Please check [Hetu Metastore](../admin/meta-store.md) for more information.
|
||||
|
||||
## Limitation
|
||||
|
||||
1. Star tree cube is only effective when the group by cardinality is considerably lower than the number of rows in
|
||||
source table.
|
||||
2. A significant amount of user effort required in maintaining Cubes for large datasets.
|
||||
3. Only incremental insert into cube is supported. Cannot delete specific rows from Cube.
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
|
||||
SHOW CUBES
|
||||
==========
|
||||
|
||||
Synopsis
|
||||
--------
|
||||
|
||||
``` sql
|
||||
SHOW CUBES [ FOR table_name ];
|
||||
```
|
||||
|
||||
Description
|
||||
-----------
|
||||
|
||||
`SHOW CUBES` lists all cubes. Adding the optional `table_name` lists only the cubes for that table.
|
||||
|
||||
Examples
|
||||
--------
|
||||
|
||||
Show all cubes:
|
||||
|
||||
```sql
|
||||
SHOW CUBES;
|
||||
```
|
||||
|
||||
Show cubes for `orders` table:
|
||||
|
||||
```sql
|
||||
SHOW CUBES FOR orders;
|
||||
```
|
||||
|
||||
See Also
|
||||
--------
|
||||
|
||||
[CREATE CUBE](./create-cube.md), [DROP CUBE](./drop-cube.md), [INSERT INTO CUBE](./insert-cube.md)
|
||||
|
|
@ -666,3 +666,10 @@ Copyright 2008-2015 the Gson author.
|
|||
|
||||
License: Apache License V2.0
|
||||
Please see above.
|
||||
|
||||
Software: Caffeine
|
||||
Copyright notice:
|
||||
Copyright 2015 Ben Manes.
|
||||
|
||||
License: Apache License 2.0
|
||||
Please see above.
|
||||
|
|
@ -17,6 +17,7 @@ package io.hetu.core.cube.startree.io;
|
|||
|
||||
import com.github.benmanes.caffeine.cache.Caffeine;
|
||||
import com.github.benmanes.caffeine.cache.LoadingCache;
|
||||
import com.google.common.collect.Sets;
|
||||
import io.hetu.core.cube.startree.tree.AggregateColumn;
|
||||
import io.hetu.core.cube.startree.tree.DimensionColumn;
|
||||
import io.hetu.core.cube.startree.tree.StarTreeColumn;
|
||||
|
|
@ -34,18 +35,14 @@ import io.prestosql.spi.metastore.model.TableEntity;
|
|||
import io.prestosql.spi.metastore.model.TableEntityType;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static io.hetu.core.cube.startree.tree.StarTreeMetadata.COLUMN_DELIMITER;
|
||||
import static io.hetu.core.cube.startree.tree.StarTreeMetadata.GROUP_DELIMITER;
|
||||
import static io.hetu.core.cube.startree.util.Constants.CUBE_CATALOG;
|
||||
import static io.hetu.core.cube.startree.util.Constants.CUBE_DATABASE;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
|
@ -53,12 +50,15 @@ import static java.util.Objects.requireNonNull;
|
|||
public class StarTreeMetaStore
|
||||
implements CubeMetaStore
|
||||
{
|
||||
public static final String ORIGINAL_TABLE_NAME = "originalTableName";
|
||||
public static final String SOURCE_TABLE_NAME = "sourceTableName";
|
||||
public static final String ORIGINAL_COLUMN = "originalColumn";
|
||||
public static final String STAR_TABLE_NAME = "starTableName";
|
||||
public static final String GROUPING_STRING = "groupingString";
|
||||
public static final String PREDICATE_STRING = "predicateString";
|
||||
public static final String CUBE_STATUS = "cubeStatus";
|
||||
public static final String SOURCE_TABLE_LAST_UPDATED_TIME = "sourceLastUpdatedTime";
|
||||
public static final String CUBE_LAST_UPDATED_TIME = "cubeLastUpdatedTime";
|
||||
|
||||
private final HetuMetastore metastore;
|
||||
private final LoadingCache<String, List<CubeMetadata>> cubeCache;
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ public class StarTreeMetaStore
|
|||
tableEntities.forEach(table -> {
|
||||
List<ColumnEntity> cols = table.getColumns();
|
||||
StarTreeMetadataBuilder builder = new StarTreeMetadataBuilder(table.getParameters().get(STAR_TABLE_NAME),
|
||||
table.getParameters().get(ORIGINAL_TABLE_NAME));
|
||||
table.getParameters().get(SOURCE_TABLE_NAME));
|
||||
cols.forEach(col -> {
|
||||
if (col.getType().equals("aggregate")) {
|
||||
builder.addAggregationColumn(col.getName(), col.getParameters().get("aggregateFunction"), col.getParameters().get(ORIGINAL_COLUMN), Boolean.parseBoolean(col.getParameters().get("distinct")));
|
||||
|
|
@ -87,21 +87,12 @@ public class StarTreeMetaStore
|
|||
}
|
||||
});
|
||||
String groupingString = table.getParameters().get(GROUPING_STRING);
|
||||
if (groupingString != null) {
|
||||
for (String columns : groupingString.split(GROUP_DELIMITER)) {
|
||||
Set<String> group;
|
||||
if (columns.equals("")) {
|
||||
group = new HashSet<>();
|
||||
}
|
||||
else {
|
||||
group = new HashSet<>(Arrays.asList(columns.split(COLUMN_DELIMITER)));
|
||||
}
|
||||
builder.addGroup(group);
|
||||
}
|
||||
}
|
||||
builder.addGroup(Sets.newHashSet(groupingString.split(COLUMN_DELIMITER)));
|
||||
builder.withPredicate(table.getParameters().get(PREDICATE_STRING));
|
||||
builder.setCubeStatus(CubeStatus.forValue(Integer.parseInt(table.getParameters().get(CUBE_STATUS))));
|
||||
cubeMetadataList.add(builder.build(table.getCreateTime()));
|
||||
builder.setTableLastUpdatedTime(Long.parseLong(table.getParameters().get(SOURCE_TABLE_LAST_UPDATED_TIME)));
|
||||
builder.setCubeLastUpdatedTime(Long.parseLong(table.getParameters().get(CUBE_LAST_UPDATED_TIME)));
|
||||
cubeMetadataList.add(builder.build());
|
||||
});
|
||||
return cubeMetadataList;
|
||||
}
|
||||
|
|
@ -119,7 +110,7 @@ public class StarTreeMetaStore
|
|||
List<TableEntity> tables = metastore.getAllTables(CUBE_CATALOG, CUBE_DATABASE);
|
||||
List<TableEntity> matchingTables = new ArrayList<>();
|
||||
tables.forEach(table -> {
|
||||
if (table.getParameters().get(ORIGINAL_TABLE_NAME).equals(tableName)) {
|
||||
if (table.getParameters().get(SOURCE_TABLE_NAME).equals(tableName)) {
|
||||
matchingTables.add(table);
|
||||
}
|
||||
});
|
||||
|
|
@ -136,8 +127,8 @@ public class StarTreeMetaStore
|
|||
@Override
|
||||
public void removeCube(CubeMetadata cubeMetadata)
|
||||
{
|
||||
metastore.dropTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata.getCubeTableName().replace(".", "_"));
|
||||
cubeCache.invalidate(cubeMetadata.getOriginalTableName());
|
||||
metastore.dropTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata.getCubeName().replace(".", "_"));
|
||||
cubeCache.invalidate(cubeMetadata.getSourceTableName());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -153,7 +144,7 @@ public class StarTreeMetaStore
|
|||
metastore.createDatabase(databaseEntity);
|
||||
}
|
||||
|
||||
String cubeNameDelimited = cubeMetadata.getCubeTableName().replace(".", "_");
|
||||
String cubeNameDelimited = cubeMetadata.getCubeName().replace(".", "_");
|
||||
TableEntity table = getTableEntity((StarTreeMetadata) cubeMetadata);
|
||||
if (metastore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeNameDelimited).isPresent()) {
|
||||
//update flow
|
||||
|
|
@ -163,7 +154,7 @@ public class StarTreeMetaStore
|
|||
//create flow
|
||||
metastore.createTable(table);
|
||||
}
|
||||
cubeCache.invalidate(cubeMetadata.getOriginalTableName());
|
||||
cubeCache.invalidate(cubeMetadata.getSourceTableName());
|
||||
}
|
||||
|
||||
private CatalogEntity catalogEntity()
|
||||
|
|
@ -189,7 +180,7 @@ public class StarTreeMetaStore
|
|||
|
||||
private TableEntity getTableEntity(StarTreeMetadata starTreeMetadata)
|
||||
{
|
||||
String cubeNameDelimited = starTreeMetadata.getCubeTableName().replace(".", "_");
|
||||
String cubeNameDelimited = starTreeMetadata.getCubeName().replace(".", "_");
|
||||
List<ColumnEntity> columns = new ArrayList<>();
|
||||
starTreeMetadata.getColumns().forEach(col -> {
|
||||
ColumnEntity newCol = new ColumnEntity();
|
||||
|
|
@ -214,11 +205,14 @@ public class StarTreeMetaStore
|
|||
columns.add(newCol);
|
||||
});
|
||||
Map<String, String> parameters = new HashMap<>();
|
||||
parameters.put(ORIGINAL_TABLE_NAME, starTreeMetadata.getOriginalTableName());
|
||||
parameters.put(STAR_TABLE_NAME, starTreeMetadata.getCubeTableName());
|
||||
parameters.put(GROUPING_STRING, starTreeMetadata.getGroupString());
|
||||
parameters.put(SOURCE_TABLE_NAME, starTreeMetadata.getSourceTableName());
|
||||
parameters.put(STAR_TABLE_NAME, starTreeMetadata.getCubeName());
|
||||
parameters.put(GROUPING_STRING, String.join(COLUMN_DELIMITER, starTreeMetadata.getGroup()));
|
||||
parameters.put(PREDICATE_STRING, starTreeMetadata.getPredicateString());
|
||||
parameters.put(CUBE_STATUS, String.valueOf(starTreeMetadata.getCubeStatus().getValue()));
|
||||
parameters.put(CUBE_LAST_UPDATED_TIME, String.valueOf(starTreeMetadata.getLastUpdatedTime()));
|
||||
parameters.put(SOURCE_TABLE_LAST_UPDATED_TIME, String.valueOf(starTreeMetadata.getSourceTableLastUpdatedTime()));
|
||||
|
||||
return TableEntity.builder()
|
||||
.setCatalogName(CUBE_CATALOG)
|
||||
.setDatabaseName(CUBE_DATABASE)
|
||||
|
|
@ -226,7 +220,6 @@ public class StarTreeMetaStore
|
|||
.setTableName(cubeNameDelimited)
|
||||
.setColumns(columns)
|
||||
.setParameters(parameters)
|
||||
.setCreateTime(starTreeMetadata.getLastUpdated())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
|
@ -237,9 +230,9 @@ public class StarTreeMetaStore
|
|||
}
|
||||
|
||||
@Override
|
||||
public CubeMetadataBuilder getBuilder(String cubeName, String originalTableName)
|
||||
public CubeMetadataBuilder getBuilder(String cubeName, String sourceTableName)
|
||||
{
|
||||
return new StarTreeMetadataBuilder(cubeName, originalTableName);
|
||||
return new StarTreeMetadataBuilder(cubeName, sourceTableName);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import java.util.stream.Collectors;
|
|||
|
||||
import static io.hetu.core.cube.startree.tree.StarTreeColumn.ColumnType.AGGREGATE;
|
||||
import static io.hetu.core.cube.startree.tree.StarTreeColumn.ColumnType.DIMENSION;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.AVG;
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public class StarTreeMetadata
|
||||
|
|
@ -45,7 +46,7 @@ public class StarTreeMetadata
|
|||
{
|
||||
private final String starTreeName;
|
||||
|
||||
private final String originalTableName;
|
||||
private final String sourceTableName;
|
||||
|
||||
private final List<StarTreeColumn> columns;
|
||||
|
||||
|
|
@ -53,47 +54,50 @@ public class StarTreeMetadata
|
|||
|
||||
private final String predicateString;
|
||||
|
||||
private final long lastUpdated;
|
||||
private final long sourceTableLastUpdatedTime;
|
||||
|
||||
private CubeStatus cubeStatus;
|
||||
private final long lastUpdatedTime;
|
||||
|
||||
private final CubeStatus cubeStatus;
|
||||
|
||||
public static final String COLUMN_DELIMITER = ",";
|
||||
public static final String GROUP_DELIMITER = "\t";
|
||||
|
||||
@JsonCreator
|
||||
public StarTreeMetadata(
|
||||
@JsonProperty("starTreeName") String starTreeName,
|
||||
@JsonProperty("originalTableName") String originalTableName,
|
||||
@JsonProperty("sourceTableName") String sourceTableName,
|
||||
@JsonProperty("sourceTableLastUpdatedTime") long sourceTableLastUpdatedTime,
|
||||
@JsonProperty("columns") List<StarTreeColumn> columns,
|
||||
@JsonProperty("groups") List<Set<String>> groups,
|
||||
@JsonProperty("predicateString") String predicateString,
|
||||
@JsonProperty("lastUpdated") long lastUpdated,
|
||||
@JsonProperty("lastUpdatedTime") long lastUpdatedTime,
|
||||
@JsonProperty("cubeStatus") CubeStatus cubeStatus)
|
||||
{
|
||||
this.starTreeName = requireNonNull(starTreeName, "starTreeName is null").toLowerCase(Locale.ENGLISH);
|
||||
this.originalTableName = requireNonNull(originalTableName, "tableName is null").toLowerCase(Locale.ENGLISH);
|
||||
this.sourceTableName = requireNonNull(sourceTableName, "tableName is null").toLowerCase(Locale.ENGLISH);
|
||||
this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
|
||||
this.groups = new ArrayList<>();
|
||||
requireNonNull(groups, "groups is null").forEach(group -> {
|
||||
this.groups.add(new TreeSet<>(group));
|
||||
});
|
||||
this.predicateString = predicateString;
|
||||
this.lastUpdated = lastUpdated;
|
||||
this.sourceTableLastUpdatedTime = sourceTableLastUpdatedTime;
|
||||
this.lastUpdatedTime = lastUpdatedTime;
|
||||
this.cubeStatus = cubeStatus;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public String getCubeTableName()
|
||||
public String getCubeName()
|
||||
{
|
||||
return starTreeName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public String getOriginalTableName()
|
||||
public String getSourceTableName()
|
||||
{
|
||||
return originalTableName;
|
||||
return sourceTableName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -117,9 +121,9 @@ public class StarTreeMetadata
|
|||
|
||||
@JsonProperty
|
||||
@Override
|
||||
public long getLastUpdated()
|
||||
public long getLastUpdatedTime()
|
||||
{
|
||||
return lastUpdated;
|
||||
return lastUpdatedTime;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
|
@ -140,18 +144,10 @@ public class StarTreeMetadata
|
|||
.map(StarTreeColumn::getName).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> getAggregationsAsString()
|
||||
{
|
||||
return Collections.unmodifiableList(this.columns.stream()
|
||||
.filter(column -> AGGREGATE == column.getType())
|
||||
.map(StarTreeColumn::getUserFriendlyName).collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean matches(CubeStatement statement)
|
||||
{
|
||||
return this.originalTableName.equals(statement.getFrom()) &&
|
||||
return this.sourceTableName.equals(statement.getFrom()) &&
|
||||
hasDimensions(statement.getSelection()) &&
|
||||
hasGroup(statement.getGroupBy()) &&
|
||||
supportAggregations(statement.getAggregations()) &&
|
||||
|
|
@ -173,7 +169,7 @@ public class StarTreeMetadata
|
|||
{
|
||||
Collection<AggregationSignature> decomposedAggregations = new ArrayList<>();
|
||||
aggregations.forEach(aggregationSignature -> {
|
||||
if (AggregationSignature.AVG_FUNCTION_NAME.equals(aggregationSignature.getFunction())) {
|
||||
if (AVG.getName().equals(aggregationSignature.getFunction())) {
|
||||
decomposedAggregations.add(AggregationSignature.sum(aggregationSignature.getDimension(), false));
|
||||
decomposedAggregations.add(AggregationSignature.count(aggregationSignature.getDimension(), false));
|
||||
}
|
||||
|
|
@ -201,9 +197,11 @@ public class StarTreeMetadata
|
|||
return cubeStatus;
|
||||
}
|
||||
|
||||
public void setCubeStatus(CubeStatus cubeStatus)
|
||||
@JsonProperty
|
||||
@Override
|
||||
public long getSourceTableLastUpdatedTime()
|
||||
{
|
||||
this.cubeStatus = cubeStatus;
|
||||
return sourceTableLastUpdatedTime;
|
||||
}
|
||||
|
||||
@JsonIgnore
|
||||
|
|
@ -214,8 +212,7 @@ public class StarTreeMetadata
|
|||
}
|
||||
|
||||
@JsonIgnore
|
||||
@Override
|
||||
public Optional<String> getAggregationColumn(String aggFunction, String originalColumn, boolean distinct)
|
||||
private Optional<String> getAggregationColumn(String aggFunction, String originalColumn, boolean distinct)
|
||||
{
|
||||
return this.columns.stream()
|
||||
.filter(column -> AGGREGATE == column.getType())
|
||||
|
|
@ -260,23 +257,6 @@ public class StarTreeMetadata
|
|||
.map(AggregateColumn::getAggregateFunction);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getGroupString()
|
||||
{
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
groups.forEach(group -> {
|
||||
if (!group.isEmpty()) {
|
||||
group.forEach(column -> {
|
||||
stringBuilder.append(column).append(COLUMN_DELIMITER);
|
||||
});
|
||||
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
|
||||
}
|
||||
stringBuilder.append(GROUP_DELIMITER);
|
||||
});
|
||||
stringBuilder.deleteCharAt(stringBuilder.length() - 1);
|
||||
return stringBuilder.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
|
|
@ -287,16 +267,20 @@ public class StarTreeMetadata
|
|||
return false;
|
||||
}
|
||||
StarTreeMetadata that = (StarTreeMetadata) o;
|
||||
return lastUpdated == that.lastUpdated &&
|
||||
starTreeName.equals(that.starTreeName) &&
|
||||
originalTableName.equals(that.originalTableName) &&
|
||||
columns.equals(that.columns);
|
||||
return sourceTableLastUpdatedTime == that.sourceTableLastUpdatedTime
|
||||
&& lastUpdatedTime == that.lastUpdatedTime
|
||||
&& starTreeName.equals(that.starTreeName)
|
||||
&& sourceTableName.equals(that.sourceTableName)
|
||||
&& columns.equals(that.columns)
|
||||
&& groups.equals(that.groups)
|
||||
&& predicateString.equals(that.predicateString)
|
||||
&& cubeStatus == that.cubeStatus;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
return Objects.hash(starTreeName, originalTableName, columns, lastUpdated);
|
||||
return Objects.hash(starTreeName, sourceTableName, columns, groups, predicateString, sourceTableLastUpdatedTime, lastUpdatedTime, cubeStatus);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -304,10 +288,13 @@ public class StarTreeMetadata
|
|||
{
|
||||
return "StarTreeMetadata{" +
|
||||
"starTreeName='" + starTreeName + '\'' +
|
||||
", originalTableName='" + originalTableName + '\'' +
|
||||
", sourceTableName='" + sourceTableName + '\'' +
|
||||
", columns=" + columns +
|
||||
", groups=" + groups +
|
||||
", predicateString='" + predicateString + '\'' +
|
||||
", lastUpdated=" + lastUpdated +
|
||||
", sourceTableLastUpdatedTime=" + sourceTableLastUpdatedTime +
|
||||
", lastUpdatedTime=" + lastUpdatedTime +
|
||||
", cubeStatus=" + cubeStatus +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,25 +28,30 @@ public class StarTreeMetadataBuilder
|
|||
implements CubeMetadataBuilder
|
||||
{
|
||||
private final String starTableName;
|
||||
private final String tableName;
|
||||
private final String sourceTableName;
|
||||
private final List<StarTreeColumn> columns = new ArrayList<>();
|
||||
private final List<Set<String>> groups = new ArrayList<>();
|
||||
private String predicateString;
|
||||
private CubeStatus cubeStatus;
|
||||
private long tableLastUpdatedTime;
|
||||
private long cubeLastUpdatedTime;
|
||||
|
||||
public StarTreeMetadataBuilder(String starTableName, String tableName)
|
||||
public StarTreeMetadataBuilder(String starTableName, String sourceTableName)
|
||||
{
|
||||
this.starTableName = starTableName;
|
||||
this.tableName = tableName;
|
||||
this.sourceTableName = sourceTableName;
|
||||
}
|
||||
|
||||
public StarTreeMetadataBuilder(StarTreeMetadata starTreeMetadata)
|
||||
{
|
||||
this.starTableName = starTreeMetadata.getCubeTableName();
|
||||
this.tableName = starTreeMetadata.getOriginalTableName();
|
||||
this.starTableName = starTreeMetadata.getCubeName();
|
||||
this.sourceTableName = starTreeMetadata.getSourceTableName();
|
||||
this.columns.addAll(starTreeMetadata.getColumns());
|
||||
this.groups.add(starTreeMetadata.getGroup());
|
||||
this.predicateString = starTreeMetadata.getPredicateString();
|
||||
this.tableLastUpdatedTime = starTreeMetadata.getSourceTableLastUpdatedTime();
|
||||
this.cubeLastUpdatedTime = starTreeMetadata.getLastUpdatedTime();
|
||||
this.cubeStatus = starTreeMetadata.getCubeStatus();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -55,6 +60,18 @@ public class StarTreeMetadataBuilder
|
|||
this.cubeStatus = cubeStatus;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setTableLastUpdatedTime(long tableLastUpdatedTime)
|
||||
{
|
||||
this.tableLastUpdatedTime = tableLastUpdatedTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCubeLastUpdatedTime(long cubeLastUpdatedTime)
|
||||
{
|
||||
this.cubeLastUpdatedTime = cubeLastUpdatedTime;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addDimensionColumn(String name, String originalColumn)
|
||||
{
|
||||
|
|
@ -84,24 +101,12 @@ public class StarTreeMetadataBuilder
|
|||
{
|
||||
return new StarTreeMetadata(
|
||||
starTableName,
|
||||
tableName,
|
||||
sourceTableName,
|
||||
tableLastUpdatedTime,
|
||||
columns,
|
||||
groups,
|
||||
predicateString,
|
||||
System.currentTimeMillis(),
|
||||
cubeStatus);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CubeMetadata build(long updatedTime)
|
||||
{
|
||||
return new StarTreeMetadata(
|
||||
starTableName,
|
||||
tableName,
|
||||
columns,
|
||||
groups,
|
||||
predicateString,
|
||||
updatedTime,
|
||||
cubeLastUpdatedTime,
|
||||
cubeStatus);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ public class TestStarTreeMetaStore
|
|||
cubeMetadataService = new StarTreeProvider().getCubeMetaStore(metaStore, properties);
|
||||
cubeMetadata1 = new StarTreeMetadata("star1",
|
||||
"a",
|
||||
1000,
|
||||
ImmutableList.of(
|
||||
new AggregateColumn("sum_cost", "SUM", "cost", false),
|
||||
new DimensionColumn("value", "value")),
|
||||
|
|
@ -77,6 +78,7 @@ public class TestStarTreeMetaStore
|
|||
CubeStatus.READY);
|
||||
cubeMetadata2 = new StarTreeMetadata("star2",
|
||||
"a",
|
||||
1000,
|
||||
ImmutableList.of(
|
||||
new AggregateColumn("sum_cost", "SUM", "cost", false),
|
||||
new DimensionColumn("value", "value")),
|
||||
|
|
@ -93,7 +95,7 @@ public class TestStarTreeMetaStore
|
|||
|
||||
assertFalse(metaStore.getCatalogs().isEmpty());
|
||||
assertFalse(metaStore.getAllDatabases(CUBE_CATALOG).isEmpty());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeTableName()).isPresent());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeName()).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -103,8 +105,8 @@ public class TestStarTreeMetaStore
|
|||
cubeMetadataService.persist(cubeMetadata2);
|
||||
assertFalse(metaStore.getCatalogs().isEmpty());
|
||||
assertFalse(metaStore.getAllDatabases(CUBE_CATALOG).isEmpty());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeTableName()).isPresent());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata2.getCubeTableName()).isPresent());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeName()).isPresent());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata2.getCubeName()).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -117,8 +119,8 @@ public class TestStarTreeMetaStore
|
|||
|
||||
assertFalse(metaStore.getCatalogs().isEmpty());
|
||||
assertFalse(metaStore.getAllDatabases(CUBE_CATALOG).isEmpty());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeTableName()).isPresent());
|
||||
assertFalse(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata2.getCubeTableName()).isPresent());
|
||||
assertTrue(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata1.getCubeName()).isPresent());
|
||||
assertFalse(metaStore.getTable(CUBE_CATALOG, CUBE_DATABASE, cubeMetadata2.getCubeName()).isPresent());
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -135,7 +137,7 @@ public class TestStarTreeMetaStore
|
|||
|
||||
assertFalse(cubeMetadataService.getMetadataList("a").isEmpty());
|
||||
cubeMetadataService.getMetadataList("a").forEach(cube -> {
|
||||
assertEquals(cube.getOriginalTableName(), "a");
|
||||
assertEquals(cube.getSourceTableName(), "a");
|
||||
assertEquals(cube.getDimensions(), Collections.singleton("value"));
|
||||
});
|
||||
assertEquals(cubeMetadataService.getMetadataList("a").size(), 2);
|
||||
|
|
@ -152,13 +154,24 @@ public class TestStarTreeMetaStore
|
|||
assertNotEquals(found, cubeMetadata2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAllCubes()
|
||||
{
|
||||
cubeMetadataService.persist(cubeMetadata1);
|
||||
cubeMetadataService.persist(cubeMetadata2);
|
||||
List<CubeMetadata> result = cubeMetadataService.getAllCubes();
|
||||
assertEquals(result.size(), 2);
|
||||
assertTrue(result.containsAll(ImmutableList.of(cubeMetadata1, cubeMetadata2)));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdateMetadata()
|
||||
{
|
||||
cubeMetadataService.persist(cubeMetadata1);
|
||||
StarTreeMetadata starTreeMetadata = (StarTreeMetadata) cubeMetadata1;
|
||||
StarTreeMetadataBuilder builder = new StarTreeMetadataBuilder(starTreeMetadata);
|
||||
CubeMetadata updated = builder.build(System.currentTimeMillis());
|
||||
builder.setCubeLastUpdatedTime(System.currentTimeMillis());
|
||||
CubeMetadata updated = builder.build();
|
||||
assertNotEquals(cubeMetadata1, updated);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ public class TestStarTreeMetadata
|
|||
private final CubeMetadata metadata = new StarTreeMetadata(
|
||||
"memory.default.cube1",
|
||||
"tpch.tiny.lineitem",
|
||||
100,
|
||||
ImmutableList.of(
|
||||
new DimensionColumn("suppkey", "suppkey"),
|
||||
new DimensionColumn("returnflag", "returnflag"),
|
||||
|
|
@ -46,18 +47,23 @@ public class TestStarTreeMetadata
|
|||
new AggregateColumn("count_discount", "count", "discount", false)),
|
||||
ImmutableList.of(ImmutableSet.of("returnflag", "linestatus")),
|
||||
null,
|
||||
1000, CubeStatus.READY);
|
||||
1000,
|
||||
CubeStatus.READY);
|
||||
|
||||
@Test
|
||||
public void testCubeMetadata()
|
||||
{
|
||||
assertEquals(metadata.getCubeTableName(), "memory.default.cube1", "incorrect name");
|
||||
assertEquals(metadata.getOriginalTableName(), "tpch.tiny.lineitem", "incorrect table name");
|
||||
assertEquals(metadata.getLastUpdated(), 1000, "incorrect updating time");
|
||||
assertEquals(metadata.getCubeName(), "memory.default.cube1", "incorrect name");
|
||||
assertEquals(metadata.getSourceTableName(), "tpch.tiny.lineitem", "incorrect table name");
|
||||
assertEquals(metadata.getLastUpdatedTime(), 1000, "incorrect updating time");
|
||||
assertEquals(metadata.getAggregations(), ImmutableList.of("sum_quantity",
|
||||
"sum_extendedprice", "count_quantity", "count_extendprice", "count_discount"));
|
||||
assertEquals(metadata.getGroup(), ImmutableSet.of("returnflag", "linestatus"));
|
||||
assertEquals(metadata.getDimensions(), ImmutableList.of("suppkey", "returnflag", "linestatus", "shipdate", "discount", "quantity"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMatchingValidStatement()
|
||||
public void testMetadataMatchesCubeStatement()
|
||||
{
|
||||
CubeStatement statement = CubeStatement.newBuilder()
|
||||
.select("returnflag", "linestatus")
|
||||
|
|
@ -69,7 +75,7 @@ public class TestStarTreeMetadata
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testNotMatchingValidStatement1()
|
||||
public void testMetadataNotMatchesCubeStatement()
|
||||
{
|
||||
CubeStatement statement = CubeStatement.newBuilder()
|
||||
.select("returnflag", "linestatus")
|
||||
|
|
|
|||
1
pom.xml
1
pom.xml
|
|
@ -1622,6 +1622,7 @@
|
|||
<item>${air.main.basedir}/src/main/resource/license/license-header.txt</item>
|
||||
<item>${air.main.basedir}/src/main/resource/license/license-header-alternate-2010.txt</item>
|
||||
<item>${air.main.basedir}/src/main/resource/license/license-header-alternate-2012.txt</item>
|
||||
<item>${air.main.basedir}/src/main/resource/license/license-header-alternate-2020.txt</item>
|
||||
</validHeaders>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ package io.prestosql.execution;
|
|||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.util.concurrent.ListenableFuture;
|
||||
import io.hetu.core.spi.cube.CubeAggregateFunction;
|
||||
import io.hetu.core.spi.cube.CubeMetadataBuilder;
|
||||
import io.hetu.core.spi.cube.CubeStatus;
|
||||
import io.hetu.core.spi.cube.aggregator.AggregationSignature;
|
||||
|
|
@ -108,7 +109,7 @@ public class CreateCubeTask
|
|||
throw new RuntimeException("HetuMetaStore is not initialized");
|
||||
}
|
||||
QualifiedObjectName cubeName = createQualifiedObjectName(session, statement, statement.getCubeName());
|
||||
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName());
|
||||
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getSourceTableName());
|
||||
Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, cubeName);
|
||||
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
|
||||
|
||||
|
|
@ -134,12 +135,12 @@ public class CreateCubeTask
|
|||
}
|
||||
|
||||
if (!tableHandle.isPresent()) {
|
||||
throw new SemanticException(MISSING_TABLE, statement, "Table %s does not exist", cubeName);
|
||||
throw new SemanticException(MISSING_TABLE, statement, "Table '%s' does not exist", tableName);
|
||||
}
|
||||
|
||||
TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle.get());
|
||||
List<String> groupingSet = statement.getGroupingSet().stream().map(Identifier::getValue).collect(Collectors.toList());
|
||||
Map<String, ColumnMetadata> originalTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
|
||||
Map<String, ColumnMetadata> sourceTableColumns = tableMetadata.getColumns().stream().collect(Collectors.toMap(ColumnMetadata::getName, col -> col));
|
||||
List<ColumnMetadata> cubeColumns = new ArrayList<>();
|
||||
Map<String, AggregationSignature> aggregations = new HashMap<>();
|
||||
Analysis analysis = analyzeStatement(statement, session, metadata, accessControl, parameters, stateMachine.getWarningCollector());
|
||||
|
|
@ -150,21 +151,22 @@ public class CreateCubeTask
|
|||
String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue();
|
||||
boolean distinct = aggFunction.isDistinct();
|
||||
String cubeColumnName = aggFunctionName + "_" + (argument == null ? "all" : argument) + (aggFunction.isDistinct() ? "_distinct" : "");
|
||||
switch (aggFunctionName) {
|
||||
case AggregationSignature.SUM_FUNCTION_NAME:
|
||||
CubeAggregateFunction cubeAggregateFunction = CubeAggregateFunction.valueOf(aggFunctionName.toUpperCase(ENGLISH));
|
||||
switch (cubeAggregateFunction) {
|
||||
case SUM:
|
||||
aggregations.put(cubeColumnName, AggregationSignature.sum(argument, distinct));
|
||||
break;
|
||||
case AggregationSignature.COUNT_FUNCTION_NAME:
|
||||
case COUNT:
|
||||
AggregationSignature aggregationSignature = argument == null ? AggregationSignature.count() : AggregationSignature.count(argument, distinct);
|
||||
aggregations.put(cubeColumnName, aggregationSignature);
|
||||
break;
|
||||
case AggregationSignature.AVG_FUNCTION_NAME:
|
||||
case AVG:
|
||||
aggregations.put(cubeColumnName, AggregationSignature.avg(argument, distinct));
|
||||
break;
|
||||
case AggregationSignature.MAX_FUNCTION_NAME:
|
||||
case MAX:
|
||||
aggregations.put(cubeColumnName, AggregationSignature.max(argument, distinct));
|
||||
break;
|
||||
case AggregationSignature.MIN_FUNCTION_NAME:
|
||||
case MIN:
|
||||
aggregations.put(cubeColumnName, AggregationSignature.min(argument, distinct));
|
||||
break;
|
||||
default:
|
||||
|
|
@ -195,15 +197,16 @@ public class CreateCubeTask
|
|||
|
||||
if (properties.containsKey("partitioned_by")) {
|
||||
List<String> partitionCols = new ArrayList<>(((List<String>) properties.get("partitioned_by")));
|
||||
// put all partition columns at the end of the list
|
||||
groupingSet.removeAll(partitionCols);
|
||||
groupingSet.addAll(partitionCols);
|
||||
}
|
||||
|
||||
for (String dimension : groupingSet) {
|
||||
if (!originalTableColumns.containsKey(dimension)) {
|
||||
if (!sourceTableColumns.containsKey(dimension)) {
|
||||
throw new SemanticException(MISSING_COLUMN, statement, "Column %s does not exist", dimension);
|
||||
}
|
||||
ColumnMetadata tableCol = originalTableColumns.get(dimension);
|
||||
ColumnMetadata tableCol = sourceTableColumns.get(dimension);
|
||||
ColumnMetadata cubeCol = new ColumnMetadata(
|
||||
dimension,
|
||||
tableCol.getType(),
|
||||
|
|
@ -230,7 +233,10 @@ public class CreateCubeTask
|
|||
groupingSet.forEach(dimension -> builder.addDimensionColumn(dimension, dimension));
|
||||
aggregations.forEach((column, aggregationSignature) -> builder.addAggregationColumn(column, aggregationSignature.getFunction(), aggregationSignature.getDimension(), aggregationSignature.isDistinct()));
|
||||
builder.addGroup(new HashSet<>(groupingSet));
|
||||
//Status and Table modified time will be updated on the first insert into the cube
|
||||
builder.setCubeStatus(CubeStatus.INACTIVE);
|
||||
builder.setTableLastUpdatedTime(-1L);
|
||||
builder.setCubeLastUpdatedTime(System.currentTimeMillis());
|
||||
optionalCubeMetaStore.get().persist(builder.build());
|
||||
|
||||
return immediateFuture(null);
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ public class DropTableTask
|
|||
if (optionalCubeMetaStore.isPresent()) {
|
||||
List<CubeMetadata> cubes = optionalCubeMetaStore.get().getMetadataList(tableName.toString());
|
||||
for (CubeMetadata cube : cubes) {
|
||||
String[] parts = cube.getCubeTableName().split("\\.");
|
||||
String[] parts = cube.getCubeName().split("\\.");
|
||||
Optional<TableHandle> cubeHandle = metadata.getTableHandle(session, createQualifiedObjectName(session, null, QualifiedName.of(parts[0], parts[1], parts[2])));
|
||||
cubeHandle.ifPresent(cubeTable -> metadata.dropTable(session, cubeTable));
|
||||
optionalCubeMetaStore.get().removeCube(cube);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import io.hetu.core.spi.cube.io.CubeMetaStore;
|
|||
import io.prestosql.Session;
|
||||
import io.prestosql.cube.CubeManager;
|
||||
import io.prestosql.spi.Page;
|
||||
import io.prestosql.spi.cube.CubeUpdateMetadata;
|
||||
import io.prestosql.spi.plan.PlanNodeId;
|
||||
import io.prestosql.spi.snapshot.RestorableConfig;
|
||||
import io.prestosql.sql.ExpressionFormatter;
|
||||
|
|
@ -46,9 +47,7 @@ public class CubeFinishOperator
|
|||
private final PlanNodeId planNodeId;
|
||||
private final Session session;
|
||||
private final CubeManager cubeManager;
|
||||
private final String cubeName;
|
||||
private final Expression newDataPredicate;
|
||||
private final boolean overwrite;
|
||||
private final CubeUpdateMetadata metadata;
|
||||
private boolean closed;
|
||||
|
||||
public CubeFinishOperatorFactory(
|
||||
|
|
@ -56,17 +55,13 @@ public class CubeFinishOperator
|
|||
PlanNodeId planNodeId,
|
||||
Session session,
|
||||
CubeManager cubeManager,
|
||||
String cubeName,
|
||||
Expression newDataPredicate,
|
||||
boolean overwrite)
|
||||
CubeUpdateMetadata metadata)
|
||||
{
|
||||
this.operatorId = operatorId;
|
||||
this.planNodeId = requireNonNull(planNodeId, "planNodeId is null");
|
||||
this.session = requireNonNull(session, "session is null");
|
||||
this.cubeManager = requireNonNull(cubeManager, "cubeManager is null");
|
||||
this.cubeName = requireNonNull(cubeName, "starTableName is null");
|
||||
this.newDataPredicate = newDataPredicate;
|
||||
this.overwrite = overwrite;
|
||||
this.metadata = requireNonNull(metadata, "metadata is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -74,7 +69,7 @@ public class CubeFinishOperator
|
|||
{
|
||||
checkState(!closed, "Factory is already closed");
|
||||
OperatorContext context = driverContext.addOperatorContext(operatorId, planNodeId, CubeFinishOperator.class.getSimpleName());
|
||||
return new CubeFinishOperator(context, cubeManager, cubeName, newDataPredicate, overwrite);
|
||||
return new CubeFinishOperator(context, cubeManager, metadata);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -86,7 +81,7 @@ public class CubeFinishOperator
|
|||
@Override
|
||||
public OperatorFactory duplicate()
|
||||
{
|
||||
return new CubeFinishOperatorFactory(operatorId, planNodeId, session, cubeManager, cubeName, newDataPredicate, overwrite);
|
||||
return new CubeFinishOperatorFactory(operatorId, planNodeId, session, cubeManager, metadata);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -99,24 +94,18 @@ public class CubeFinishOperator
|
|||
|
||||
private final OperatorContext operatorContext;
|
||||
private final CubeMetaStore cubeMetastore;
|
||||
private final String cubeName;
|
||||
private final Expression newDataPredicate;
|
||||
private final boolean overwrite;
|
||||
private final CubeUpdateMetadata updateMetadata;
|
||||
private State state = State.NEEDS_INPUT;
|
||||
private Page page;
|
||||
|
||||
public CubeFinishOperator(
|
||||
OperatorContext operatorContext,
|
||||
CubeManager cubeManager,
|
||||
String cubeName,
|
||||
Expression newDataPredicate,
|
||||
boolean overwrite)
|
||||
CubeUpdateMetadata updateMetadata)
|
||||
{
|
||||
this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");
|
||||
this.cubeMetastore = cubeManager.getMetaStore(STAR_TREE).get();
|
||||
this.cubeName = cubeName;
|
||||
this.newDataPredicate = newDataPredicate;
|
||||
this.overwrite = overwrite;
|
||||
this.updateMetadata = updateMetadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -146,21 +135,26 @@ public class CubeFinishOperator
|
|||
if (state != State.HAS_OUTPUT) {
|
||||
return null;
|
||||
}
|
||||
CubeMetadata cubeMetadata = cubeMetastore.getMetadataFromCubeName(cubeName).get();
|
||||
CubeMetadata cubeMetadata = cubeMetastore.getMetadataFromCubeName(updateMetadata.getCubeName()).get();
|
||||
CubeMetadataBuilder builder = cubeMetastore.getBuilder(cubeMetadata);
|
||||
Expression updatable;
|
||||
if (overwrite || cubeMetadata.getPredicateString() == null) {
|
||||
updatable = newDataPredicate;
|
||||
if (updateMetadata.getDataPredicateString() == null) {
|
||||
//Ensure that existing predicate metadata is reset.
|
||||
builder.withPredicate(null);
|
||||
}
|
||||
else {
|
||||
Expression existing = new SqlParser().createExpression(cubeMetadata.getPredicateString(), new ParsingOptions());
|
||||
updatable = ExpressionUtils.or(existing, newDataPredicate);
|
||||
Expression updatable = new SqlParser().createExpression(updateMetadata.getDataPredicateString(), new ParsingOptions());
|
||||
//Merge new data predicate with existing predicate string
|
||||
if (!updateMetadata.isOverwrite() && cubeMetadata.getPredicateString() != null) {
|
||||
Expression existing = new SqlParser().createExpression(cubeMetadata.getPredicateString(), new ParsingOptions());
|
||||
updatable = ExpressionUtils.or(existing, updatable);
|
||||
}
|
||||
//TODO: Add Logic to simplify expression. Check if Two between predicates can be merged into one
|
||||
builder.withPredicate(ExpressionFormatter.formatExpression(updatable, Optional.empty()));
|
||||
}
|
||||
//TODO: Add Logic to simplify expression. Check if Two between predicates can be merged into one
|
||||
builder.withPredicate(ExpressionFormatter.formatExpression(updatable, Optional.empty()));
|
||||
builder.setTableLastUpdatedTime(updateMetadata.getTableLastUpdatedTime());
|
||||
builder.setCubeLastUpdatedTime(System.currentTimeMillis());
|
||||
builder.setCubeStatus(READY);
|
||||
CubeMetadata update = builder.build(System.currentTimeMillis());
|
||||
cubeMetastore.persist(update);
|
||||
cubeMetastore.persist(builder.build());
|
||||
state = State.FINISHED;
|
||||
return page;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -875,11 +875,13 @@ public class Analysis
|
|||
public static final class CubeInsert
|
||||
{
|
||||
private final TableHandle target;
|
||||
private final TableHandle sourceTable;
|
||||
private final List<ColumnHandle> columns;
|
||||
|
||||
public CubeInsert(TableHandle target, List<ColumnHandle> columns)
|
||||
public CubeInsert(TableHandle target, TableHandle sourceTable, List<ColumnHandle> columns)
|
||||
{
|
||||
this.target = requireNonNull(target, "target is null");
|
||||
this.sourceTable = requireNonNull(sourceTable, "sourceTable is null");
|
||||
this.columns = requireNonNull(columns, "columns is null");
|
||||
checkArgument(columns.size() > 0, "No columns given to insert");
|
||||
}
|
||||
|
|
@ -893,6 +895,11 @@ public class Analysis
|
|||
{
|
||||
return target;
|
||||
}
|
||||
|
||||
public TableHandle getSourceTable()
|
||||
{
|
||||
return sourceTable;
|
||||
}
|
||||
}
|
||||
|
||||
@Immutable
|
||||
|
|
|
|||
|
|
@ -1341,7 +1341,7 @@ public class FeaturesConfig
|
|||
}
|
||||
|
||||
@Config("cube.metadata-cache-ttl")
|
||||
@ConfigDescription("The maximum time to live that are be loaded into cache before eviction happens")
|
||||
@ConfigDescription("The maximum time to live for cube metadata that were loaded into cache before eviction happens")
|
||||
public FeaturesConfig setCubeMetadataCacheTtl(Duration cubeMetadataCacheTtl)
|
||||
{
|
||||
this.cubeMetadataCacheTtl = cubeMetadataCacheTtl;
|
||||
|
|
|
|||
|
|
@ -118,4 +118,7 @@ public enum SemanticErrorCode
|
|||
INVALID_FETCH_FIRST_ROW_COUNT,
|
||||
INVALID_LIMIT_ROW_COUNT,
|
||||
MISSING_ORDER_BY,
|
||||
|
||||
TABLE_STATE_INCORRECT,
|
||||
PREDICATE_OVERLAP
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ import com.google.common.collect.ImmutableSet;
|
|||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Multimap;
|
||||
import io.hetu.core.spi.cube.CubeAggregateFunction;
|
||||
import io.hetu.core.spi.cube.CubeMetadata;
|
||||
import io.hetu.core.spi.cube.CubeStatus;
|
||||
import io.hetu.core.spi.cube.io.CubeMetaStore;
|
||||
import io.prestosql.Session;
|
||||
import io.prestosql.SystemSessionProperties;
|
||||
|
|
@ -50,6 +52,7 @@ import io.prestosql.spi.function.OperatorType;
|
|||
import io.prestosql.spi.heuristicindex.IndexClient;
|
||||
import io.prestosql.spi.heuristicindex.Pair;
|
||||
import io.prestosql.spi.metadata.TableHandle;
|
||||
import io.prestosql.spi.plan.Symbol;
|
||||
import io.prestosql.spi.security.AccessDeniedException;
|
||||
import io.prestosql.spi.security.Identity;
|
||||
import io.prestosql.spi.security.ViewExpression;
|
||||
|
|
@ -62,9 +65,12 @@ import io.prestosql.spi.type.Type;
|
|||
import io.prestosql.spi.type.TypeNotFoundException;
|
||||
import io.prestosql.spi.type.TypeSignature;
|
||||
import io.prestosql.spi.type.VarcharType;
|
||||
import io.prestosql.sql.ExpressionFormatter;
|
||||
import io.prestosql.sql.ExpressionUtils;
|
||||
import io.prestosql.sql.SqlPath;
|
||||
import io.prestosql.sql.parser.ParsingException;
|
||||
import io.prestosql.sql.parser.SqlParser;
|
||||
import io.prestosql.sql.planner.ExpressionDomainTranslator;
|
||||
import io.prestosql.sql.planner.ExpressionInterpreter;
|
||||
import io.prestosql.sql.planner.SymbolsExtractor;
|
||||
import io.prestosql.sql.planner.TypeProvider;
|
||||
|
|
@ -73,6 +79,7 @@ import io.prestosql.sql.tree.AliasedRelation;
|
|||
import io.prestosql.sql.tree.AllColumns;
|
||||
import io.prestosql.sql.tree.Analyze;
|
||||
import io.prestosql.sql.tree.AssignmentItem;
|
||||
import io.prestosql.sql.tree.BooleanLiteral;
|
||||
import io.prestosql.sql.tree.Call;
|
||||
import io.prestosql.sql.tree.Comment;
|
||||
import io.prestosql.sql.tree.Commit;
|
||||
|
|
@ -182,6 +189,7 @@ import java.util.Optional;
|
|||
import java.util.OptionalLong;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.function.LongSupplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
|
|
@ -245,6 +253,7 @@ import static io.prestosql.sql.analyzer.SemanticErrorCode.MISMATCHED_SET_COLUMN_
|
|||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ATTRIBUTE;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CATALOG;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_COLUMN;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_CUBE;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_ORDER_BY;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_SCHEMA;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE;
|
||||
|
|
@ -254,7 +263,9 @@ import static io.prestosql.sql.analyzer.SemanticErrorCode.NONDETERMINISTIC_ORDER
|
|||
import static io.prestosql.sql.analyzer.SemanticErrorCode.NON_NUMERIC_SAMPLE_PERCENTAGE;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.ORDER_BY_MUST_BE_IN_SELECT;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.PREDICATE_OVERLAP;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_ALREADY_EXISTS;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.TABLE_STATE_INCORRECT;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_ARGUMENTS;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.TOO_MANY_GROUPING_SETS;
|
||||
import static io.prestosql.sql.analyzer.SemanticErrorCode.TYPE_MISMATCH;
|
||||
|
|
@ -446,38 +457,6 @@ class StatementAnalyzer
|
|||
return createAndAssignScope(insert, scope, Field.newUnqualified("rows", BIGINT));
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Scope visitInsertCube(InsertCube insertCube, Optional<Scope> scope)
|
||||
{
|
||||
QualifiedObjectName targetCube = createQualifiedObjectName(session, insertCube, insertCube.getCubeName()); // check if target is view
|
||||
if (metadata.getView(session, targetCube).isPresent()) {
|
||||
throw new SemanticException(NOT_SUPPORTED, insertCube, "Inserting into view is not supported");
|
||||
} // check if target cube is present
|
||||
Optional<CubeMetaStore> optionalCubeMetaStore = cubeManager.getMetaStore(STAR_TREE);
|
||||
if (!optionalCubeMetaStore.isPresent() || !optionalCubeMetaStore.get().getMetadataFromCubeName(targetCube.toString()).isPresent()) {
|
||||
throw new SemanticException(INSERT_INTO_CUBE, insertCube, "%s is not a star-tree cube, INSERT INTO CUBE is not applicable.", targetCube);
|
||||
} // check if target cube is present as a table
|
||||
Optional<TableHandle> targetCubeHandle = metadata.getTableHandle(session, targetCube);
|
||||
if (!targetCubeHandle.isPresent()) {
|
||||
throw new SemanticException(MISSING_TABLE, insertCube, "Table '%s' does not exist", targetCube);
|
||||
} // analyze the query that creates the data
|
||||
Scope queryScope = process(insertCube.getQuery(), scope);
|
||||
accessControl.checkCanInsertIntoTable(session.getRequiredTransactionId(), session.getIdentity(), targetCube);
|
||||
if (insertCube.isOverwrite()) {
|
||||
// set the insert as insert overwrite
|
||||
analysis.setUpdateType("INSERT OVERWRITE CUBE", targetCube);
|
||||
analysis.setCubeOverwrite(true);
|
||||
}
|
||||
else {
|
||||
analysis.setUpdateType("INSERT CUBE", targetCube);
|
||||
}
|
||||
Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetCubeHandle.get());
|
||||
analysis.setCubeInsert(new Analysis.CubeInsert(
|
||||
targetCubeHandle.get(),
|
||||
insertCube.getColumns().stream().map(Identifier::getValue).map(columnHandles::get).collect(Collectors.toList())));
|
||||
return createAndAssignScope(insertCube, scope, Field.newUnqualified("rows", BIGINT));
|
||||
}
|
||||
|
||||
private boolean typesMatchForInsert(Iterable<Type> tableTypes, Iterable<Type> queryTypes)
|
||||
{
|
||||
if (Iterables.size(tableTypes) != Iterables.size(queryTypes)) {
|
||||
|
|
@ -530,6 +509,107 @@ class StatementAnalyzer
|
|||
return type instanceof CharType || (type instanceof VarcharType && !((VarcharType) type).isUnbounded()) || hasNestedBoundedCharacterType(type);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Scope visitInsertCube(InsertCube insertCube, Optional<Scope> scope)
|
||||
{
|
||||
QualifiedObjectName targetCube = createQualifiedObjectName(session, insertCube, insertCube.getCubeName());
|
||||
CubeMetaStore cubeMetaStore = cubeManager.getMetaStore(STAR_TREE).orElseThrow(() -> new RuntimeException("Hetu metastore must be initialized"));
|
||||
CubeMetadata cubeMetadata = cubeMetaStore.getMetadataFromCubeName(targetCube.toString())
|
||||
.orElseThrow(() -> new SemanticException(INSERT_INTO_CUBE, insertCube, "Cube '%s' is not found, INSERT INTO CUBE is not applicable.", targetCube));
|
||||
Optional<TableHandle> targetCubeHandle = metadata.getTableHandle(session, targetCube);
|
||||
if (!targetCubeHandle.isPresent()) {
|
||||
throw new SemanticException(MISSING_CUBE, insertCube, "Cube '%s' table handle does not exist", targetCube);
|
||||
}
|
||||
|
||||
QualifiedObjectName tableName = QualifiedObjectName.valueOf(cubeMetadata.getSourceTableName());
|
||||
TableHandle sourceTableHandle = metadata.getTableHandle(session, tableName)
|
||||
.orElseThrow(() -> new SemanticException(MISSING_TABLE, insertCube, "Source table '%s' on which cube was built is missing", tableName.toString()));
|
||||
|
||||
//Cube status is determined based on the last modified timestamp of the source table
|
||||
//Without that Cube might return incorrect results if the table was updated but cube was not.
|
||||
LongSupplier tableLastModifiedTime = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
|
||||
if (tableLastModifiedTime == null) {
|
||||
throw new SemanticException(TABLE_STATE_INCORRECT, insertCube, "Cannot allow insert into cube. Cube might return incorrect results. Unable to identify last modified of the time source table.");
|
||||
}
|
||||
// If Original table was updated since Cube was built then We cannot allow any more updates on the Cube.
|
||||
// User must create new cube from the source table and try insert overwrite cube
|
||||
if (!insertCube.isOverwrite() && cubeMetadata.getCubeStatus() == CubeStatus.READY && tableLastModifiedTime.getAsLong() > cubeMetadata.getSourceTableLastUpdatedTime()) {
|
||||
throw new SemanticException(TABLE_STATE_INCORRECT, insertCube, "Cannot insert into cube. Source table has been updated since Cube was last updated. Try INSERT OVERWRITE CUBE or Create new a cube");
|
||||
}
|
||||
|
||||
Scope queryScope = process(insertCube.getQuery(), scope);
|
||||
accessControl.checkCanInsertIntoTable(session.getRequiredTransactionId(), session.getIdentity(), targetCube);
|
||||
if (insertCube.isOverwrite()) {
|
||||
// set the insert as insert overwrite
|
||||
analysis.setUpdateType("INSERT OVERWRITE CUBE", targetCube);
|
||||
analysis.setCubeOverwrite(true);
|
||||
}
|
||||
else {
|
||||
analysis.setUpdateType("INSERT CUBE", targetCube);
|
||||
}
|
||||
if (!insertCube.isOverwrite() && !insertCube.getWhere().isPresent() && cubeMetadata.getCubeStatus() != CubeStatus.INACTIVE) {
|
||||
//Means data some data was inserted before, but trying to insert entire dataset
|
||||
throw new SemanticException(PREDICATE_OVERLAP, insertCube, "Cannot allow insert. Inserting entire dataset but cube already has partial data");
|
||||
}
|
||||
else if (!insertCube.isOverwrite() && insertCube.getWhere().isPresent() && arePredicatesOverlapping(insertCube.getWhere().get(), cubeMetadata)) {
|
||||
throw new SemanticException(PREDICATE_OVERLAP, insertCube, "Cannot allow insert. Cube already contains data for the given predicate '%s'", ExpressionFormatter.formatExpression(insertCube.getWhere().get(), Optional.empty()));
|
||||
}
|
||||
Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, targetCubeHandle.get());
|
||||
analysis.setCubeInsert(new Analysis.CubeInsert(
|
||||
targetCubeHandle.get(),
|
||||
sourceTableHandle,
|
||||
insertCube.getColumns().stream().map(Identifier::getValue).map(columnHandles::get).collect(Collectors.toList())));
|
||||
return createAndAssignScope(insertCube, scope, Field.newUnqualified("rows", BIGINT));
|
||||
}
|
||||
|
||||
private boolean arePredicatesOverlapping(Expression newDataPredicate, CubeMetadata cubeMetadata)
|
||||
{
|
||||
ImmutableMap.Builder<Symbol, Type> typesBuilder = ImmutableMap.builder();
|
||||
new SymbolTypeBuilderVisitor(analysis.getTypes()).process(newDataPredicate, typesBuilder);
|
||||
TypeProvider types = TypeProvider.viewOf(typesBuilder.build());
|
||||
|
||||
newDataPredicate = ExpressionUtils.rewriteIdentifiersToSymbolReferences(newDataPredicate);
|
||||
ExpressionDomainTranslator.ExtractionResult decomposedNewDataPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, newDataPredicate, types);
|
||||
if (!BooleanLiteral.TRUE_LITERAL.equals(decomposedNewDataPredicate.getRemainingExpression())) {
|
||||
throw new RuntimeException(String.format("Cannot support predicate '%s'", ExpressionFormatter.formatExpression(newDataPredicate, Optional.empty())));
|
||||
}
|
||||
if (cubeMetadata.getCubeStatus() == CubeStatus.INACTIVE) {
|
||||
//Inactive cubes are empty. So inserts should be allowed.
|
||||
return false;
|
||||
}
|
||||
|
||||
if (cubeMetadata.getPredicateString() == null) {
|
||||
//Means Cube was created for entire dataset.
|
||||
return true;
|
||||
}
|
||||
SqlParser sqlParser = new SqlParser();
|
||||
Expression cubePredicateAsExpr = sqlParser.createExpression(cubeMetadata.getPredicateString(), createParsingOptions(session));
|
||||
cubePredicateAsExpr = ExpressionUtils.rewriteIdentifiersToSymbolReferences(cubePredicateAsExpr);
|
||||
ExpressionDomainTranslator.ExtractionResult decomposedCubePredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, cubePredicateAsExpr, types);
|
||||
return decomposedCubePredicate.getTupleDomain().overlaps(decomposedNewDataPredicate.getTupleDomain());
|
||||
}
|
||||
|
||||
private class SymbolTypeBuilderVisitor
|
||||
extends DefaultTraversalVisitor<Void, ImmutableMap.Builder<Symbol, Type>>
|
||||
{
|
||||
private final Map<NodeRef<Expression>, Type> types;
|
||||
|
||||
private SymbolTypeBuilderVisitor(Map<NodeRef<Expression>, Type> types)
|
||||
{
|
||||
this.types = requireNonNull(types, "types is null");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Void visitIdentifier(Identifier identifier, ImmutableMap.Builder<Symbol, Type> builder)
|
||||
{
|
||||
NodeRef<Expression> expressionRef = NodeRef.of(identifier);
|
||||
if (types.containsKey(expressionRef)) {
|
||||
builder.put(new Symbol(identifier.getValue()), types.get(expressionRef));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Scope visitDelete(Delete node, Optional<Scope> scope)
|
||||
{
|
||||
|
|
@ -717,7 +797,7 @@ class StatementAnalyzer
|
|||
|
||||
Set<String> cubeSupportedFunctions = CubeAggregateFunction.SUPPORTED_FUNCTIONS;
|
||||
Set<FunctionCall> aggFunctions = node.getAggregations();
|
||||
Scope queryScope = process(new Table(node.getTableName()), scope);
|
||||
Scope queryScope = process(new Table(node.getSourceTableName()), scope);
|
||||
ImmutableList.Builder<Field> outputFields = ImmutableList.builder();
|
||||
for (FunctionCall aggFunction : aggFunctions) {
|
||||
String argument = aggFunction.getArguments().isEmpty() || aggFunction.getArguments().get(0) instanceof LongLiteral ? null : ((Identifier) aggFunction.getArguments().get(0)).getValue();
|
||||
|
|
|
|||
|
|
@ -2907,9 +2907,7 @@ public class LocalExecutionPlanner
|
|||
node.getId(),
|
||||
session,
|
||||
cubeManager,
|
||||
node.getCubeName(),
|
||||
node.getDataPredicate(),
|
||||
node.isOverwrite());
|
||||
node.getMetadata());
|
||||
Map<Symbol, Integer> layout = ImmutableMap.of(node.getOutputSymbols().get(0), 0);
|
||||
return new PhysicalOperation(operatorFactory, layout, context, source);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import io.prestosql.spi.connector.ColumnHandle;
|
|||
import io.prestosql.spi.connector.ColumnMetadata;
|
||||
import io.prestosql.spi.connector.ConnectorTableMetadata;
|
||||
import io.prestosql.spi.connector.QualifiedObjectName;
|
||||
import io.prestosql.spi.cube.CubeUpdateMetadata;
|
||||
import io.prestosql.spi.function.Signature;
|
||||
import io.prestosql.spi.metadata.TableHandle;
|
||||
import io.prestosql.spi.operator.ReuseExchangeOperator;
|
||||
|
|
@ -51,6 +52,7 @@ import io.prestosql.spi.statistics.TableStatisticsMetadata;
|
|||
import io.prestosql.spi.type.CharType;
|
||||
import io.prestosql.spi.type.Type;
|
||||
import io.prestosql.spi.type.VarcharType;
|
||||
import io.prestosql.sql.ExpressionFormatter;
|
||||
import io.prestosql.sql.analyzer.Analysis;
|
||||
import io.prestosql.sql.analyzer.Field;
|
||||
import io.prestosql.sql.analyzer.RelationId;
|
||||
|
|
@ -107,6 +109,7 @@ import java.util.Map;
|
|||
import java.util.Map.Entry;
|
||||
import java.util.Optional;
|
||||
import java.util.UUID;
|
||||
import java.util.function.LongSupplier;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
|
|
@ -474,15 +477,25 @@ public class LogicalPlanner
|
|||
newTableLayout,
|
||||
statisticsMetadata);
|
||||
Expression cubeWhere = analysis.getWhere((QuerySpecification) (insertCubeStatement.getQuery().getQueryBody()));
|
||||
Expression rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator)
|
||||
.rewriteExpression(tableWriterPlan, cubeWhere, analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
|
||||
Expression rewritten = null;
|
||||
if (cubeWhere != null) {
|
||||
rewritten = new QueryPlanner(analysis, planSymbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator), metadata, session, namedSubPlan, uniqueIdAllocator)
|
||||
.rewriteExpression(tableWriterPlan, cubeWhere, analysis, buildLambdaDeclarationToSymbolMap(analysis, planSymbolAllocator));
|
||||
}
|
||||
TableHandle sourceTableHandle = insert.getSourceTable();
|
||||
//At this point it has been verified that source table has not been updated
|
||||
//so insert into cube should be allowed
|
||||
LongSupplier tableLastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, sourceTableHandle);
|
||||
checkState(tableLastModifiedTimeSupplier != null, "Table last modified time is null");
|
||||
CubeFinishNode cubeFinishNode = new CubeFinishNode(
|
||||
idAllocator.getNextId(),
|
||||
tableWriterPlan.getRoot(),
|
||||
planSymbolAllocator.newSymbol("rows", BIGINT),
|
||||
tableMetadata.getQualifiedName().toString(),
|
||||
rewritten,
|
||||
insertCubeStatement.isOverwrite());
|
||||
new CubeUpdateMetadata(
|
||||
tableMetadata.getQualifiedName().toString(),
|
||||
tableLastModifiedTimeSupplier.getAsLong(),
|
||||
cubeWhere != null ? ExpressionFormatter.formatExpression(rewritten, Optional.empty()) : null,
|
||||
insertCubeStatement.isOverwrite()));
|
||||
return new RelationPlan(cubeFinishNode, analysis.getScope(insertCubeStatement), cubeFinishNode.getOutputSymbols());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,8 @@ import java.util.Set;
|
|||
import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.COUNT;
|
||||
import static io.hetu.core.spi.cube.CubeAggregateFunction.SUM;
|
||||
import static io.prestosql.spi.StandardErrorCode.CUBE_ERROR;
|
||||
import static io.prestosql.spi.plan.AggregationNode.singleGroupingSet;
|
||||
import static io.prestosql.sql.planner.SymbolUtils.toSymbolReference;
|
||||
|
|
@ -98,7 +100,7 @@ public class AggregationRewriteWithCube
|
|||
|
||||
public PlanNode rewrite(AggregationNode originalAggregationNode, PlanNode filterNode)
|
||||
{
|
||||
QualifiedObjectName starTreeTableName = QualifiedObjectName.valueOf(cubeMetadata.getCubeTableName());
|
||||
QualifiedObjectName starTreeTableName = QualifiedObjectName.valueOf(cubeMetadata.getCubeName());
|
||||
TableHandle cubeTableHandle = metadata.getTableHandle(session, starTreeTableName)
|
||||
.orElseThrow(() -> new CubeNotFoundException(starTreeTableName.toString()));
|
||||
Map<String, ColumnHandle> cubeColumnsMap = metadata.getColumnHandles(session, cubeTableHandle);
|
||||
|
|
@ -130,7 +132,7 @@ public class AggregationRewriteWithCube
|
|||
ColumnMetadata cubeColumnMetadata = metadata.getColumnMetadata(session, cubeTableHandle, cubeColHandle);
|
||||
AggregationSignature aggregationSignature = cubeMetadata.getAggregationSignature(cubeColumnMetadata.getName())
|
||||
.orElseThrow(() -> new ColumnNotFoundException(new SchemaTableName(starTreeTableName.getSchemaName(), starTreeTableName.getObjectName()), cubeColHandle.getColumnName()));
|
||||
String aggFunction = AggregationSignature.COUNT_FUNCTION_NAME.equals(aggregationSignature.getFunction()) ? "sum" : aggregationSignature.getFunction();
|
||||
String aggFunction = COUNT.getName().equals(aggregationSignature.getFunction()) ? "sum" : aggregationSignature.getFunction();
|
||||
SymbolReference argument = toSymbolReference(aggregatorSource.getScanSymbol());
|
||||
FunctionHandle functionHandle = metadata.getFunctionAndTypeManager().lookupFunction(aggFunction, TypeSignatureProvider.fromTypeSignatures(typeSignature));
|
||||
aggregationsBuilder.put(aggregatorSource.getOriginalAggSymbol(), new AggregationNode.Aggregation(
|
||||
|
|
@ -320,7 +322,7 @@ public class AggregationRewriteWithCube
|
|||
}
|
||||
break;
|
||||
case "avg":
|
||||
AggregationSignature sumSignature = new AggregationSignature(AggregationSignature.SUM_FUNCTION_NAME, originalColumnName, distinct);
|
||||
AggregationSignature sumSignature = new AggregationSignature(SUM.getName(), originalColumnName, distinct);
|
||||
String sumColumnName = cubeMetadata.getColumn(sumSignature)
|
||||
.orElseThrow(() -> new PrestoException(CUBE_ERROR, "Cannot find column associated with aggregation " + sumSignature));
|
||||
ColumnHandle sumColumnHandle = cubeColumnsMap.get(sumColumnName);
|
||||
|
|
@ -341,7 +343,7 @@ public class AggregationRewriteWithCube
|
|||
}
|
||||
}
|
||||
}
|
||||
AggregationSignature countSignature = new AggregationSignature(AggregationSignature.COUNT_FUNCTION_NAME, originalColumnName, distinct);
|
||||
AggregationSignature countSignature = new AggregationSignature(COUNT.getName(), originalColumnName, distinct);
|
||||
String countColumnName = cubeMetadata.getColumn(countSignature)
|
||||
.orElseThrow(() -> new PrestoException(CUBE_ERROR, "Cannot find column associated with aggregation " + countSignature));
|
||||
ColumnHandle countColumnHandle = cubeColumnsMap.get(countColumnName);
|
||||
|
|
|
|||
|
|
@ -764,9 +764,7 @@ public class PruneUnreferencedOutputs
|
|||
node.getId(),
|
||||
source,
|
||||
node.getRowCountSymbol(),
|
||||
node.getCubeName(),
|
||||
node.getDataPredicate(),
|
||||
node.isOverwrite());
|
||||
node.getMetadata());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -275,13 +275,17 @@ public class StarTreeAggregationRule
|
|||
}
|
||||
|
||||
LongSupplier lastModifiedTimeSupplier = metadata.getTableLastModifiedTimeSupplier(session, tableHandle);
|
||||
if (lastModifiedTimeSupplier != null) {
|
||||
long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
|
||||
matchedCubeMetadataList = matchedCubeMetadataList.stream()
|
||||
.filter(cubeMetadata -> cubeMetadata.getLastUpdated() > lastModifiedTime)
|
||||
.collect(Collectors.toList());
|
||||
if (lastModifiedTimeSupplier == null) {
|
||||
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, "Unable to identify last modified time of " + tableName + ". Ignoring star tree cubes."));
|
||||
return Result.empty();
|
||||
}
|
||||
|
||||
//Filter out cubes that were created before the source table was updated
|
||||
long lastModifiedTime = lastModifiedTimeSupplier.getAsLong();
|
||||
matchedCubeMetadataList = matchedCubeMetadataList.stream()
|
||||
.filter(cubeMetadata -> cubeMetadata.getSourceTableLastUpdatedTime() >= lastModifiedTime)
|
||||
.collect(Collectors.toList());
|
||||
|
||||
if (matchedCubeMetadataList.isEmpty()) {
|
||||
warningCollector.add(new PrestoWarning(EXPIRED_CUBE, tableName + " has been modified after creating cubes. Ignoring expired cubes."));
|
||||
return Result.empty();
|
||||
|
|
@ -289,7 +293,7 @@ public class StarTreeAggregationRule
|
|||
|
||||
//If multiple cubes are matching then lets select the recent built cube
|
||||
//so sort the cube based on the last updated time stamp
|
||||
matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdated).reversed());
|
||||
matchedCubeMetadataList.sort(Comparator.comparingLong(CubeMetadata::getLastUpdatedTime).reversed());
|
||||
|
||||
AggregationRewriteWithCube aggregationRewriteWithCube = new AggregationRewriteWithCube(metadata, session, symbolAllocator, idAllocator, symbolMapping, matchedCubeMetadataList.get(0));
|
||||
return Result.ofPlanNode(aggregationRewriteWithCube.rewrite(aggregationNode, filterNode.orElse(null)));
|
||||
|
|
|
|||
|
|
@ -253,9 +253,7 @@ public class SymbolMapper
|
|||
node.getId(),
|
||||
source,
|
||||
map(node.getRowCountSymbol()),
|
||||
node.getCubeName(),
|
||||
node.getDataPredicate(),
|
||||
node.isOverwrite());
|
||||
node.getMetadata());
|
||||
}
|
||||
|
||||
private PartitioningScheme canonicalize(PartitioningScheme scheme, PlanNode source)
|
||||
|
|
|
|||
|
|
@ -17,42 +17,34 @@ import com.fasterxml.jackson.annotation.JsonCreator;
|
|||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
import io.prestosql.spi.cube.CubeUpdateMetadata;
|
||||
import io.prestosql.spi.plan.PlanNode;
|
||||
import io.prestosql.spi.plan.PlanNodeId;
|
||||
import io.prestosql.spi.plan.Symbol;
|
||||
import io.prestosql.sql.tree.Expression;
|
||||
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
@Immutable
|
||||
public class CubeFinishNode
|
||||
extends InternalPlanNode
|
||||
{
|
||||
private final PlanNode source;
|
||||
private final String cubeName;
|
||||
private final Symbol rowCountSymbol;
|
||||
private final Expression dataPredicate;
|
||||
private final boolean overwrite;
|
||||
private final CubeUpdateMetadata metadata;
|
||||
|
||||
@JsonCreator
|
||||
public CubeFinishNode(
|
||||
@JsonProperty("id") PlanNodeId id,
|
||||
@JsonProperty("source") PlanNode source,
|
||||
@JsonProperty("rowCountSymbol") Symbol rowCountSymbol,
|
||||
@JsonProperty("cubeName") String cubeName,
|
||||
@JsonProperty("dataPredicate") Expression dataPredicate,
|
||||
@JsonProperty("overwrite") boolean overwrite)
|
||||
@JsonProperty("metadata") CubeUpdateMetadata metadata)
|
||||
{
|
||||
super(id);
|
||||
this.source = requireNonNull(source, "source is null");
|
||||
this.cubeName = requireNonNull(cubeName, "Cube name is null");
|
||||
this.rowCountSymbol = requireNonNull(rowCountSymbol, "rowCountSymbol is null");
|
||||
this.dataPredicate = requireNonNull(dataPredicate, "Predicate is null");
|
||||
this.overwrite = overwrite;
|
||||
this.source = source;
|
||||
this.rowCountSymbol = rowCountSymbol;
|
||||
this.metadata = metadata;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
|
|
@ -68,21 +60,9 @@ public class CubeFinishNode
|
|||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getCubeName()
|
||||
public CubeUpdateMetadata getMetadata()
|
||||
{
|
||||
return cubeName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public Expression getDataPredicate()
|
||||
{
|
||||
return dataPredicate;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isOverwrite()
|
||||
{
|
||||
return overwrite;
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -110,8 +90,6 @@ public class CubeFinishNode
|
|||
getId(),
|
||||
Iterables.getOnlyElement(newChildren),
|
||||
rowCountSymbol,
|
||||
cubeName,
|
||||
dataPredicate,
|
||||
overwrite);
|
||||
metadata);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -536,7 +536,7 @@ public class IoPlanPrinter
|
|||
@Override
|
||||
public Void visitCubeFinish(CubeFinishNode node, IoPlanBuilder context)
|
||||
{
|
||||
QualifiedObjectName qualifiedObjectName = QualifiedObjectName.valueOf(node.getCubeName());
|
||||
QualifiedObjectName qualifiedObjectName = QualifiedObjectName.valueOf(node.getMetadata().getCubeName());
|
||||
context.setOutputTable(new CatalogSchemaTableName(
|
||||
qualifiedObjectName.getCatalogName(),
|
||||
qualifiedObjectName.getSchemaName(),
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import io.prestosql.security.AccessControl;
|
|||
import io.prestosql.spi.PrestoException;
|
||||
import io.prestosql.spi.StandardErrorCode;
|
||||
import io.prestosql.spi.connector.QualifiedObjectName;
|
||||
import io.prestosql.sql.ExpressionFormatter;
|
||||
import io.prestosql.sql.analyzer.QueryExplainer;
|
||||
import io.prestosql.sql.parser.SqlParser;
|
||||
import io.prestosql.sql.tree.AstVisitor;
|
||||
|
|
@ -101,23 +102,28 @@ public class InsertCubeRewrite
|
|||
CubeMetadata cubeMetadata = cubeMetaStore.getMetadataFromCubeName(targetCube.toString()).orElseThrow(() -> new PrestoException(StandardErrorCode.CUBE_ERROR, String.format("Cube not found '%s'", targetCube.toString())));
|
||||
Set<String> group = cubeMetadata.getGroup();
|
||||
ImmutableList.Builder<Identifier> builder = ImmutableList.builder();
|
||||
new IdentifierBuilderVisitor().process(node.getWhere(), builder);
|
||||
Set<String> whereColumns = builder.build()
|
||||
.stream()
|
||||
.map(Identifier::getValue)
|
||||
.collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER)));
|
||||
if (!group.containsAll(whereColumns)) {
|
||||
throw new IllegalArgumentException("All columns in where clause must be part Cube group.");
|
||||
if (node.getWhere().isPresent()) {
|
||||
new IdentifierBuilderVisitor().process(node.getWhere().get(), builder);
|
||||
Set<String> whereColumns = builder.build()
|
||||
.stream()
|
||||
.map(Identifier::getValue)
|
||||
.collect(Collectors.toCollection(() -> new TreeSet<>(String.CASE_INSENSITIVE_ORDER)));
|
||||
if (whereColumns.isEmpty()) {
|
||||
throw new IllegalArgumentException("Invalid predicate. " + ExpressionFormatter.formatExpression(node.getWhere().get(), Optional.empty()));
|
||||
}
|
||||
if (!group.containsAll(whereColumns)) {
|
||||
throw new IllegalArgumentException("All columns in where clause must be part Cube group.");
|
||||
}
|
||||
}
|
||||
return buildCubeInsert(cubeMetadata, node, group);
|
||||
}
|
||||
|
||||
private InsertCube buildCubeInsert(CubeMetadata cubeMetadata, InsertCube node, Set<String> cubeGroup)
|
||||
{
|
||||
Expression newDataPredicate = node.getWhere();
|
||||
QualifiedObjectName originalTableName = QualifiedObjectName.valueOf(cubeMetadata.getOriginalTableName());
|
||||
Optional<Expression> newDataPredicate = node.getWhere();
|
||||
QualifiedObjectName sourceTableName = QualifiedObjectName.valueOf(cubeMetadata.getSourceTableName());
|
||||
List<Identifier> insertColumns = new ArrayList<>();
|
||||
QualifiedName sourceTable = QualifiedName.of(originalTableName.getCatalogName(), originalTableName.getSchemaName(), originalTableName.getObjectName());
|
||||
QualifiedName sourceTable = QualifiedName.of(sourceTableName.getCatalogName(), sourceTableName.getSchemaName(), sourceTableName.getObjectName());
|
||||
List<SelectItem> selectItems = new ArrayList<>();
|
||||
cubeMetadata.getAggregations().forEach(aggColumn -> {
|
||||
AggregationSignature aggregationSignature = cubeMetadata.getAggregationSignature(aggColumn).get();
|
||||
|
|
@ -152,7 +158,7 @@ public class InsertCubeRewrite
|
|||
QuerySpecification selectQuery = new QuerySpecification(
|
||||
new Select(false, selectItems),
|
||||
Optional.of(new Table(sourceTable)),
|
||||
Optional.of(newDataPredicate),
|
||||
newDataPredicate,
|
||||
Optional.of(groupBy),
|
||||
Optional.empty(),
|
||||
Optional.empty(),
|
||||
|
|
|
|||
|
|
@ -271,11 +271,15 @@ final class ShowQueriesRewrite
|
|||
}
|
||||
else {
|
||||
QualifiedObjectName qualifiedTableName = createQualifiedObjectName(session, node, node.getTableName().get());
|
||||
Optional<TableHandle> tableHandle = metadata.getTableHandle(session, qualifiedTableName);
|
||||
if (!tableHandle.isPresent()) {
|
||||
throw new SemanticException(MISSING_TABLE, node, "Table %s does not exist", qualifiedTableName.toString());
|
||||
}
|
||||
cubeMetadataList = cubeMetaStore.getMetadataList(qualifiedTableName.toString());
|
||||
}
|
||||
Map<String, String> cubeStatusMap = new HashMap<>();
|
||||
cubeMetadataList.forEach(cubeMetadata -> {
|
||||
QualifiedObjectName qualifiedTableName = QualifiedObjectName.valueOf(cubeMetadata.getOriginalTableName());
|
||||
QualifiedObjectName qualifiedTableName = QualifiedObjectName.valueOf(cubeMetadata.getSourceTableName());
|
||||
Map<QualifiedObjectName, Long> tableLastModifiedTimeMap = new HashMap<>();
|
||||
long tableLastModifiedTime = tableLastModifiedTimeMap.computeIfAbsent(qualifiedTableName, ignored -> {
|
||||
TableHandle tableHandle = metadata.getTableHandle(session, qualifiedTableName).get();
|
||||
|
|
@ -284,10 +288,10 @@ final class ShowQueriesRewrite
|
|||
});
|
||||
CubeStatus status = cubeMetadata.getCubeStatus();
|
||||
if (status == CubeStatus.INACTIVE) {
|
||||
cubeStatusMap.put(cubeMetadata.getCubeTableName(), "InActive");
|
||||
cubeStatusMap.put(cubeMetadata.getCubeName(), "Inactive");
|
||||
}
|
||||
else {
|
||||
cubeStatusMap.put(cubeMetadata.getCubeTableName(), tableLastModifiedTime > cubeMetadata.getLastUpdated() ? "Expired" : "Active");
|
||||
cubeStatusMap.put(cubeMetadata.getCubeName(), tableLastModifiedTime > cubeMetadata.getSourceTableLastUpdatedTime() ? "Expired" : "Active");
|
||||
}
|
||||
});
|
||||
rows.add(row(
|
||||
|
|
@ -300,9 +304,9 @@ final class ShowQueriesRewrite
|
|||
FALSE_LITERAL));
|
||||
cubeMetadataList.forEach(cubeMetadata -> {
|
||||
rows.add(row(
|
||||
new StringLiteral(cubeMetadata.getCubeTableName()),
|
||||
new StringLiteral(cubeMetadata.getOriginalTableName()),
|
||||
new StringLiteral(cubeStatusMap.get(cubeMetadata.getCubeTableName())),
|
||||
new StringLiteral(cubeMetadata.getCubeName()),
|
||||
new StringLiteral(cubeMetadata.getSourceTableName()),
|
||||
new StringLiteral(cubeStatusMap.get(cubeMetadata.getCubeName())),
|
||||
new StringLiteral(String.join(",", cubeMetadata.getDimensions())),
|
||||
new StringLiteral(cubeMetadata.getAggregationSignatures().stream().map(AggregationSignature::toString).collect(Collectors.joining(","))),
|
||||
new StringLiteral(String.join(",", cubeMetadata.getPredicateString())),
|
||||
|
|
|
|||
|
|
@ -747,7 +747,7 @@ public class TestStarTreeAggregationRule
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testDoNotUseCubeIfOriginalTableUpdatedAfterCubeCreated()
|
||||
public void testDoNotUseCubeIfSourceTableUpdatedAfterCubeCreated()
|
||||
{
|
||||
Mockito.when(cubeManager.getCubeProvider(anyString())).then(new Returns(Optional.of(provider)));
|
||||
Mockito.when(cubeManager.getMetaStore(anyString())).then(new Returns(Optional.of(cubeMetaStore)));
|
||||
|
|
@ -764,7 +764,7 @@ public class TestStarTreeAggregationRule
|
|||
List<CubeMetadata> metadataList = ImmutableList.of(cubeMetadata);
|
||||
Mockito.when(cubeMetaStore.getMetadataList(eq("local.sf1.0.orders"))).then(new Returns(metadataList));
|
||||
Mockito.when(cubeMetadata.matches(any(CubeStatement.class))).thenReturn(true);
|
||||
Mockito.when(cubeMetadata.getLastUpdated()).thenReturn(DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-01 12:00:00"));
|
||||
Mockito.when(cubeMetadata.getLastUpdatedTime()).thenReturn(DateTimeUtils.parseTimestampWithoutTimeZone("2020-01-01 12:00:00"));
|
||||
|
||||
StarTreeAggregationRule starTreeAggregationRule = new StarTreeAggregationRule(cubeManager, metadata);
|
||||
tester().assertThat(starTreeAggregationRule)
|
||||
|
|
|
|||
|
|
@ -61,8 +61,8 @@ statement
|
|||
| CREATE CUBE (IF NOT EXISTS)? cubeName=qualifiedName
|
||||
ON tableName=qualifiedName
|
||||
WITH '(' AGGREGATIONS EQ '(' aggregations ')' ',' GROUP EQ '(' cubeGroup ')' (',' cubeProperties)? ')' #createCube
|
||||
| INSERT INTO CUBE cubeName=qualifiedName WHERE expression #insertCube
|
||||
| INSERT OVERWRITE CUBE cubeName=qualifiedName WHERE expression #insertOverwriteCube
|
||||
| INSERT INTO CUBE cubeName=qualifiedName (WHERE expression)? #insertCube
|
||||
| INSERT OVERWRITE CUBE cubeName=qualifiedName (WHERE expression)? #insertOverwriteCube
|
||||
| DROP CUBE (IF EXISTS)? cubeName=qualifiedName #dropCube
|
||||
| SHOW CUBES (FOR tableName=qualifiedName)? #showCubes
|
||||
| CREATE INDEX (IF NOT EXISTS)? indexName=qualifiedName
|
||||
|
|
|
|||
|
|
@ -63,6 +63,7 @@ import io.prestosql.sql.tree.GrantRoles;
|
|||
import io.prestosql.sql.tree.GrantorSpecification;
|
||||
import io.prestosql.sql.tree.Identifier;
|
||||
import io.prestosql.sql.tree.Insert;
|
||||
import io.prestosql.sql.tree.InsertCube;
|
||||
import io.prestosql.sql.tree.Intersect;
|
||||
import io.prestosql.sql.tree.Isolation;
|
||||
import io.prestosql.sql.tree.Join;
|
||||
|
|
@ -979,19 +980,22 @@ public final class SqlFormatter
|
|||
}
|
||||
builder.append(formatName(node.getCubeName()));
|
||||
builder.append(" ON ");
|
||||
builder.append(formatName(node.getTableName()));
|
||||
builder.append(" WITH ");
|
||||
List<String> aggregations = node.getAggregations().stream().map(Expression::toString).collect(Collectors.toList());
|
||||
String propertyList = node.getProperties().stream()
|
||||
.map(element -> formatExpression(element.getName(), parameters) + " = " +
|
||||
formatExpression(element.getValue(), parameters))
|
||||
builder.append(formatName(node.getSourceTableName()));
|
||||
builder.append(" WITH (");
|
||||
String aggregations = node.getAggregations().stream()
|
||||
.map(Expression::toString)
|
||||
.collect(joining(", "));
|
||||
String groupsList = node.getGroupingSet().stream()
|
||||
.map(Identifier::toString)
|
||||
String group = node.getGroupingSet().stream()
|
||||
.map(Identifier::getValue)
|
||||
.collect(joining(", "));
|
||||
builder.append(" ( AGGREGATIONS = (").append(String.join(", ", aggregations)).append(")");
|
||||
builder.append(", GROUP=(").append(groupsList).append(")");
|
||||
builder.append(", PROPERTIES = (").append(propertyList).append(")");
|
||||
builder.append("AGGREGATIONS = (").append(aggregations).append("), ");
|
||||
builder.append("GROUP=(").append(group).append(")");
|
||||
if (!node.getProperties().isEmpty()) {
|
||||
String properties = node.getProperties().stream()
|
||||
.map(element -> formatExpression(element.getName(), parameters) + " = " + formatExpression(element.getValue(), parameters))
|
||||
.collect(joining(", "));
|
||||
builder.append(", ").append(properties);
|
||||
}
|
||||
builder.append(" )");
|
||||
return null;
|
||||
}
|
||||
|
|
@ -1298,6 +1302,24 @@ public final class SqlFormatter
|
|||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void visitInsertCube(InsertCube node, Integer indent)
|
||||
{
|
||||
if (node.isOverwrite()) {
|
||||
builder.append("INSERT OVERWRITE CUBE ")
|
||||
.append(node.getCubeName());
|
||||
}
|
||||
else {
|
||||
builder.append("INSERT INTO CUBE ")
|
||||
.append(node.getCubeName());
|
||||
}
|
||||
if (node.getWhere().isPresent()) {
|
||||
builder.append(" WHERE ")
|
||||
.append(formatExpression(node.getWhere().get(), Optional.empty()));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Void visitUpdate(Update node, Integer indent)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ import io.prestosql.sql.tree.CurrentTime;
|
|||
import io.prestosql.sql.tree.CurrentUser;
|
||||
import io.prestosql.sql.tree.Deallocate;
|
||||
import io.prestosql.sql.tree.DecimalLiteral;
|
||||
import io.prestosql.sql.tree.DefaultExpressionTraversalVisitor;
|
||||
import io.prestosql.sql.tree.Delete;
|
||||
import io.prestosql.sql.tree.DereferenceExpression;
|
||||
import io.prestosql.sql.tree.DescribeInput;
|
||||
|
|
@ -332,14 +331,14 @@ class AstBuilder
|
|||
});
|
||||
|
||||
QualifiedName cubeName = getQualifiedName(context.cubeName);
|
||||
QualifiedName originalTableName = getQualifiedName(context.tableName);
|
||||
QualifiedName sourceTableName = getQualifiedName(context.tableName);
|
||||
|
||||
List<Property> properties = ImmutableList.of();
|
||||
if (context.cubeProperties() != null) {
|
||||
properties = visit(context.cubeProperties().property(), Property.class);
|
||||
}
|
||||
|
||||
return new CreateCube(getLocation(context), cubeName, originalTableName, groupingSet, decomposedAggregations, context.EXISTS() != null, properties);
|
||||
return new CreateCube(getLocation(context), cubeName, sourceTableName, groupingSet, decomposedAggregations, context.EXISTS() != null, properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -347,10 +346,7 @@ class AstBuilder
|
|||
{
|
||||
QualifiedName cubeName = getQualifiedName(context.qualifiedName());
|
||||
Optional<Expression> optionalExpression = visitIfPresent(context.expression(), Expression.class);
|
||||
if (!optionalExpression.isPresent()) {
|
||||
throw new IllegalArgumentException("WHERE expression is mandatory!");
|
||||
}
|
||||
return new InsertCube(getLocation(context), cubeName, optionalExpression.get(), null, false);
|
||||
return new InsertCube(getLocation(context), cubeName, optionalExpression, false);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -358,21 +354,7 @@ class AstBuilder
|
|||
{
|
||||
QualifiedName cubeName = getQualifiedName(context.qualifiedName());
|
||||
Optional<Expression> optionalExpression = visitIfPresent(context.expression(), Expression.class);
|
||||
if (!optionalExpression.isPresent()) {
|
||||
throw new IllegalArgumentException("WHERE expression is mandatory!");
|
||||
}
|
||||
return new InsertCube(getLocation(context), cubeName, optionalExpression.get(), null, true);
|
||||
}
|
||||
|
||||
private static class IdentifierBuilderVisitor
|
||||
extends DefaultExpressionTraversalVisitor<Void, ImmutableList.Builder<Identifier>>
|
||||
{
|
||||
@Override
|
||||
protected Void visitIdentifier(Identifier node, ImmutableList.Builder<Identifier> builder)
|
||||
{
|
||||
builder.add(node);
|
||||
return null;
|
||||
}
|
||||
return new InsertCube(getLocation(context), cubeName, optionalExpression, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -30,29 +30,29 @@ public class CreateCube
|
|||
{
|
||||
private final QualifiedName cubeName;
|
||||
private final boolean notExists;
|
||||
private final QualifiedName tableName;
|
||||
private final QualifiedName sourceTableName;
|
||||
private final List<Identifier> groupingSet;
|
||||
private final Set<FunctionCall> aggregations;
|
||||
private final List<Property> properties;
|
||||
|
||||
public CreateCube(QualifiedName cubeName, QualifiedName tableName, List<Identifier> groupingSet,
|
||||
public CreateCube(QualifiedName cubeName, QualifiedName sourceTableName, List<Identifier> groupingSet,
|
||||
Set<FunctionCall> aggregations, boolean notExists, List<Property> properties)
|
||||
{
|
||||
this(Optional.empty(), cubeName, tableName, groupingSet, aggregations, notExists, properties);
|
||||
this(Optional.empty(), cubeName, sourceTableName, groupingSet, aggregations, notExists, properties);
|
||||
}
|
||||
|
||||
public CreateCube(NodeLocation location, QualifiedName cubeName, QualifiedName tableName, List<Identifier> groupingSet,
|
||||
public CreateCube(NodeLocation location, QualifiedName cubeName, QualifiedName sourceTableName, List<Identifier> groupingSet,
|
||||
Set<FunctionCall> aggregations, boolean notExists, List<Property> properties)
|
||||
{
|
||||
this(Optional.of(location), cubeName, tableName, groupingSet, aggregations, notExists, properties);
|
||||
this(Optional.of(location), cubeName, sourceTableName, groupingSet, aggregations, notExists, properties);
|
||||
}
|
||||
|
||||
private CreateCube(Optional<NodeLocation> location, QualifiedName cubeName, QualifiedName tableName, List<Identifier> groupingSet,
|
||||
private CreateCube(Optional<NodeLocation> location, QualifiedName cubeName, QualifiedName sourceTableName, List<Identifier> groupingSet,
|
||||
Set<FunctionCall> aggregations, boolean notExists, List<Property> properties)
|
||||
{
|
||||
super(location);
|
||||
this.cubeName = requireNonNull(cubeName, "cube name is null");
|
||||
this.tableName = requireNonNull(tableName, "table name is null");
|
||||
this.sourceTableName = requireNonNull(sourceTableName, "table name is null");
|
||||
this.groupingSet = groupingSet;
|
||||
this.aggregations = aggregations;
|
||||
this.notExists = notExists;
|
||||
|
|
@ -64,9 +64,9 @@ public class CreateCube
|
|||
return cubeName;
|
||||
}
|
||||
|
||||
public QualifiedName getTableName()
|
||||
public QualifiedName getSourceTableName()
|
||||
{
|
||||
return tableName;
|
||||
return sourceTableName;
|
||||
}
|
||||
|
||||
public List<Property> getProperties()
|
||||
|
|
@ -109,7 +109,7 @@ public class CreateCube
|
|||
{
|
||||
return Objects.hash(
|
||||
cubeName,
|
||||
tableName,
|
||||
sourceTableName,
|
||||
groupingSet,
|
||||
aggregations,
|
||||
notExists,
|
||||
|
|
@ -121,7 +121,7 @@ public class CreateCube
|
|||
{
|
||||
return toStringHelper(this)
|
||||
.add("cubeName", cubeName)
|
||||
.add("tableName", tableName)
|
||||
.add("tableName", sourceTableName)
|
||||
.add("groupingSet", groupingSet)
|
||||
.add("aggregations", aggregations)
|
||||
.add("notExists", notExists)
|
||||
|
|
@ -140,7 +140,7 @@ public class CreateCube
|
|||
}
|
||||
CreateCube that = (CreateCube) o;
|
||||
return Objects.equals(cubeName, that.cubeName) &&
|
||||
Objects.equals(tableName, that.tableName) &&
|
||||
Objects.equals(sourceTableName, that.sourceTableName) &&
|
||||
Objects.equals(groupingSet, that.groupingSet) &&
|
||||
Objects.equals(aggregations, that.aggregations) &&
|
||||
Objects.equals(notExists, that.notExists) &&
|
||||
|
|
|
|||
|
|
@ -25,22 +25,32 @@ public class InsertCube
|
|||
extends Statement
|
||||
{
|
||||
private final QualifiedName cubeName;
|
||||
private final Expression where;
|
||||
private final Optional<Expression> where;
|
||||
private final List<Identifier> columns;
|
||||
private final Query query;
|
||||
private final boolean overwrite;
|
||||
|
||||
public InsertCube(QualifiedName cubeName, Expression where, List<Identifier> columns, boolean overwrite)
|
||||
public InsertCube(QualifiedName cubeName, Optional<Expression> where, boolean overwrite)
|
||||
{
|
||||
this(cubeName, where, null, overwrite);
|
||||
}
|
||||
|
||||
public InsertCube(NodeLocation location, QualifiedName cubeName, Optional<Expression> where, boolean overwrite)
|
||||
{
|
||||
this(location, cubeName, where, null, overwrite);
|
||||
}
|
||||
|
||||
public InsertCube(QualifiedName cubeName, Optional<Expression> where, List<Identifier> columns, boolean overwrite)
|
||||
{
|
||||
this(cubeName, where, columns, overwrite, null);
|
||||
}
|
||||
|
||||
public InsertCube(NodeLocation location, QualifiedName cubeName, Expression where, List<Identifier> columns, boolean overwrite)
|
||||
public InsertCube(NodeLocation location, QualifiedName cubeName, Optional<Expression> where, List<Identifier> columns, boolean overwrite)
|
||||
{
|
||||
this(location, cubeName, where, columns, overwrite, null);
|
||||
}
|
||||
|
||||
public InsertCube(QualifiedName cubeName, Expression where, List<Identifier> columns, boolean overwrite, Query query)
|
||||
public InsertCube(QualifiedName cubeName, Optional<Expression> where, List<Identifier> columns, boolean overwrite, Query query)
|
||||
{
|
||||
super(Optional.empty());
|
||||
this.cubeName = cubeName;
|
||||
|
|
@ -50,7 +60,7 @@ public class InsertCube
|
|||
this.query = query;
|
||||
}
|
||||
|
||||
public InsertCube(NodeLocation location, QualifiedName cubeName, Expression where, List<Identifier> columns, boolean overwrite, Query query)
|
||||
public InsertCube(NodeLocation location, QualifiedName cubeName, Optional<Expression> where, List<Identifier> columns, boolean overwrite, Query query)
|
||||
{
|
||||
super(Optional.of(location));
|
||||
this.cubeName = cubeName;
|
||||
|
|
@ -65,7 +75,7 @@ public class InsertCube
|
|||
return cubeName;
|
||||
}
|
||||
|
||||
public Expression getWhere()
|
||||
public Optional<Expression> getWhere()
|
||||
{
|
||||
return where;
|
||||
}
|
||||
|
|
@ -126,6 +136,7 @@ public class InsertCube
|
|||
}
|
||||
InsertCube o = (InsertCube) obj;
|
||||
return Objects.equals(cubeName, o.cubeName) &&
|
||||
Objects.equals(where, o.where);
|
||||
Objects.equals(where, o.where) &&
|
||||
Objects.equals(overwrite, o.overwrite);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import io.prestosql.sql.tree.ColumnDefinition;
|
|||
import io.prestosql.sql.tree.Comment;
|
||||
import io.prestosql.sql.tree.Commit;
|
||||
import io.prestosql.sql.tree.ComparisonExpression;
|
||||
import io.prestosql.sql.tree.CreateCube;
|
||||
import io.prestosql.sql.tree.CreateRole;
|
||||
import io.prestosql.sql.tree.CreateSchema;
|
||||
import io.prestosql.sql.tree.CreateTable;
|
||||
|
|
@ -76,6 +77,7 @@ import io.prestosql.sql.tree.GroupingSets;
|
|||
import io.prestosql.sql.tree.Identifier;
|
||||
import io.prestosql.sql.tree.IfExpression;
|
||||
import io.prestosql.sql.tree.Insert;
|
||||
import io.prestosql.sql.tree.InsertCube;
|
||||
import io.prestosql.sql.tree.Intersect;
|
||||
import io.prestosql.sql.tree.IntervalLiteral;
|
||||
import io.prestosql.sql.tree.IntervalLiteral.IntervalField;
|
||||
|
|
@ -1384,38 +1386,6 @@ public class TestSqlParser
|
|||
new Identifier("b")))));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateCube()
|
||||
{
|
||||
// assertStatement("CREATE CUBE foo ON bar WITH (DIMENSIONS=key,val, AGGREGATIONS=count(\"*\"))",
|
||||
// new CreateCube(QualifiedName.of("foo"),
|
||||
// QualifiedName.of("bar"),
|
||||
// ImmutableList.of(
|
||||
// new Identifier("key"),
|
||||
// new Identifier("val")),
|
||||
// false,
|
||||
// ImmutableList.of(new FunctionCall(QualifiedName.of("COUNT"), ImmutableList.of(new Identifier("*")))),
|
||||
// Optional.empty()));
|
||||
// assertStatement("CREATE CUBE foo ON bar WITH (DIMENSIONS=key,val, AGGREGATIONS=sum(cost))",
|
||||
// new CreateCube(QualifiedName.of("foo"),
|
||||
// QualifiedName.of("bar"),
|
||||
// ImmutableList.of(
|
||||
// new Identifier("key"),
|
||||
// new Identifier("val")),
|
||||
// false,
|
||||
// ImmutableList.of(new FunctionCall(QualifiedName.of("SUM"), ImmutableList.of(new Identifier("cost")))),
|
||||
// Optional.empty()));
|
||||
// assertStatement("CREATE CUBE IF NOT EXISTS foo ON bar WITH (DIMENSIONS=key,val, AGGREGATIONS=sum(cost))",
|
||||
// new CreateCube(QualifiedName.of("foo"),
|
||||
// QualifiedName.of("bar"),
|
||||
// ImmutableList.of(
|
||||
// new Identifier("key"),
|
||||
// new Identifier("val")),
|
||||
// true,
|
||||
// ImmutableList.of(new FunctionCall(QualifiedName.of("SUM"), ImmutableList.of(new Identifier("cost")))),
|
||||
// Optional.empty()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateTable()
|
||||
{
|
||||
|
|
@ -1661,6 +1631,68 @@ public class TestSqlParser
|
|||
assertStatement(queryUnparenthesizedWithHasAlias, new CreateTableAsSelect(table, query, false, ImmutableList.of(), false, Optional.of(ImmutableList.of(new Identifier("a"))), Optional.empty()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateCube()
|
||||
{
|
||||
assertStatement("CREATE CUBE foo ON bar WITH (AGGREGATIONS=(count(c)), GROUP = (a, b))",
|
||||
new CreateCube(QualifiedName.of("foo"),
|
||||
QualifiedName.of("bar"),
|
||||
ImmutableList.of(
|
||||
new Identifier("a"),
|
||||
new Identifier("b")),
|
||||
ImmutableSet.of(
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("c")))),
|
||||
false,
|
||||
ImmutableList.of()));
|
||||
|
||||
assertStatement("CREATE CUBE foo ON bar WITH (AGGREGATIONS=(count(c), sum(d), avg(e)), GROUP = (a, b))",
|
||||
new CreateCube(QualifiedName.of("foo"),
|
||||
QualifiedName.of("bar"),
|
||||
ImmutableList.of(
|
||||
new Identifier("a"),
|
||||
new Identifier("b")),
|
||||
ImmutableSet.of(
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("c"))),
|
||||
new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(new Identifier("d"))),
|
||||
new FunctionCall(QualifiedName.of("sum"), ImmutableList.of(new Identifier("e"))),
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("e")))),
|
||||
false,
|
||||
ImmutableList.of()));
|
||||
|
||||
assertStatement("CREATE CUBE c1.s1.foo ON c2.s2.bar WITH (AGGREGATIONS=(count(c)), GROUP = (a, b))",
|
||||
new CreateCube(QualifiedName.of("c1", "s1", "foo"),
|
||||
QualifiedName.of("c2", "s2", "bar"),
|
||||
ImmutableList.of(
|
||||
new Identifier("a"),
|
||||
new Identifier("b")),
|
||||
ImmutableSet.of(
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("c")))),
|
||||
false,
|
||||
ImmutableList.of()));
|
||||
|
||||
assertStatement("CREATE CUBE IF NOT EXISTS foo ON bar WITH (AGGREGATIONS=(count(c)), GROUP = (a, b))",
|
||||
new CreateCube(QualifiedName.of("foo"),
|
||||
QualifiedName.of("bar"),
|
||||
ImmutableList.of(
|
||||
new Identifier("a"),
|
||||
new Identifier("b")),
|
||||
ImmutableSet.of(
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("c")))),
|
||||
true,
|
||||
ImmutableList.of()));
|
||||
|
||||
assertStatement("CREATE CUBE IF NOT EXISTS foo ON bar WITH (AGGREGATIONS=(count(c)), GROUP = (a, b), format = 'ORC', partitioned_by = ARRAY[ 'd' ])",
|
||||
new CreateCube(QualifiedName.of("foo"),
|
||||
QualifiedName.of("bar"),
|
||||
ImmutableList.of(
|
||||
new Identifier("a"),
|
||||
new Identifier("b")),
|
||||
ImmutableSet.of(
|
||||
new FunctionCall(QualifiedName.of("count"), ImmutableList.of(new Identifier("c")))),
|
||||
true,
|
||||
ImmutableList.of(new Property(new Identifier("format"), new StringLiteral("ORC")), new Property(new Identifier("partitioned_by"), new ArrayConstructor(ImmutableList.of(new StringLiteral("d")))))));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDropCache()
|
||||
{
|
||||
|
|
@ -1735,6 +1767,32 @@ public class TestSqlParser
|
|||
new Insert(table, Optional.of(ImmutableList.of(identifier("c1"), identifier("c2"))), query, true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsertIntoCube()
|
||||
{
|
||||
assertStatement("INSERT INTO CUBE foo WHERE d1 > 10",
|
||||
new InsertCube(QualifiedName.of("foo"),
|
||||
Optional.of(new ComparisonExpression(GREATER_THAN, new Identifier("d1"), new LongLiteral("10"))),
|
||||
false));
|
||||
assertStatement("INSERT INTO CUBE c1.s1.foo WHERE d1 > 10",
|
||||
new InsertCube(QualifiedName.of("c1", "s1", "foo"),
|
||||
Optional.of(new ComparisonExpression(GREATER_THAN, new Identifier("d1"), new LongLiteral("10"))),
|
||||
false));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsertOverwriteCube()
|
||||
{
|
||||
assertStatement("INSERT OVERWRITE CUBE foo WHERE d1 BETWEEN 1012020 AND 31012020",
|
||||
new InsertCube(QualifiedName.of("foo"),
|
||||
Optional.of(new BetweenPredicate(new Identifier("d1"), new LongLiteral("1012020"), new LongLiteral("31012020"))),
|
||||
true));
|
||||
assertStatement("INSERT OVERWRITE CUBE c1.s1.foo WHERE d1 BETWEEN 1012020 AND 31012020",
|
||||
new InsertCube(QualifiedName.of("c1", "s1", "foo"),
|
||||
Optional.of(new BetweenPredicate(new Identifier("d1"), new LongLiteral("1012020"), new LongLiteral("31012020"))),
|
||||
true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDelete()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package io.prestosql.spi.cube;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
import static java.util.Objects.requireNonNull;
|
||||
|
||||
public class CubeUpdateMetadata
|
||||
{
|
||||
private final String cubeName;
|
||||
private final long tableLastUpdatedTime;
|
||||
private final String dataPredicateString;
|
||||
private final boolean overwrite;
|
||||
|
||||
@JsonCreator
|
||||
public CubeUpdateMetadata(
|
||||
@JsonProperty("cubeName") String cubeName,
|
||||
@JsonProperty("tableLastUpdatedTime") long tableLastUpdatedTime,
|
||||
@JsonProperty("dataPredicate") String dataPredicateString,
|
||||
@JsonProperty("overwrite") boolean overwrite)
|
||||
{
|
||||
this.cubeName = requireNonNull(cubeName, "cubeName is null");
|
||||
this.tableLastUpdatedTime = tableLastUpdatedTime;
|
||||
this.dataPredicateString = dataPredicateString;
|
||||
this.overwrite = overwrite;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getCubeName()
|
||||
{
|
||||
return cubeName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getTableLastUpdatedTime()
|
||||
{
|
||||
return tableLastUpdatedTime;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getDataPredicateString()
|
||||
{
|
||||
return dataPredicateString;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isOverwrite()
|
||||
{
|
||||
return overwrite;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return "CubeUpdateMetadata{" +
|
||||
"cubeName='" + cubeName + '\'' +
|
||||
", tableLastUpdatedTime=" + tableLastUpdatedTime +
|
||||
", dataPredicateString='" + dataPredicateString + '\'' +
|
||||
", overwrite=" + overwrite +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
|
@ -15,24 +15,45 @@
|
|||
|
||||
package io.prestosql.tests;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import io.prestosql.Session;
|
||||
import io.prestosql.SystemSessionProperties;
|
||||
import io.prestosql.testing.MaterializedResult;
|
||||
import io.prestosql.testing.MaterializedRow;
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static com.google.common.collect.Iterables.getOnlyElement;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
|
||||
public abstract class AbstractTestStarTreeQueries
|
||||
extends AbstractTestQueryFramework
|
||||
{
|
||||
Session sessionStarTree;
|
||||
Session sessionNoStarTree;
|
||||
|
||||
protected AbstractTestStarTreeQueries(QueryRunnerSupplier supplier)
|
||||
{
|
||||
super(supplier);
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public void setUp()
|
||||
{
|
||||
sessionStarTree = Session.builder(getSession())
|
||||
.setSystemProperty(SystemSessionProperties.ENABLE_STAR_TREE_INDEX, "true")
|
||||
.build();
|
||||
sessionNoStarTree = Session.builder(getSession())
|
||||
.setSystemProperty(SystemSessionProperties.ENABLE_STAR_TREE_INDEX, "false")
|
||||
.build();
|
||||
//Create Empty to force create Metadata catalog and schema. To avoid concurrency issue.
|
||||
assertUpdate(sessionNoStarTree, "CREATE CUBE nation_count_all ON nation WITH (AGGREGATIONS=(count(*)), group=())");
|
||||
assertUpdate("DROP CUBE nation_count_all");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testStarTreeSessionProperty()
|
||||
{
|
||||
|
|
@ -45,22 +66,134 @@ public abstract class AbstractTestStarTreeQueries
|
|||
}
|
||||
|
||||
@Test
|
||||
public void testStarTree()
|
||||
public void testAggregations()
|
||||
{
|
||||
Session sessionStarTree = Session.builder(getSession())
|
||||
.setSystemProperty(SystemSessionProperties.ENABLE_STAR_TREE_INDEX, "true")
|
||||
.build();
|
||||
Session sessionNoStarTree = Session.builder(getSession())
|
||||
.setSystemProperty(SystemSessionProperties.ENABLE_STAR_TREE_INDEX, "false")
|
||||
.build();
|
||||
assertUpdate(sessionNoStarTree, "CREATE CUBE nation_cube ON nation " +
|
||||
assertUpdate(sessionNoStarTree, "CREATE CUBE nation_aggregations_cube_1 ON nation " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
assertUpdate(sessionNoStarTree, "INSERT INTO CUBE nation_cube where nationkey > -1", 25);
|
||||
assertUpdate(sessionNoStarTree, "INSERT INTO CUBE nation_aggregations_cube_1 where nationkey > -1", 25);
|
||||
assertQueryFails(sessionNoStarTree, "INSERT INTO CUBE nation_aggregations_cube_1 where 1 > 0", "Invalid predicate\\. \\(1 > 0\\)");
|
||||
assertQuery(sessionStarTree, "SELECT min(regionkey), max(regionkey), sum(regionkey) from nation group by nationkey");
|
||||
assertQuery(sessionStarTree, "SELECT COUNT(distinct nationkey), count(distinct regionkey) from nation");
|
||||
assertQuery(sessionStarTree, "SELECT COUNT(distinct nationkey), count(distinct regionkey) from nation group by nationkey");
|
||||
assertQuery(sessionStarTree, "SELECT avg(nationkey) from nation group by nationkey");
|
||||
assertUpdate("DROP CUBE nation_cube");
|
||||
assertUpdate("DROP CUBE nation_aggregations_cube_1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testShowCubes()
|
||||
{
|
||||
assertUpdate(sessionNoStarTree, "CREATE CUBE nation_show_cube_1 ON nation " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
assertUpdate(sessionNoStarTree, "CREATE CUBE nation_show_cube_2 ON nation " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=())");
|
||||
MaterializedResult result = computeActual("SHOW CUBES");
|
||||
MaterializedRow matchingRow1 = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_show_cube_1")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow1);
|
||||
assertTrue(matchingRow1.getFields().containsAll(ImmutableList.of("hive.tpch.nation_show_cube_1", "hive.tpch.nation", "Inactive", "nationkey")));
|
||||
|
||||
MaterializedRow matchingRow2 = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_show_cube_2")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow2);
|
||||
assertTrue(matchingRow2.getFields().containsAll(ImmutableList.of("hive.tpch.nation_show_cube_2", "hive.tpch.nation", "Inactive", "")));
|
||||
|
||||
result = computeActual("SHOW CUBES FOR nation");
|
||||
assertEquals(result.getRowCount(), 2);
|
||||
|
||||
matchingRow1 = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_show_cube_1")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow1);
|
||||
assertTrue(result.getMaterializedRows().get(0).getFields().containsAll(ImmutableList.of("hive.tpch.nation_show_cube_1", "hive.tpch.nation", "Inactive", "nationkey")));
|
||||
|
||||
matchingRow2 = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_show_cube_2")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow2);
|
||||
assertTrue(result.getMaterializedRows().get(1).getFields().containsAll(ImmutableList.of("hive.tpch.nation_show_cube_2", "hive.tpch.nation", "Inactive", "")));
|
||||
assertUpdate("DROP CUBE nation_show_cube_1");
|
||||
assertUpdate("DROP CUBE nation_show_cube_2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsertIntoCube()
|
||||
{
|
||||
computeActual("CREATE TABLE nation_table_cube_insert_test_1 AS SELECT * FROM nation");
|
||||
assertUpdate("CREATE CUBE nation_insert_cube_1 ON nation_table_cube_insert_test_1 " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
assertUpdate("INSERT INTO CUBE nation_insert_cube_1 where nationkey > 5", 19);
|
||||
assertQueryFails("INSERT INTO CUBE nation where 1 > 0", "Cube not found 'hive.tpch.nation'");
|
||||
assertQueryFails("INSERT INTO CUBE nation_insert_cube_1 where regionkey > 5", "All columns in where clause must be part Cube group\\.");
|
||||
assertUpdate("DROP CUBE nation_insert_cube_1");
|
||||
assertUpdate("DROP TABLE nation_table_cube_insert_test_1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInsertOverwriteCube()
|
||||
{
|
||||
computeActual("CREATE TABLE nation_table_cube_insert_overwrite_test_1 AS SELECT * FROM nation");
|
||||
assertUpdate("CREATE CUBE nation_insert_overwrite_cube_1 ON nation_table_cube_insert_overwrite_test_1 " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
assertUpdate("INSERT INTO CUBE nation_insert_overwrite_cube_1 where nationkey > 5", 19);
|
||||
assertEquals(computeScalar("SELECT COUNT(*) FROM nation_insert_overwrite_cube_1"), 19L);
|
||||
assertUpdate("INSERT OVERWRITE CUBE nation_insert_overwrite_cube_1 where nationkey > 5", 19);
|
||||
assertEquals(computeScalar("SELECT COUNT(*) FROM nation_insert_overwrite_cube_1"), 19L);
|
||||
assertUpdate("DROP CUBE nation_insert_overwrite_cube_1");
|
||||
assertUpdate("DROP TABLE nation_table_cube_insert_overwrite_test_1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCreateCube()
|
||||
{
|
||||
computeActual("CREATE TABLE nation_table_create_cube_test_1 AS SELECT * FROM nation");
|
||||
assertQueryFails("CREATE CUBE nation ON nation " +
|
||||
"WITH (AGGREGATIONS=(count(*))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])", "line 1:1: Table 'hive.tpch.nation' already exists");
|
||||
assertQueryFails("CREATE CUBE nation_create_cube_1 ON abcd " +
|
||||
"WITH (AGGREGATIONS=(count(*), count(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])", "line 1:1: Table 'hive.tpch.abcd' does not exist");
|
||||
assertQueryFails("CREATE CUBE nation_create_cube_1 ON nation " +
|
||||
"WITH (AGGREGATIONS=(sum(distinct nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])", "line 1:1: Distinct is currently only supported for count");
|
||||
assertUpdate("CREATE CUBE nation_create_cube_1 ON nation_table_create_cube_test_1 " +
|
||||
"WITH (AGGREGATIONS=(count(*))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
assertQueryFails("CREATE CUBE nation_create_cube_1 ON nation_table_create_cube_test_1 " +
|
||||
"WITH (AGGREGATIONS=(count(*), count(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])", "line 1:1: Cube 'hive.tpch.nation_create_cube_1' already exists");
|
||||
assertUpdate("DROP CUBE nation_create_cube_1");
|
||||
assertUpdate("DROP TABLE nation_table_create_cube_test_1");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCubeStatusChange()
|
||||
{
|
||||
computeActual("CREATE TABLE nation_table_status_test AS SELECT * FROM nation");
|
||||
assertUpdate("CREATE CUBE nation_status_cube_1 ON nation_table_status_test " +
|
||||
"WITH (AGGREGATIONS=(count(*), COUNT(distinct nationkey), count(distinct regionkey), avg(nationkey), count(regionkey), sum(regionkey)," +
|
||||
" min(regionkey), max(regionkey), max(nationkey), min(nationkey))," +
|
||||
" group=(nationkey), format= 'orc', partitioned_by = ARRAY['nationkey'])");
|
||||
MaterializedResult result = computeActual("SHOW CUBES FOR nation_table_status_test");
|
||||
MaterializedRow matchingRow = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_status_cube_1")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow);
|
||||
assertEquals(matchingRow.getField(2), "Inactive");
|
||||
|
||||
assertUpdate("INSERT INTO CUBE nation_status_cube_1 where nationkey > 5", 19);
|
||||
result = computeActual("SHOW CUBES FOR nation_table_status_test");
|
||||
matchingRow = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_status_cube_1")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow);
|
||||
assertEquals(matchingRow.getField(2), "Active");
|
||||
|
||||
assertUpdate("INSERT INTO nation_table_status_test VALUES (12345, 'name', 54321, 'comment')", 1);
|
||||
result = computeActual("SHOW CUBES FOR nation_table_status_test");
|
||||
matchingRow = result.getMaterializedRows().stream().filter(row -> row.getField(0).toString().contains("nation_status_cube_1")).findFirst().orElse(null);
|
||||
assertNotNull(matchingRow);
|
||||
assertEquals(matchingRow.getField(2), "Expired");
|
||||
assertUpdate("DROP CUBE nation_status_cube_1");
|
||||
assertUpdate("DROP TABLE nation_table_status_test");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
Copyright (C) 2018-2021. Huawei Technologies Co., Ltd. All rights reserved.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
|
|
|||
Loading…
Reference in New Issue