Compare commits

...

3 Commits

Author SHA1 Message Date
lizheng920625 0ec44314ad Init commits for materialized view 2021-03-29 09:21:25 +08:00
Raghunandan c2b572e88f [maven-release-plugin] prepare for next development iteration 2020-12-30 19:29:09 +05:30
Raghunandan 35aeedf3d5 [maven-release-plugin] prepare release 1.1.0 2020-12-30 19:29:08 +05:30
144 changed files with 4389 additions and 108 deletions

View File

@ -22,7 +22,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-carbondata</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-common</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-datacenter</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-hana</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-hazelcast</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-hbase</artifactId>
@ -297,6 +297,12 @@
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-tests</artifactId>
<exclusions>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>

View File

@ -3,7 +3,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

81
hetu-mv/pom.xml Normal file
View File

@ -0,0 +1,81 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hetu-mv</artifactId>
<packaging>hetu-plugin</packaging>
<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
</properties>
<dependencies>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-spi</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>bootstrap</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>json</artifactId>
</dependency>
<dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>configuration</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>log</artifactId>
</dependency>
<dependency>
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-main</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

View File

@ -0,0 +1,102 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.materializedview;
import io.hetu.core.materializedview.conf.MaterializedViewConfig;
import io.hetu.core.materializedview.metadata.LocalMaterializedViewMetastore;
import io.hetu.core.materializedview.metadata.MaterializedView;
import io.hetu.core.materializedview.utils.MaterializedViewStatus;
import io.prestosql.spi.connector.SchemaTableName;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.File;
import java.util.List;
import java.util.Set;
import static io.hetu.core.materializedview.TestUtils.CURRENT_SCHEMA;
import static io.hetu.core.materializedview.TestUtils.DATA_SCHEMA;
import static io.hetu.core.materializedview.TestUtils.TEST_FILE;
import static io.hetu.core.materializedview.TestUtils.VIEW_NAME;
import static io.hetu.core.materializedview.TestUtils.createMaterializedView;
import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL;
import static io.hetu.core.materializedview.utils.MaterializedViewStatus.getStatusString;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
public class TestLocalMvMetastore
{
private LocalMaterializedViewMetastore localMaterializedViewMetastore;
private final String newName = "test_view";
@BeforeClass
public void setUp()
{
MaterializedViewConfig config = new MaterializedViewConfig();
config.setMetastoreType(TYPE_LOCAL);
config.setMetastorePath(TEST_FILE);
localMaterializedViewMetastore = new LocalMaterializedViewMetastore(config);
localMaterializedViewMetastore.addSchema(DATA_SCHEMA);
localMaterializedViewMetastore.addMaterializedView(DATA_SCHEMA, createMaterializedView(VIEW_NAME));
}
@Test
public void testSchema()
{
localMaterializedViewMetastore.addSchema(CURRENT_SCHEMA);
Set<String> schemas = localMaterializedViewMetastore.getSchemas();
assertEquals(schemas.size(), 2);
assertTrue(schemas.contains(CURRENT_SCHEMA));
localMaterializedViewMetastore.dropSchema(CURRENT_SCHEMA);
schemas = localMaterializedViewMetastore.getSchemas();
assertEquals(schemas.size(), 1);
assertFalse(schemas.contains(CURRENT_SCHEMA));
}
@Test
public void testMaterializedView()
{
SchemaTableName viewName = new SchemaTableName(DATA_SCHEMA, newName);
List<String> views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA);
assertEquals(views.size(), 1);
MaterializedView view = createMaterializedView(newName);
localMaterializedViewMetastore.addMaterializedView(DATA_SCHEMA, view);
views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA);
assertEquals(views.size(), 2);
assertTrue(views.contains(newName));
MaterializedView actualView = localMaterializedViewMetastore.getMaterializedView(viewName);
assertEquals(actualView.toString(), view.toString());
localMaterializedViewMetastore.setViewStatus(viewName, MaterializedViewStatus.ENABLE);
actualView = localMaterializedViewMetastore.getMaterializedView(viewName);
assertEquals(actualView.getStatus(), getStatusString(MaterializedViewStatus.ENABLE));
localMaterializedViewMetastore.dropMaterializedView(DATA_SCHEMA, newName);
views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA);
assertEquals(views.size(), 1);
assertFalse(views.contains(newName));
}
@AfterClass
public void down()
{
new File(TEST_FILE).delete();
}
}

View File

@ -0,0 +1,94 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.materializedview;
import io.hetu.core.materializedview.connector.MaterializedViewColumnHandle;
import io.hetu.core.materializedview.metadata.MaterializedView;
import io.hetu.core.materializedview.utils.MaterializedViewDateUtils;
import io.hetu.core.materializedview.utils.MaterializedViewStatus;
import io.prestosql.spi.type.TypeSignature;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import static io.hetu.core.materializedview.TestUtils.CURRENT_CATALOG;
import static io.hetu.core.materializedview.TestUtils.CURRENT_SCHEMA;
import static io.hetu.core.materializedview.TestUtils.DATA_CATALOG;
import static io.hetu.core.materializedview.TestUtils.DATA_SCHEMA;
import static io.hetu.core.materializedview.TestUtils.DATE;
import static io.hetu.core.materializedview.TestUtils.FULL_SQL;
import static io.hetu.core.materializedview.TestUtils.META_CATALOG;
import static io.hetu.core.materializedview.TestUtils.OWNER;
import static io.hetu.core.materializedview.TestUtils.VIEW_NAME;
import static io.hetu.core.materializedview.TestUtils.VIEW_SQL;
import static io.hetu.core.materializedview.utils.MaterializedViewDateUtils.getStringFromDate;
import static io.hetu.core.materializedview.utils.MaterializedViewStatus.DISABLE;
import static io.hetu.core.materializedview.utils.MaterializedViewStatus.ENABLE;
import static io.hetu.core.materializedview.utils.MaterializedViewStatus.getStatusFromString;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
public class TestMaterializedView
{
@Test
public void testMaterializedView()
{
MaterializedView materializedView = createMaterializedView(VIEW_NAME);
assertEquals(VIEW_NAME, materializedView.getViewName());
assertEquals(VIEW_SQL, materializedView.getViewSql());
assertEquals(FULL_SQL, materializedView.getFullSql());
assertEquals(DATA_CATALOG, materializedView.getDataCatalog());
assertEquals(DATA_SCHEMA, materializedView.getDataSchema());
assertEquals(META_CATALOG, materializedView.getMetaCatalog());
assertEquals(CURRENT_CATALOG, materializedView.getCurrentCatalog());
assertEquals(CURRENT_SCHEMA, materializedView.getCurrentSchema());
assertTrue(materializedView.getColumns().equals(createMaterializedViewColumns()));
assertEquals(OWNER, materializedView.getOwner());
assertFalse(materializedView.isRunAsInvoker());
assertEquals(materializedView.getLastRefreshTime(), getStringFromDate(DATE));
assertEquals(getStatusFromString(materializedView.getStatus()), DISABLE);
materializedView.setStatus(ENABLE);
assertEquals(getStatusFromString(materializedView.getStatus()), ENABLE);
}
private MaterializedView createMaterializedView(String viewName)
{
return new MaterializedView(
viewName,
VIEW_SQL,
FULL_SQL,
DATA_CATALOG,
DATA_SCHEMA,
META_CATALOG,
CURRENT_CATALOG,
CURRENT_SCHEMA,
createMaterializedViewColumns(),
OWNER,
false,
MaterializedViewDateUtils.getStringFromDate(DATE),
MaterializedViewStatus.getStatusString(DISABLE));
}
private List<MaterializedViewColumnHandle> createMaterializedViewColumns()
{
List<MaterializedViewColumnHandle> columns = new ArrayList<>();
columns.add(new MaterializedViewColumnHandle("column0", 0, TypeSignature.parseTypeSignature("char")));
columns.add(new MaterializedViewColumnHandle("column1", 1, TypeSignature.parseTypeSignature("int")));
return columns;
}
}

View File

@ -0,0 +1,36 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.materializedview;
import io.hetu.core.materializedview.conf.MaterializedViewConfig;
import org.testng.annotations.Test;
import static io.hetu.core.materializedview.TestUtils.TEST_FILE;
import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL;
import static org.testng.Assert.assertEquals;
public class TestMaterializedViewConfig
{
@Test
public void testConfig()
{
MaterializedViewConfig config = new MaterializedViewConfig();
config.setMetastoreType(TYPE_LOCAL);
config.setMetastorePath(TEST_FILE);
assertEquals(config.getMetastoreType(), TYPE_LOCAL);
assertEquals(config.getMetastorePath(), TEST_FILE);
}
}

View File

@ -0,0 +1,242 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.materializedview;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.hetu.core.materializedview.conf.MaterializedViewConfig;
import io.hetu.core.materializedview.metadata.MaterializedView;
import io.hetu.core.materializedview.metadata.MaterializedViewConnectorMetadata;
import io.hetu.core.materializedview.metadata.MaterializedViewConnectorMetadataFactory;
import io.hetu.core.materializedview.metadata.MaterializedViewMetastore;
import io.hetu.core.materializedview.metadata.MaterializedViewMetastoreFactory;
import io.hetu.core.materializedview.utils.MaterializedViewStatus;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorSession;
import io.prestosql.spi.connector.ConnectorTableHandle;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.metastore.HetuMetastore;
import io.prestosql.spi.type.testing.TestingTypeManager;
import io.prestosql.testing.TestingConnectorSession;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL;
import static io.prestosql.spi.type.VarcharType.VARCHAR;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
public class TestMvConnectorMetadata
{
private static final String SCHEMA_NAME1 = "test1";
private static final String SCHEMA_NAME2 = "test2";
private static final String TABLE_NAME = "table";
private static final String COLUMN_NAME = "column";
private static final String CATALOG_NAME = "memory";
private static final String SCHEMA_NAME = "demo";
private static final String META_CATALOG_NAME = "mv";
private static final String CURRENT_CATALOG_NAME = "hive";
private static final String CURRENT_SCHEMA_NAME = "default";
private static final String EXECUTE_SQL = "select column from test1.table";
private static final String FULL_SQL = "create materialized view mv.demo.view as select column from test1.table";
private static final String STATUS = "ENABLE";
private static final String OWNER = "root";
private MaterializedViewConnectorMetadata metadata;
private MockMaterializedViewMetastore mockMvMetastore;
private ConnectorSession session;
@BeforeClass
public void setUp()
{
MaterializedViewConfig config = new MaterializedViewConfig();
config.setMetastoreType(TYPE_LOCAL);
mockMvMetastore = new MockMaterializedViewMetastore();
MaterializedViewMetastoreFactory mockMetastoreFactory = mock(MaterializedViewMetastoreFactory.class);
when(mockMetastoreFactory.create(any())).thenReturn(mockMvMetastore);
HetuMetastore hetuMetastore = mock(HetuMetastore.class);
MaterializedViewConnectorMetadataFactory factory = new MaterializedViewConnectorMetadataFactory(mockMetastoreFactory, new TestingTypeManager(), hetuMetastore);
metadata = factory.create();
session = new TestingConnectorSession(ImmutableList.of());
metadata.createSchema(session, SCHEMA_NAME1, ImmutableMap.of());
}
@Test
public void testSchema()
{
//listSchemaNames
//createSchema
//dropSchema
metadata.createSchema(session, SCHEMA_NAME2, ImmutableMap.of());
List<String> schemas = metadata.listSchemaNames(session);
assertEquals(schemas.size(), 2);
assertTrue(schemas.contains(SCHEMA_NAME1));
assertTrue(schemas.contains(SCHEMA_NAME2));
metadata.dropSchema(session, SCHEMA_NAME2);
schemas = metadata.listSchemaNames(session);
assertEquals(schemas.size(), 1);
assertTrue(schemas.contains(SCHEMA_NAME1));
}
@Test
public void testViews()
{
ColumnMetadata columnMetadata = new ColumnMetadata(COLUMN_NAME, VARCHAR);
ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(new SchemaTableName(SCHEMA_NAME1, TABLE_NAME), ImmutableList.of(columnMetadata));
ViewColumn viewColumn = new ViewColumn(COLUMN_NAME, VARCHAR.getTypeSignature());
ConnectorMaterializedViewDefinition definition = new ConnectorMaterializedViewDefinition(
EXECUTE_SQL,
Optional.of(CATALOG_NAME),
Optional.of(SCHEMA_NAME),
ImmutableList.of(viewColumn),
Optional.of(OWNER),
false,
FULL_SQL,
META_CATALOG_NAME,
CURRENT_CATALOG_NAME,
CURRENT_SCHEMA_NAME,
STATUS);
metadata.beginCreateMaterializedView(tableMetadata, definition);
SchemaTableName schemaTableName = new SchemaTableName(SCHEMA_NAME1, TABLE_NAME);
metadata.finishCreateMaterializedView(schemaTableName);
Optional<ConnectorViewDefinition> viewDefinition = metadata.getView(session, schemaTableName);
assertTrue(viewDefinition.isPresent());
ConnectorMaterializedViewDefinition materializedViewDefinition = (ConnectorMaterializedViewDefinition) viewDefinition.get();
assertEquals(materializedViewDefinition.toString(), definition.toString());
ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName);
ConnectorTableMetadata getTableMetadata = metadata.getTableMetadata(session, tableHandle);
assertEquals(getTableMetadata.toString(), tableMetadata.toString());
Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
ColumnMetadata getColumnMetadata = metadata.getColumnMetadata(session, tableHandle, columnHandles.get(COLUMN_NAME));
assertEquals(getColumnMetadata, columnMetadata);
List<SchemaTableName> tables = metadata.listTables(session, Optional.empty());
assertEquals(tables.size(), 1);
assertEquals(tables.get(0), schemaTableName);
metadata.dropMaterializedView(session, tableHandle);
tables = metadata.listTables(session, Optional.empty());
assertEquals(tables.size(), 0);
}
class MockMaterializedViewMetastore
implements MaterializedViewMetastore
{
private Map<String, Map<String, MaterializedView>> views;
@Override
public void init()
{
views = new HashMap<>();
}
@Override
public void addMaterializedView(String schema, MaterializedView materializedView)
{
synchronized (views) {
Map<String, MaterializedView> viewMap = views.get(schema);
viewMap.put(materializedView.getViewName(), materializedView);
}
}
@Override
public void dropMaterializedView(String schema, String view)
{
synchronized (views) {
Map<String, MaterializedView> viewMap = views.get(schema);
viewMap.remove(view);
}
}
@Override
public void addSchema(String schemaName)
{
synchronized (views) {
if (views.containsKey(schemaName)) {
return;
}
Map<String, MaterializedView> viewMap = new HashMap<>();
views.put(schemaName, viewMap);
}
}
@Override
public void dropSchema(String schemaName)
{
synchronized (views) {
views.remove(schemaName);
}
}
@Override
public void setViewStatus(SchemaTableName viewName, MaterializedViewStatus status)
{
synchronized (views) {
Map<String, MaterializedView> viewMap = views.get(viewName.getSchemaName());
MaterializedView materializedView = viewMap.get(viewName.getTableName());
materializedView.setStatus(status);
}
}
@Override
public Set<String> getSchemas()
{
synchronized (views) {
return views.keySet();
}
}
@Override
public List<String> getAllMaterializedView(String schemaName)
{
List<String> viewList = new ArrayList<>();
synchronized (views) {
Map<String, MaterializedView> viewMap = views.get(schemaName);
for (MaterializedView view : viewMap.values()) {
viewList.add(view.getViewName());
}
}
return viewList;
}
@Override
public MaterializedView getMaterializedView(SchemaTableName viewName)
{
synchronized (views) {
Map<String, MaterializedView> viewMap = views.get(viewName.getSchemaName());
return viewMap.get(viewName.getTableName());
}
}
}
}

View File

@ -0,0 +1,73 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.materializedview;
import io.hetu.core.materializedview.connector.MaterializedViewColumnHandle;
import io.hetu.core.materializedview.metadata.MaterializedView;
import io.hetu.core.materializedview.utils.MaterializedViewDateUtils;
import io.hetu.core.materializedview.utils.MaterializedViewStatus;
import io.prestosql.spi.type.TypeSignature;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import static io.hetu.core.materializedview.utils.MaterializedViewStatus.DISABLE;
public class TestUtils
{
public static final String TEST_FILE = "testLocalMvMetadata.ini";
public static final String VIEW_NAME = "mv.test.view";
public static final String FULL_SQL = "create materialized view mv.test.view as select * from hive.default.test";
public static final String VIEW_SQL = "select * from hive.default.test";
public static final String DATA_CATALOG = "memory";
public static final String DATA_SCHEMA = "test";
public static final String META_CATALOG = "mv";
public static final String CURRENT_CATALOG = "hive";
public static final String CURRENT_SCHEMA = "default";
public static final String OWNER = "test";
public static final Date DATE = new Date();
private TestUtils()
{
}
public static MaterializedView createMaterializedView(String viewName)
{
return new MaterializedView(
viewName,
VIEW_SQL,
FULL_SQL,
DATA_CATALOG,
DATA_SCHEMA,
META_CATALOG,
CURRENT_CATALOG,
CURRENT_SCHEMA,
createMaterializedViewColumns(),
OWNER,
false,
MaterializedViewDateUtils.getStringFromDate(DATE),
MaterializedViewStatus.getStatusString(DISABLE));
}
public static List<MaterializedViewColumnHandle> createMaterializedViewColumns()
{
List<MaterializedViewColumnHandle> columns = new ArrayList<>();
columns.add(new MaterializedViewColumnHandle("column0", 0, TypeSignature.parseTypeSignature("char")));
columns.add(new MaterializedViewColumnHandle("column1", 1, TypeSignature.parseTypeSignature("int")));
return columns;
}
}

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -153,7 +153,7 @@
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-postgresql</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
<scope>compile</scope>
<exclusions>
<exclusion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-oracle</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-server-rpm</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-server</artifactId>

View File

@ -138,6 +138,12 @@
</artifact>
</artifactSet>
<artifactSet to="plugin/materialized-view">
<artifact id="${project.groupId}:hetu-mv:zip:${project.version}">
<unpack />
</artifact>
</artifactSet>
<!--
Skipped as they are not first class connectors
<artifactSet to="plugin/cassandra">

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-sql-migration-tool</artifactId>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -3,7 +3,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-vdm</artifactId>

View File

@ -10,7 +10,7 @@
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>presto-root</name>
@ -143,6 +143,7 @@
<module>hetu-server-rpm</module>
<module>hetu-common</module>
<module>hetu-hazelcast</module>
<module>hetu-mv</module>
</modules>
<dependencyManagement>
@ -1706,8 +1707,4 @@
</build>
</profile>
</profiles>
<scm>
<tag>branch-1.1</tag>
</scm>
</project>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-array</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-atop</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-base-jdbc</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-benchmark-driver</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-benchmark</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-benchto-benchmarks</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-cli</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-client</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-elasticsearch</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-example-http</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-geospatial-toolkit</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-geospatial</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-hive-hadoop2</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-hive</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>hetu-jdbc</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-jmx</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-kafka</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-local-file</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-main</artifactId>
@ -181,6 +181,42 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.example</groupId>
<artifactId>omnicache</artifactId>
<version>1.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
<version>1.26.0</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>security</artifactId>
@ -500,17 +536,14 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>

View File

@ -149,6 +149,9 @@ public final class SystemSessionProperties
public static final String REUSE_TABLE_SCAN = "reuse_table_scan";
public static final String SPILL_REUSE_TABLESCAN = "spill_reuse_tablescan";
public static final String SPILL_THRESHOLD_REUSE_TABLESCAN = "spill_threshold_reuse_tablescan";
public static final String MATERIALIZED_VIEW_CATALOG_NAME = "materialized_view_catalog_name";
public static final String MATERIALIZED_VIEW_ENABLED = "materialized_view_enabled";
public static final String MATERIALIZED_VIEW_USE_CBO_PLANNER = "materialized_view_use_cbo_planner";
private final List<PropertyMetadata<?>> sessionProperties;
@ -684,6 +687,19 @@ public final class SystemSessionProperties
SPILL_THRESHOLD_REUSE_TABLESCAN,
"Spiller Threshold (in MB) for TableScanOperator and WorkProcessorSourceOperatorAdapter for Reuse Exchange",
featuresConfig.getSpillOperatorThresholdReuseExchange(),
false),
stringProperty(
MATERIALIZED_VIEW_CATALOG_NAME,
"Catalog used to store materialized view",
featuresConfig.getMaterializedViewCatalogName(),
false),
booleanProperty(MATERIALIZED_VIEW_ENABLED,
"Enable materialized view",
false,
false),
booleanProperty(MATERIALIZED_VIEW_USE_CBO_PLANNER,
"Materialized view use volcanoplanner or hepplanner",
true,
false));
}
@ -1207,4 +1223,19 @@ public final class SystemSessionProperties
{
return session.getSystemProperty(SPILL_THRESHOLD_REUSE_TABLESCAN, Integer.class);
}
public static String getMaterializedViewCatalogName(Session session)
{
return session.getSystemProperty(MATERIALIZED_VIEW_CATALOG_NAME, String.class);
}
public static boolean isMaterializedViewEnabled(Session session)
{
return session.getSystemProperty(MATERIALIZED_VIEW_ENABLED, Boolean.class);
}
public static boolean isMaterializedViewCboPlannerUsed(Session session)
{
return session.getSystemProperty(MATERIALIZED_VIEW_USE_CBO_PLANNER, Boolean.class);
}
}

View File

@ -20,6 +20,7 @@ import io.airlift.discovery.client.Announcer;
import io.airlift.discovery.client.ServiceAnnouncement;
import io.airlift.log.Logger;
import io.airlift.node.NodeInfo;
import io.omnicache.OmniCache;
import io.prestosql.connector.informationschema.InformationSchemaConnector;
import io.prestosql.connector.system.DelegatingSystemTablesProvider;
import io.prestosql.connector.system.MetadataBasedSystemTablesProvider;
@ -82,6 +83,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Verify.verify;
import static io.omnicache.OmniCacheImplType.OPEN_LOOKENG;
import static io.prestosql.connector.CatalogName.createInformationSchemaCatalogName;
import static io.prestosql.connector.CatalogName.createSystemTablesCatalogName;
import static io.prestosql.spi.HetuConstant.DATA_CENTER_CONNECTOR_NAME;
@ -124,6 +126,7 @@ public class ConnectorManager
private final Announcer announcer;
private final ServerConfig serverConfig;
private final NodeSchedulerConfig schedulerConfig;
private final OmniCache omniCache;
@Inject
public ConnectorManager(
@ -147,7 +150,8 @@ public class ConnectorManager
Announcer announcer,
ServerConfig serverConfig,
NodeSchedulerConfig schedulerConfig,
HeuristicIndexerManager heuristicIndexerManager)
HeuristicIndexerManager heuristicIndexerManager,
OmniCache omniCache)
{
this.hetuMetaStoreManager = hetuMetaStoreManager;
this.metadataManager = metadataManager;
@ -170,6 +174,7 @@ public class ConnectorManager
this.serverConfig = serverConfig;
this.schedulerConfig = schedulerConfig;
this.heuristicIndexerManager = heuristicIndexerManager;
this.omniCache = omniCache;
}
@PreDestroy
@ -243,9 +248,34 @@ public class ConnectorManager
addCatalogConnector(catalog, connectorFactory, properties);
if (serverConfig.isCoordinator()) {
addSchema(catalogName);
}
return catalog;
}
private void addSchema(String catalogName)
{
Map<String, Object> operand = new HashMap<>();
if (serverConfig.getOmniCacheImplType() == OPEN_LOOKENG) {
// for openlookeng catalog
operand.put("catalog", catalogName);
operand.put("transaction", transactionManager);
operand.put("metadata", metadataManager);
}
else {
// for jdbc catalog
String jdbcUrl = "jdbc:lk:" + nodeManager.getCurrentNode().getInternalUri().getRawSchemeSpecificPart();
operand.put("jdbcDriver", "io.hetu.core.jdbc.OpenLooKengDriver");
operand.put("jdbcUrl", jdbcUrl);
operand.put("jdbcUser", "user");
operand.put("jdbcCatalog", catalogName);
}
omniCache.addSchema(catalogName, operand);
}
/**
* Hetu requires this method to treat DC Connectors differently from other connectors.
*

View File

@ -13,10 +13,13 @@
*/
package io.prestosql.dispatcher;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.ListenableFuture;
import io.airlift.log.Logger;
import io.omnicache.OmniCache;
import io.prestosql.Session;
import io.prestosql.SystemSessionProperties;
import io.prestosql.execution.QueryIdGenerator;
import io.prestosql.execution.QueryInfo;
import io.prestosql.execution.QueryManagerConfig;
@ -26,6 +29,9 @@ import io.prestosql.execution.QueryPreparer.PreparedQuery;
import io.prestosql.execution.QueryState;
import io.prestosql.execution.QueryTracker;
import io.prestosql.execution.resourcegroups.ResourceGroupManager;
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.metadata.QualifiedTablePrefix;
import io.prestosql.metadata.SessionPropertyManager;
import io.prestosql.queryeditorui.QueryEditorUIModule;
import io.prestosql.security.AccessControl;
@ -36,6 +42,8 @@ import io.prestosql.server.SessionSupplier;
import io.prestosql.spi.HetuConstant;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.QueryId;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.resourcegroups.SelectionContext;
import io.prestosql.spi.resourcegroups.SelectionCriteria;
import io.prestosql.spi.service.PropertyService;
@ -47,6 +55,11 @@ import io.prestosql.statestore.StateStoreProvider;
import io.prestosql.statestore.StateUpdater;
import io.prestosql.transaction.TransactionManager;
import io.prestosql.utils.HetuConfig;
import org.apache.calcite.jdbc.CalcitePrepare;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.prepare.Prepare;
import org.apache.calcite.sql.validate.SqlValidator;
import org.apache.calcite.sql2rel.SqlToRelConverter;
import org.weakref.jmx.Flatten;
import org.weakref.jmx.Managed;
@ -56,8 +69,10 @@ import javax.inject.Inject;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.Executor;
@ -68,6 +83,8 @@ import java.util.stream.Collectors;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static io.prestosql.SystemSessionProperties.isMaterializedViewCboPlannerUsed;
import static io.prestosql.SystemSessionProperties.isMaterializedViewEnabled;
import static io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static io.prestosql.spi.StandardErrorCode.QUERY_TEXT_TOO_LARGE;
import static io.prestosql.util.StatementUtils.getQueryType;
@ -79,6 +96,11 @@ import static java.util.Objects.requireNonNull;
public class DispatchManager
{
private static final Logger LOG = Logger.get(DispatchManager.class);
private static final String SYSTEM_JDBC = "system.jdbc";
private static final String SELECT = "select";
private static final String WITH = "with";
private static final String MV_CATALOG_NAME = "mv";
private static final String INFORMATION_SCHEMA = "information_schema";
private final QueryIdGenerator queryIdGenerator;
private final QueryPreparer queryPreparer;
@ -101,6 +123,8 @@ public class DispatchManager
private final QueryTracker<DispatchQuery> queryTracker;
private final QueryManagerStats stats = new QueryManagerStats();
private final Metadata metadata;
private final OmniCache omniCache;
@Inject
public DispatchManager(
@ -116,7 +140,9 @@ public class DispatchManager
QueryManagerConfig queryManagerConfig,
DispatchExecutor dispatchExecutor,
StateStoreProvider stateStoreProvider,
HetuConfig hetuConfig)
HetuConfig hetuConfig,
Metadata metadata,
OmniCache omniCache)
{
this.queryIdGenerator = requireNonNull(queryIdGenerator, "queryIdGenerator is null");
this.queryPreparer = requireNonNull(queryPreparer, "queryPreparer is null");
@ -138,6 +164,8 @@ public class DispatchManager
this.queryExecutor = requireNonNull(dispatchExecutor, "dispatchExecutor is null").getExecutor();
this.queryTracker = new QueryTracker<>(queryManagerConfig, dispatchExecutor.getScheduledExecutor());
this.metadata = metadata;
this.omniCache = omniCache;
}
@PostConstruct
@ -205,6 +233,9 @@ public class DispatchManager
// decode session
session = sessionSupplier.createSession(queryId, sessionContext);
if (isMaterializedViewEnabled(session)) {
query = rewriteQuery(session, query);
}
// prepare query
PreparedQuery preparedQuery = queryPreparer.prepareQuery(session, query);
@ -267,6 +298,89 @@ public class DispatchManager
}
}
private String rewriteQuery(Session session, String query)
{
String rewrittenQuery = query;
if (query.contains(SYSTEM_JDBC)) {
return rewrittenQuery;
}
if (!query.toLowerCase(Locale.ENGLISH).startsWith(SELECT) && !query.toLowerCase(Locale.ENGLISH).startsWith(WITH)) {
return rewrittenQuery;
}
try {
if (!session.getTransactionId().isPresent()) {
session = session.beginTransactionId(transactionManager.beginTransaction(true), transactionManager, accessControl);
}
// rewrite sql using materialized view
CalcitePrepare.Context prepareContext = omniCache.getPrepareContext();
RelOptPlanner planner = omniCache.getRelOptPlanner(prepareContext, isMaterializedViewCboPlannerUsed(session));
List<String> schemas = metadata.listSchemaNames(session, MV_CATALOG_NAME);
for (String schema : schemas) {
if (schema.equals(INFORMATION_SCHEMA)) {
continue;
}
List<QualifiedObjectName> tables = metadata.listTables(session, new QualifiedTablePrefix(MV_CATALOG_NAME, schema));
for (QualifiedObjectName tableName : tables) {
addMaterializedView(session, tableName, prepareContext, planner);
}
}
if (planner.getMaterializations().size() == 0) {
return rewrittenQuery;
}
List<String> defaultSchemaPath = new ArrayList<>();
if (session.getCatalog().isPresent()) {
defaultSchemaPath.add(session.getCatalog().get());
}
if (session.getSchema().isPresent()) {
defaultSchemaPath.add(session.getSchema().get());
}
Prepare.CatalogReader catalogReader = omniCache.getCatalogReader(prepareContext, defaultSchemaPath);
SqlValidator sqlValidator = omniCache.getSqlValidator(prepareContext, catalogReader);
SqlToRelConverter sqlToRelConverter = omniCache.getSqlToRelConverter(prepareContext, catalogReader, sqlValidator, planner);
rewrittenQuery = omniCache.rewriteSql(sqlValidator, sqlToRelConverter, planner, query);
}
catch (Exception e) {
LOG.warn("rewrite sql failed since : " + e.getMessage());
}
finally {
return rewrittenQuery;
}
}
private void addMaterializedView(Session session, QualifiedObjectName tableName, CalcitePrepare.Context prepareContext, RelOptPlanner planner)
{
Optional<ConnectorViewDefinition> connectorViewDefinition = metadata.getView(session, tableName);
if (connectorViewDefinition.isPresent()) {
List<String> viewDefaultSchemaPath = new ArrayList<>();
if (connectorViewDefinition.get() instanceof ConnectorMaterializedViewDefinition) {
String viewCatalog = ((ConnectorMaterializedViewDefinition) connectorViewDefinition.get()).getCurrentCatalog();
String viewSchema = ((ConnectorMaterializedViewDefinition) connectorViewDefinition.get()).getCurrentSchema();
if (viewCatalog != null) {
viewDefaultSchemaPath.add(viewCatalog);
}
if (viewSchema != null) {
viewDefaultSchemaPath.add(viewSchema);
}
}
Prepare.CatalogReader viewCatalogReader = omniCache.getCatalogReader(prepareContext, viewDefaultSchemaPath);
SqlValidator viewSqlValidator = omniCache.getSqlValidator(prepareContext, viewCatalogReader);
SqlToRelConverter viewSqlToRelConverter = omniCache.getSqlToRelConverter(prepareContext, viewCatalogReader, viewSqlValidator, planner);
omniCache.addMaterializationView(viewCatalogReader, viewSqlValidator, viewSqlToRelConverter, planner,
ImmutableList.of(SystemSessionProperties.getMaterializedViewCatalogName(session), tableName.getSchemaName(), tableName.getObjectName()),
connectorViewDefinition.get().getOriginalSql());
}
}
private boolean queryCreated(DispatchQuery dispatchQuery)
{
boolean queryAdded = queryTracker.addQuery(dispatchQuery);

View File

@ -0,0 +1,74 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.execution;
import com.google.common.util.concurrent.ListenableFuture;
import io.prestosql.Session;
import io.prestosql.heuristicindex.HeuristicIndexerManager;
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.security.AccessControl;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.sql.analyzer.SemanticException;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.Expression;
import io.prestosql.transaction.TransactionManager;
import java.util.List;
import java.util.Optional;
import static com.google.common.util.concurrent.Futures.immediateFuture;
import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName;
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE;
import static io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED;
public class DropMaterializedViewTask
implements DataDefinitionTask<DropMaterializedView>
{
@Override
public String getName()
{
return "DROP MATERIALIZED VIEW";
}
@Override
public ListenableFuture<?> execute(DropMaterializedView statement, TransactionManager transactionManager, Metadata metadata, AccessControl accessControl,
QueryStateMachine stateMachine, List<Expression> parameters, HeuristicIndexerManager heuristicIndexerManager)
{
Session session = stateMachine.getSession();
QualifiedObjectName tableName = createQualifiedObjectName(session, statement, statement.getTableName());
// materialized view exists?
// Optional<TableHandle> tableHandle = metadata.getTableHandle(session, tableName);
Optional<ConnectorViewDefinition> definition = metadata.getView(session, tableName);
if (!definition.isPresent()) {
if (!statement.isExists()) {
throw new SemanticException(MISSING_TABLE, statement, "MATERIALIZED VIEW '%s' does not exist", tableName);
}
return immediateFuture(null);
}
if (!(definition.get() instanceof ConnectorMaterializedViewDefinition)) {
throw new SemanticException(NOT_SUPPORTED, statement, "'%s' is not a materialized view.", tableName);
}
accessControl.checkCanDropMaterializedTable(session.getRequiredTransactionId(), session.getIdentity(), tableName);
metadata.dropMaterializedView(session, (ConnectorMaterializedViewDefinition) definition.get(), tableName);
return immediateFuture(null);
}
}

View File

@ -254,6 +254,11 @@ public class QueryStateMachine
return queryId;
}
public String getQuery()
{
return query;
}
public Session getSession()
{
return session;

View File

@ -549,7 +549,7 @@ public class SqlQueryExecution
WarningCollector warningCollector)
{
LogicalPlanner logicalPlanner = new LogicalPlanner(session, planOptimizers, idAllocator, metadata, typeAnalyzer, statsCalculator, costCalculator, warningCollector);
return logicalPlanner.plan(analysis);
return logicalPlanner.plan(analysis, stateMachine.getQuery());
}
private static Set<CatalogName> extractConnectors(Analysis analysis)
@ -708,6 +708,11 @@ public class SqlQueryExecution
return stateMachine.getQueryId();
}
public String getQuery()
{
return stateMachine.getQuery();
}
@Override
public QueryInfo getQueryInfo()
{

View File

@ -25,6 +25,7 @@ import io.prestosql.spi.connector.CatalogSchemaName;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorCapabilities;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorOutputMetadata;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorViewDefinition;
@ -33,6 +34,7 @@ import io.prestosql.spi.connector.ConstraintApplicationResult;
import io.prestosql.spi.connector.LimitApplicationResult;
import io.prestosql.spi.connector.ProjectionApplicationResult;
import io.prestosql.spi.connector.SampleType;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.connector.SubQueryApplicationResult;
import io.prestosql.spi.connector.SystemTable;
import io.prestosql.spi.expression.ConnectorExpression;
@ -196,6 +198,20 @@ public interface Metadata
*/
void dropTable(Session session, TableHandle tableHandle);
/**
* Drops the specified materialized view
*
* @throws RuntimeException if the table can not be dropped or table handle is no longer valid
*/
void dropMaterializedView(Session session, ConnectorMaterializedViewDefinition definition, QualifiedObjectName name);
/**
* Refresh the specified materialized view
*
* @throws RuntimeException if the table can not be dropped or table handle is no longer valid
*/
void refreshMaterializedView(Session session, TableHandle tableHandle);
Optional<NewTableLayout> getNewTableLayout(Session session, String catalogName, ConnectorTableMetadata tableMetadata);
/**
@ -208,6 +224,16 @@ public interface Metadata
*/
Optional<ConnectorOutputMetadata> finishCreateTable(Session session, OutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics);
/**
* Begin the atomic creation of a table with data.
*/
OutputTableHandle beginCreateMaterializedView(Session session, String catalogName, ConnectorTableMetadata tableMetadata, ConnectorMaterializedViewDefinition definition, Optional<NewTableLayout> layout);
/**
* Finish a materialized view creation with data after the data is written.
*/
Optional<ConnectorOutputMetadata> finishCreateMaterializedView(Session session, OutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics, String originalTargetCatalog, SchemaTableName name);
Optional<NewTableLayout> getInsertLayout(Session session, TableHandle target);
/**

View File

@ -51,6 +51,7 @@ import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorCapabilities;
import io.prestosql.spi.connector.ConnectorDeleteAsInsertTableHandle;
import io.prestosql.spi.connector.ConnectorInsertTableHandle;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorMetadata;
import io.prestosql.spi.connector.ConnectorOutputMetadata;
import io.prestosql.spi.connector.ConnectorOutputTableHandle;
@ -668,6 +669,38 @@ public final class MetadataManager
metadata.dropTable(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle());
}
@Override
public void dropMaterializedView(Session session, ConnectorMaterializedViewDefinition definition, QualifiedObjectName name)
{
//drop table data
CatalogMetadata dataCatalogMetadata = getCatalogMetadataForWrite(session, definition.getCatalog().orElse(null));
CatalogName dataCatalogName = dataCatalogMetadata.getCatalogName();
ConnectorMetadata dataMetadata = getMetadataForWrite(session, dataCatalogName);
QualifiedObjectName dataTable = new QualifiedObjectName(
definition.getCatalog().orElse(null),
name.getSchemaName(),
name.getObjectName());
Optional<TableHandle> dataTableHandle = getTableHandle(session, dataTable);
dataTableHandle.ifPresent(tableHandle -> dataMetadata.dropTable(session.toConnectorSession(dataCatalogName), tableHandle.getConnectorHandle()));
//drop material view
Optional<CatalogMetadata> mvCatalogMetadata = getOptionalCatalogMetadata(session, name.getCatalogName());
if (mvCatalogMetadata.isPresent()) {
CatalogName mvCatalogName = mvCatalogMetadata.get().getCatalogName();
ConnectorMetadata mvMetadata = getMetadata(session, mvCatalogName);
Optional<TableHandle> mvTableHandle = getTableHandle(session, name);
mvTableHandle.ifPresent(tableHandle -> mvMetadata.dropMaterializedView(session.toConnectorSession(mvCatalogName), tableHandle.getConnectorHandle()));
}
}
@Override
public void refreshMaterializedView(Session session, TableHandle tableHandle)
{
CatalogName catalogName = tableHandle.getCatalogName();
ConnectorMetadata metadata = getMetadataForWrite(session, catalogName);
metadata.refreshMaterializedView(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle());
}
@Override
public Optional<NewTableLayout> getInsertLayout(Session session, TableHandle table)
{
@ -770,6 +803,37 @@ public final class MetadataManager
return metadata.finishCreateTable(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle(), fragments, computedStatistics);
}
@Override
public OutputTableHandle beginCreateMaterializedView(Session session, String catalogName, ConnectorTableMetadata tableMetadata, ConnectorMaterializedViewDefinition definition, Optional<NewTableLayout> layout)
{
String metaCatalog = definition.getMetaCatalog();
CatalogMetadata catalogMetadata = getOptionalCatalogMetadata(session, metaCatalog).get();
CatalogName catalog = catalogMetadata.getCatalogName();
ConnectorMetadata metadata = catalogMetadata.getMetadata();
metadata.beginCreateMaterializedView(tableMetadata, definition);
CatalogMetadata dataCatalogMetadata = getCatalogMetadataForWrite(session, catalogName);
CatalogName dataCatalog = dataCatalogMetadata.getCatalogName();
ConnectorMetadata dataConnectorMetadata = dataCatalogMetadata.getMetadata();
ConnectorTransactionHandle transactionHandle = dataCatalogMetadata.getTransactionHandleFor(dataCatalog);
ConnectorSession dataConnectorSession = session.toConnectorSession(dataCatalog);
ConnectorOutputTableHandle outputTablehandle = dataConnectorMetadata.beginCreateTable(dataConnectorSession, tableMetadata, layout.map(NewTableLayout::getLayout));
return new OutputTableHandle(dataCatalog, transactionHandle, outputTablehandle);
}
@Override
public Optional<ConnectorOutputMetadata> finishCreateMaterializedView(Session session, OutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics, String originalTargetCatalog, SchemaTableName name)
{
CatalogMetadata catalogMetadata = getOptionalCatalogMetadata(session, originalTargetCatalog).get();
ConnectorMetadata metadata = catalogMetadata.getMetadata();
metadata.finishCreateMaterializedView(name);
CatalogMetadata dataCatalogMetadata = getCatalogMetadataForWrite(session, tableHandle.getCatalogName().getCatalogName());
CatalogName dataCatalogName = dataCatalogMetadata.getCatalogName();
ConnectorMetadata dataConnectorMetadata = getMetadata(session, dataCatalogName);
return dataConnectorMetadata.finishCreateTable(session.toConnectorSession(dataCatalogName), tableHandle.getConnectorHandle(), fragments, computedStatistics);
}
@Override
public InsertTableHandle beginInsert(Session session, TableHandle tableHandle, boolean isOverwrite)
{

View File

@ -97,7 +97,8 @@ public class TableWriterOperator
this.columnChannels = requireNonNull(columnChannels, "columnChannels is null");
this.pageSinkManager = requireNonNull(pageSinkManager, "pageSinkManager is null");
checkArgument(writerTarget instanceof CreateTarget || writerTarget instanceof InsertTarget || writerTarget instanceof TableWriterNode.UpdateTarget
|| writerTarget instanceof TableWriterNode.DeleteAsInsertTarget, "writerTarget must be CreateTarget or InsertTarget or UpdateTarget");
|| writerTarget instanceof TableWriterNode.DeleteAsInsertTarget || writerTarget instanceof TableWriterNode.CreateMaterializedTarget,
"writerTarget must be CreateTarget or CreateMaterializedTarget or InsertTarget or UpdateTarget");
this.target = requireNonNull(writerTarget, "writerTarget is null");
this.session = session;
this.taskId = taskId;
@ -121,6 +122,9 @@ public class TableWriterOperator
if (target instanceof CreateTarget) {
return pageSinkManager.createPageSink(session, driverTaskId, ((CreateTarget) target).getHandle());
}
if (target instanceof TableWriterNode.CreateMaterializedTarget) {
return pageSinkManager.createPageSink(session, driverTaskId, ((TableWriterNode.CreateMaterializedTarget) target).getHandle());
}
if (target instanceof InsertTarget) {
return pageSinkManager.createPageSink(session, driverTaskId, ((InsertTarget) target).getHandle());
}

View File

@ -254,7 +254,7 @@ public class CachedSqlQueryExecution
Map<String, Object> systemSessionProperties)
{
// build a new plan
Plan plan = logicalPlanner.plan(analysis);
Plan plan = logicalPlanner.plan(analysis, getQuery());
// Cache the plan
CachedSqlQueryExecutionPlan newCachedPlan = new CachedSqlQueryExecutionPlan(statement, tableNames, tableStatistics, planOptimizers, plan,
analysis.getParameters(), columnTypes, getSession().getTimeZoneKey(), getSession().getIdentity(), systemSessionProperties);

View File

@ -82,9 +82,9 @@ public class HetuLogicalPlanner
}
@Override
public Plan plan(Analysis analysis, Stage stage)
public Plan plan(Analysis analysis, String originQuery, Stage stage)
{
PlanNode root = planStatement(analysis, analysis.getStatement());
PlanNode root = planStatement(analysis, analysis.getStatement(), originQuery);
planSanityChecker.validateIntermediatePlan(root, session, metadata, typeAnalyzer, symbolAllocator.getTypes(),
warningCollector);

View File

@ -142,6 +142,27 @@ public interface AccessControl
*/
void checkCanSetTableComment(TransactionId transactionId, Identity identity, QualifiedObjectName tableName);
/**
* Check if identity is allowed to create the materialized table.
*
* @throws io.prestosql.spi.security.AccessDeniedException if not allowed
*/
void checkCanCreateMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName, Set<String> columnNames);
/**
* Check if identity is allowed to refresh the materialized table.
*
* @throws io.prestosql.spi.security.AccessDeniedException if not allowed
*/
void checkCanRefreshMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName);
/**
* Check if identity is allowed to drop the materialized table.
*
* @throws io.prestosql.spi.security.AccessDeniedException if not allowed
*/
void checkCanDropMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName);
/**
* Check if identity is allowed to show metadata of tables by executing SHOW TABLES, SHOW GRANTS etc. in a catalog.
* <p>

View File

@ -369,6 +369,24 @@ public class AccessControlManager
}
}
@Override
public void checkCanCreateMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName, Set<String> columnNames)
{
return;
}
@Override
public void checkCanRefreshMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
return;
}
@Override
public void checkCanDropMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
return;
}
@Override
public void checkCanShowTablesMetadata(TransactionId transactionId, Identity identity, CatalogSchemaName schema)
{

View File

@ -98,6 +98,21 @@ public class AllowAllAccessControl
{
}
@Override
public void checkCanCreateMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName, Set<String> columnNames)
{
}
@Override
public void checkCanRefreshMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanDropMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanShowTablesMetadata(TransactionId transactionId, Identity identity, CatalogSchemaName schema)
{

View File

@ -138,6 +138,21 @@ public class DenyAllAccessControl
denyCommentTable(tableName.toString());
}
@Override
public void checkCanCreateMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName, Set<String> columnNames)
{
}
@Override
public void checkCanRefreshMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanDropMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanShowTablesMetadata(TransactionId transactionId, Identity identity, CatalogSchemaName schema)
{

View File

@ -58,6 +58,7 @@ import io.prestosql.execution.DeallocateTask;
import io.prestosql.execution.DropCacheTask;
import io.prestosql.execution.DropColumnTask;
import io.prestosql.execution.DropIndexTask;
import io.prestosql.execution.DropMaterializedViewTask;
import io.prestosql.execution.DropRoleTask;
import io.prestosql.execution.DropSchemaTask;
import io.prestosql.execution.DropTableTask;
@ -130,6 +131,7 @@ import io.prestosql.sql.tree.Deallocate;
import io.prestosql.sql.tree.DropCache;
import io.prestosql.sql.tree.DropColumn;
import io.prestosql.sql.tree.DropIndex;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropRole;
import io.prestosql.sql.tree.DropSchema;
import io.prestosql.sql.tree.DropTable;
@ -362,6 +364,7 @@ public class CoordinatorModule
bindDataDefinitionTask(binder, executionBinder, DropTable.class, DropTableTask.class);
bindDataDefinitionTask(binder, executionBinder, CreateView.class, CreateViewTask.class);
bindDataDefinitionTask(binder, executionBinder, DropView.class, DropViewTask.class);
bindDataDefinitionTask(binder, executionBinder, DropMaterializedView.class, DropMaterializedViewTask.class);
bindDataDefinitionTask(binder, executionBinder, Use.class, UseTask.class);
bindDataDefinitionTask(binder, executionBinder, SetSession.class, SetSessionTask.class);
bindDataDefinitionTask(binder, executionBinder, ResetSession.class, ResetSessionTask.class);

View File

@ -17,9 +17,11 @@ import io.airlift.configuration.Config;
import io.airlift.units.Duration;
import io.airlift.units.MaxDuration;
import io.airlift.units.MinDuration;
import io.omnicache.OmniCacheImplType;
import javax.validation.constraints.NotNull;
import static io.omnicache.OmniCacheImplType.OPEN_LOOKENG;
import static java.util.concurrent.TimeUnit.MINUTES;
public class ServerConfig
@ -29,6 +31,7 @@ public class ServerConfig
private boolean includeExceptionInResponse = true;
private Duration gracePeriod = new Duration(2, MINUTES);
private boolean enhancedErrorReporting = true;
private OmniCacheImplType omniCacheImplType = OPEN_LOOKENG;
// Main coordinator TODO: remove this when main coordinator election is implemented
public boolean isCoordinator()
@ -96,4 +99,16 @@ public class ServerConfig
this.enhancedErrorReporting = value;
return this;
}
public OmniCacheImplType getOmniCacheImplType()
{
return omniCacheImplType;
}
@Config("omnicache.impl-type")
public ServerConfig setOmniCacheImplType(OmniCacheImplType omniCacheImplType)
{
this.omniCacheImplType = omniCacheImplType;
return this;
}
}

View File

@ -27,6 +27,8 @@ import io.airlift.stats.JmxGcMonitor;
import io.airlift.stats.PauseMeter;
import io.airlift.units.DataSize;
import io.airlift.units.Duration;
import io.omnicache.OmniCache;
import io.omnicache.OmniCacheFactory;
import io.prestosql.GroupByHashPageIndexerFactory;
import io.prestosql.PagesIndexPageSorter;
import io.prestosql.SystemSessionProperties;
@ -261,6 +263,9 @@ public class ServerMainModule
newOptionalBinder(binder, ExplainAnalyzeContext.class);
//omni cache
binder.bind(OmniCache.class).toInstance(OmniCacheFactory.create(serverConfig.getOmniCacheImplType()));
// GC Monitor
binder.bind(GcMonitor.class).to(JmxGcMonitor.class).in(Scopes.SINGLETON);

View File

@ -155,6 +155,7 @@ public class FeaturesConfig
private int maxGroupingSets = 2048;
//transform selfjoin to aggregates if applicable
private boolean transformSelfJoinToGroupby = true;
private String materializedViewCatalogName = "hive";
public enum JoinReorderingStrategy
{
@ -1214,4 +1215,18 @@ public class FeaturesConfig
this.spillOperatorThresholdReuseExchange = spillOperatorThresholdReuseExchange;
return this;
}
@NotNull
public String getMaterializedViewCatalogName()
{
return materializedViewCatalogName;
}
@Config("materialized-view-catalog-name")
@ConfigDescription("Catalog used to store materialized view")
public FeaturesConfig setMaterializedViewCatalogName(String catalogName)
{
this.materializedViewCatalogName = catalogName;
return this;
}
}

View File

@ -182,7 +182,7 @@ public class QueryExplainer
// plan statement
LogicalPlanner logicalPlanner = new LogicalPlanner(session, planOptimizers, idAllocator, metadata, new TypeAnalyzer(sqlParser, metadata), statsCalculator, costCalculator, warningCollector);
return logicalPlanner.plan(analysis);
return logicalPlanner.plan(analysis, null);
}
private SubPlan getDistributedPlan(Session session, Statement statement, List<Expression> parameters, WarningCollector warningCollector)

View File

@ -70,6 +70,7 @@ import io.prestosql.sql.tree.Call;
import io.prestosql.sql.tree.Comment;
import io.prestosql.sql.tree.Commit;
import io.prestosql.sql.tree.CreateIndex;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateSchema;
import io.prestosql.sql.tree.CreateTable;
import io.prestosql.sql.tree.CreateTableAsSelect;
@ -82,6 +83,7 @@ import io.prestosql.sql.tree.DereferenceExpression;
import io.prestosql.sql.tree.DropCache;
import io.prestosql.sql.tree.DropColumn;
import io.prestosql.sql.tree.DropIndex;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropSchema;
import io.prestosql.sql.tree.DropTable;
import io.prestosql.sql.tree.DropView;
@ -630,6 +632,60 @@ class StatementAnalyzer
return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT));
}
@Override
protected Scope visitCreateMaterializedView(CreateMaterializedView node, Optional<Scope> scope)
{
analysis.setUpdateType("CREATE MATERIALIZED VIEW");
// turn this into a query that has a new table writer node on top.
//get catalog name from config.properties file: materialized-view-catalog-name=;
String configuredCatalogName = SystemSessionProperties.getMaterializedViewCatalogName(session);
String targetCatalogName = node.getTargetCatalogName(configuredCatalogName);
if (!metadata.catalogExists(session, targetCatalogName)) {
throw new SemanticException(MISSING_CATALOG, node, "Destination catalog '%s' not exists", targetCatalogName);
}
QualifiedObjectName targetTable = createQualifiedObjectName(session, node, node.getName());
targetTable = new QualifiedObjectName(targetCatalogName, targetTable.getSchemaName(), targetTable.getObjectName());
analysis.setCreateTableDestination(targetTable);
Optional<TableHandle> targetTableHandle = metadata.getTableHandle(session, targetTable);
if (targetTableHandle.isPresent()) {
if (node.isNotExists()) {
analysis.setCreateTableAsSelectNoOp(true);
return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT));
}
throw new SemanticException(TABLE_ALREADY_EXISTS, node, "Destination table '%s' already exists", targetTable);
}
validateProperties(node.getProperties(), scope);
analysis.setCreateTableProperties(mapFromProperties(node.getProperties()));
node.getColumnAliases().ifPresent(analysis::setCreateTableColumnAliases);
analysis.setCreateTableComment(node.getComment());
accessControl.checkCanCreateTable(session.getRequiredTransactionId(), session.getIdentity(), targetTable);
// analyze the query that creates the table
Scope queryScope = process(node.getQuery(), scope);
if (node.getColumnAliases().isPresent()) {
validateColumnAliases(node.getColumnAliases().get(), queryScope.getRelationType().getVisibleFieldCount());
// analzie only column types in subquery if column alias exists
for (Field field : queryScope.getRelationType().getVisibleFields()) {
if (field.getType().equals(UNKNOWN)) {
throw new SemanticException(COLUMN_TYPE_UNKNOWN, node, "Column type is unknown at position %s", queryScope.getRelationType().indexOf(field) + 1);
}
}
}
else {
validateColumns(node, queryScope.getRelationType());
}
return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT));
}
@Override
protected Scope visitCreateTableAsSelect(CreateTableAsSelect node, Optional<Scope> scope)
{
@ -798,6 +854,12 @@ class StatementAnalyzer
return createAndAssignScope(node, scope);
}
@Override
protected Scope visitDropMaterializedView(DropMaterializedView node, Optional<Scope> scope)
{
return createAndAssignScope(node, scope);
}
@Override
protected Scope visitRenameTable(RenameTable node, Optional<Scope> scope)
{

View File

@ -289,6 +289,7 @@ import static io.prestosql.sql.planner.plan.ExchangeNode.Scope.LOCAL;
import static io.prestosql.sql.planner.plan.JoinNode.Type.FULL;
import static io.prestosql.sql.planner.plan.JoinNode.Type.INNER;
import static io.prestosql.sql.planner.plan.JoinNode.Type.RIGHT;
import static io.prestosql.sql.planner.plan.TableWriterNode.CreateMaterializedTarget;
import static io.prestosql.sql.planner.plan.TableWriterNode.CreateTarget;
import static io.prestosql.sql.planner.plan.TableWriterNode.DeleteAsInsertTarget;
import static io.prestosql.sql.planner.plan.TableWriterNode.InsertTarget;
@ -3056,6 +3057,10 @@ public class LocalExecutionPlanner
if (target instanceof CreateTarget) {
return metadata.finishCreateTable(session, ((CreateTarget) target).getHandle(), fragments, statistics);
}
else if (target instanceof CreateMaterializedTarget) {
CreateMaterializedTarget materializedTarget = (CreateMaterializedTarget) target;
return metadata.finishCreateMaterializedView(session, materializedTarget.getHandle(), fragments, statistics, materializedTarget.getOriginalTargetCatalog(), materializedTarget.getSchemaTableName());
}
else if (target instanceof InsertTarget) {
return metadata.finishInsert(session, ((InsertTarget) target).getHandle(), fragments, statistics);
}

View File

@ -35,7 +35,9 @@ import io.prestosql.operator.ReuseExchangeOperator;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.function.Signature;
import io.prestosql.spi.statistics.TableStatisticsMetadata;
import io.prestosql.spi.type.CharType;
@ -69,6 +71,7 @@ import io.prestosql.sql.planner.sanity.PlanSanityChecker;
import io.prestosql.sql.tree.Analyze;
import io.prestosql.sql.tree.Cast;
import io.prestosql.sql.tree.ComparisonExpression;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateTableAsSelect;
import io.prestosql.sql.tree.Delete;
import io.prestosql.sql.tree.Explain;
@ -96,6 +99,7 @@ import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
@ -106,6 +110,7 @@ import static com.google.common.base.Verify.verify;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableMap.toImmutableMap;
import static com.google.common.collect.Streams.zip;
import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName;
import static io.prestosql.spi.StandardErrorCode.NOT_FOUND;
import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED;
import static io.prestosql.spi.statistics.TableStatisticType.ROW_COUNT;
@ -129,6 +134,7 @@ public class LogicalPlanner
CREATED, OPTIMIZED, OPTIMIZED_AND_VALIDATED
}
private static final String AS = "as";
private final PlanNodeIdAllocator idAllocator;
private final Session session;
@ -178,14 +184,14 @@ public class LogicalPlanner
this.warningCollector = requireNonNull(warningCollector, "warningCollector is null");
}
public Plan plan(Analysis analysis)
public Plan plan(Analysis analysis, String originQuery)
{
return plan(analysis, Stage.OPTIMIZED_AND_VALIDATED);
return plan(analysis, originQuery, Stage.OPTIMIZED_AND_VALIDATED);
}
public Plan plan(Analysis analysis, Stage stage)
public Plan plan(Analysis analysis, String originQuery, Stage stage)
{
PlanNode root = planStatement(analysis, analysis.getStatement());
PlanNode root = planStatement(analysis, analysis.getStatement(), originQuery);
planSanityChecker.validateIntermediatePlan(root, session, metadata, typeAnalyzer, symbolAllocator.getTypes(), warningCollector);
@ -193,7 +199,7 @@ public class LogicalPlanner
for (PlanOptimizer optimizer : planOptimizers) {
if (OptimizerUtils.isEnabledLegacy(optimizer, session, root)) {
root = optimizer.optimize(root, session, symbolAllocator.getTypes(), symbolAllocator, idAllocator,
warningCollector);
warningCollector);
requireNonNull(root, format("%s returned a null plan", optimizer.getClass().getName()));
}
}
@ -210,7 +216,7 @@ public class LogicalPlanner
return new Plan(root, types, StatsAndCosts.create(root, statsProvider, costProvider));
}
public PlanNode planStatement(Analysis analysis, Statement statement)
public PlanNode planStatement(Analysis analysis, Statement statement, String originQuery)
{
if (statement instanceof CreateTableAsSelect && analysis.isCreateTableAsSelectNoOp()) {
checkState(analysis.getCreateTableDestination().isPresent(), "Table destination is missing");
@ -218,10 +224,10 @@ public class LogicalPlanner
PlanNode source = new ValuesNode(idAllocator.getNextId(), ImmutableList.of(symbol), ImmutableList.of(ImmutableList.of(new LongLiteral("0"))));
return new OutputNode(idAllocator.getNextId(), source, ImmutableList.of("rows"), ImmutableList.of(symbol));
}
return createOutputPlan(planStatementWithoutOutput(analysis, statement), analysis);
return createOutputPlan(planStatementWithoutOutput(analysis, statement, originQuery), analysis);
}
private RelationPlan planStatementWithoutOutput(Analysis analysis, Statement statement)
private RelationPlan planStatementWithoutOutput(Analysis analysis, Statement statement, String originQuery)
{
if (statement instanceof CreateTableAsSelect) {
if (analysis.isCreateTableAsSelectNoOp()) {
@ -229,6 +235,15 @@ public class LogicalPlanner
}
return createTableCreationPlan(analysis, ((CreateTableAsSelect) statement).getQuery());
}
else if (statement instanceof CreateMaterializedView) {
if (analysis.isCreateTableAsSelectNoOp()) {
throw new PrestoException(NOT_SUPPORTED, "CREATE MATERIALIZED VIEW IF NOT EXISTS is not supported in this context " + statement.getClass().getSimpleName());
}
CreateMaterializedView createMaterializedView = (CreateMaterializedView) statement;
String originalFullSql = originQuery;
String executeSql = originQuery.substring(originQuery.toLowerCase(Locale.ENGLISH).indexOf(AS) + AS.length()).trim();
return createMaterializedViewCreationPlan(analysis, createMaterializedView, executeSql, originalFullSql);
}
else if (statement instanceof Analyze) {
return createAnalyzePlan(analysis, (Analyze) statement);
}
@ -258,7 +273,7 @@ public class LogicalPlanner
private RelationPlan createExplainAnalyzePlan(Analysis analysis, Explain statement)
{
RelationPlan underlyingPlan = planStatementWithoutOutput(analysis, statement.getStatement());
RelationPlan underlyingPlan = planStatementWithoutOutput(analysis, statement.getStatement(), null);
PlanNode root = underlyingPlan.getRoot();
Scope scope = analysis.getScope(statement);
Symbol outputSymbol = symbolAllocator.newSymbol(scope.getRelationType().getFieldByIndex(0));
@ -310,6 +325,51 @@ public class LogicalPlanner
return new RelationPlan(planNode, analysis.getScope(analyzeStatement), planNode.getOutputSymbols());
}
private RelationPlan createMaterializedViewCreationPlan(Analysis analysis, CreateMaterializedView statement, String executeSql, String originalFullSql)
{
QualifiedObjectName destination = analysis.getCreateTableDestination().get();
RelationPlan plan = createRelationPlan(analysis, statement.getQuery());
String originalTargetCatalog = createQualifiedObjectName(session, statement, statement.getName()).getCatalogName();
String originalSqlCatalog = session.getCatalog().orElse(null);
String originalSqlSchema = session.getSchema().orElse(null);
ConnectorTableMetadata tableMetadata = createTableMetadata(
destination,
getOutputTableColumns(plan, analysis.getColumnAliases()),
analysis.getCreateTableProperties(),
analysis.getParameters(),
analysis.getCreateTableComment());
ConnectorMaterializedViewDefinition definition = new ConnectorMaterializedViewDefinition(
executeSql,
Optional.of(destination.getCatalogName()),
Optional.of(destination.getSchemaName()),
getViewColumns(plan, analysis.getColumnAliases()),
Optional.of(session.getUser()),
false,
originalFullSql,
originalTargetCatalog,
originalSqlCatalog,
originalSqlSchema,
"DISABLE");
Optional<NewTableLayout> newTableLayout = metadata.getNewTableLayout(session, destination.getCatalogName(), tableMetadata);
List<String> columnNames = tableMetadata.getColumns().stream()
.filter(column -> !column.isHidden())
.map(ColumnMetadata::getName)
.collect(toImmutableList());
TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, destination.getCatalogName(), tableMetadata);
return createTableWriterPlan(
analysis,
plan,
new TableWriterNode.CreateMaterializedReference(destination.getCatalogName(), tableMetadata, newTableLayout, definition),
columnNames,
newTableLayout,
statisticsMetadata);
}
private RelationPlan createTableCreationPlan(Analysis analysis, Query query)
{
QualifiedObjectName destination = analysis.getCreateTableDestination().get();
@ -542,7 +602,7 @@ public class LogicalPlanner
TableHandle handle = analysis.getTableHandle(node.getTable());
if (handle.getConnectorHandle().isDeleteAsInsertSupported()) {
QueryPlanner.UpdateDeleteRelationPlan deletePlan = new QueryPlanner(analysis, symbolAllocator, idAllocator, buildLambdaDeclarationToSymbolMap(analysis, symbolAllocator), metadata, session)
.planDeleteRowAsInsert(node);
.planDeleteRowAsInsert(node);
RelationPlan plan = deletePlan.getPlan();
@ -732,6 +792,18 @@ public class LogicalPlanner
return columns.build();
}
private static List<ConnectorViewDefinition.ViewColumn> getViewColumns(RelationPlan plan, Optional<List<Identifier>> columnAliases)
{
ImmutableList.Builder<ConnectorViewDefinition.ViewColumn> columns = ImmutableList.builder();
int aliasPosition = 0;
for (Field field : plan.getDescriptor().getVisibleFields()) {
String columnName = columnAliases.isPresent() ? columnAliases.get().get(aliasPosition).getValue() : field.getName().get();
columns.add(new ConnectorViewDefinition.ViewColumn(columnName, field.getType().getTypeSignature()));
aliasPosition++;
}
return columns.build();
}
private static Map<NodeRef<LambdaArgumentDeclaration>, Symbol> buildLambdaDeclarationToSymbolMap(Analysis analysis, SymbolAllocator symbolAllocator)
{
Map<NodeRef<LambdaArgumentDeclaration>, Symbol> resultMap = new LinkedHashMap<>();

View File

@ -65,6 +65,13 @@ public class OutputExtractor
"cannot have more than a single create, insert or delete in a query");
schemaTableName = target.getSchemaTableName();
}
else if (writerTarget instanceof TableWriterNode.CreateMaterializedTarget) {
TableWriterNode.CreateMaterializedTarget target = (TableWriterNode.CreateMaterializedTarget) writerTarget;
catalogName = target.getHandle().getCatalogName();
checkState(schemaTableName == null || schemaTableName.equals(target.getSchemaTableName()),
"cannot have more than a single create, insert or delete in a query");
schemaTableName = target.getSchemaTableName();
}
else if (writerTarget instanceof InsertTarget) {
InsertTarget target = (InsertTarget) writerTarget;
catalogName = target.getHandle().getCatalogName();

View File

@ -19,6 +19,7 @@ import io.prestosql.Session;
import io.prestosql.execution.warnings.WarningCollector;
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.TableHandle;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.sql.planner.PlanNodeIdAllocator;
import io.prestosql.sql.planner.SymbolAllocator;
import io.prestosql.sql.planner.TypeProvider;
@ -34,6 +35,8 @@ import io.prestosql.sql.planner.plan.StatisticsWriterNode;
import io.prestosql.sql.planner.plan.TableFinishNode;
import io.prestosql.sql.planner.plan.TableScanNode;
import io.prestosql.sql.planner.plan.TableWriterNode;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateMaterializedReference;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateMaterializedTarget;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateReference;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateTarget;
import io.prestosql.sql.planner.plan.TableWriterNode.DeleteAsInsertReference;
@ -204,6 +207,12 @@ public class BeginTableWrite
CreateReference create = (CreateReference) target;
return new CreateTarget(metadata.beginCreateTable(session, create.getCatalog(), create.getTableMetadata(), create.getLayout()), create.getTableMetadata().getTable());
}
if (target instanceof CreateMaterializedReference) {
CreateMaterializedReference create = (CreateMaterializedReference) target;
return new CreateMaterializedTarget(metadata.beginCreateMaterializedView(session, create.getCatalog(), create.getTableMetadata(), (ConnectorMaterializedViewDefinition) create.getDefinition(), create.getLayout()),
create.getTableMetadata().getTable(),
((ConnectorMaterializedViewDefinition) create.getDefinition()).getMetaCatalog());
}
if (target instanceof InsertReference) {
InsertReference insert = (InsertReference) target;
return new InsertTarget(metadata.beginInsert(session, insert.getHandle(), insert.isOverwrite()), metadata.getTableMetadata(session, insert.getHandle()).getTable(), insert.isOverwrite());

View File

@ -27,7 +27,9 @@ import io.prestosql.metadata.TableHandle;
import io.prestosql.metadata.UpdateTableHandle;
import io.prestosql.metadata.VacuumTableHandle;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.sql.planner.PartitioningScheme;
import io.prestosql.sql.planner.Symbol;
@ -178,6 +180,7 @@ public class TableWriterNode
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "@type")
@JsonSubTypes({
@JsonSubTypes.Type(value = CreateTarget.class, name = "CreateTarget"),
@JsonSubTypes.Type(value = CreateMaterializedTarget.class, name = "CreateMaterializedTarget"),
@JsonSubTypes.Type(value = InsertTarget.class, name = "InsertTarget"),
@JsonSubTypes.Type(value = UpdateTarget.class, name = "UpdateTarget"),
@JsonSubTypes.Type(value = DeleteTarget.class, name = "DeleteTarget"),
@ -227,6 +230,49 @@ public class TableWriterNode
}
}
public static class CreateMaterializedTarget
extends WriterTarget
{
private final OutputTableHandle handle;
private final SchemaTableName schemaTableName;
private final String originalTargetCatalog;
@JsonCreator
public CreateMaterializedTarget(
@JsonProperty("handle") OutputTableHandle handle,
@JsonProperty("schemaTableName") SchemaTableName schemaTableName,
@JsonProperty("originalTargetCatalog") String originalTargetCatalog)
{
this.handle = requireNonNull(handle, "handle is null");
this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null");
this.originalTargetCatalog = requireNonNull(originalTargetCatalog, "schemaTableName is null");
}
@JsonProperty
public OutputTableHandle getHandle()
{
return handle;
}
@JsonProperty
public SchemaTableName getSchemaTableName()
{
return schemaTableName;
}
@JsonProperty
public String getOriginalTargetCatalog()
{
return originalTargetCatalog;
}
@Override
public String toString()
{
return handle.toString();
}
}
public static class CreateTarget
extends WriterTarget
{
@ -261,6 +307,50 @@ public class TableWriterNode
}
}
// only used during planning -- will not be serialized
public static class CreateMaterializedReference
extends WriterTarget
{
private final String catalog;
private final ConnectorTableMetadata tableMetadata;
private final ConnectorMaterializedViewDefinition definition;
private final Optional<NewTableLayout> layout;
public CreateMaterializedReference(String catalog, ConnectorTableMetadata tableMetadata, Optional<NewTableLayout> layout, ConnectorMaterializedViewDefinition definition)
{
this.catalog = requireNonNull(catalog, "catalog is null");
this.tableMetadata = requireNonNull(tableMetadata, "tableMetadata is null");
this.layout = requireNonNull(layout, "layout is null");
this.definition = definition;
}
public String getCatalog()
{
return catalog;
}
public ConnectorTableMetadata getTableMetadata()
{
return tableMetadata;
}
public Optional<NewTableLayout> getLayout()
{
return layout;
}
public ConnectorViewDefinition getDefinition()
{
return definition;
}
@Override
public String toString()
{
return catalog + "." + tableMetadata.getTable();
}
}
// only used during planning -- will not be serialized
public static class InsertReference
extends WriterTarget

View File

@ -33,6 +33,8 @@ import io.prestosql.sql.planner.plan.PlanNode;
import io.prestosql.sql.planner.plan.PlanVisitor;
import io.prestosql.sql.planner.plan.TableFinishNode;
import io.prestosql.sql.planner.plan.TableScanNode;
import io.prestosql.sql.planner.plan.TableWriterNode;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateMaterializedTarget;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateReference;
import io.prestosql.sql.planner.plan.TableWriterNode.CreateTarget;
import io.prestosql.sql.planner.plan.TableWriterNode.DeleteAsInsertTarget;
@ -494,6 +496,13 @@ public class IoPlanPrinter
target.getSchemaTableName().getSchemaName(),
target.getSchemaTableName().getTableName()));
}
else if (writerTarget instanceof CreateMaterializedTarget) {
CreateMaterializedTarget target = (CreateMaterializedTarget) writerTarget;
context.setOutputTable(new CatalogSchemaTableName(
target.getHandle().getCatalogName().getCatalogName(),
target.getSchemaTableName().getSchemaName(),
target.getSchemaTableName().getTableName()));
}
else if (writerTarget instanceof InsertTarget) {
InsertTarget target = (InsertTarget) writerTarget;
context.setOutputTable(new CatalogSchemaTableName(
@ -522,7 +531,7 @@ public class IoPlanPrinter
target.getSchemaTableName().getSchemaName(),
target.getSchemaTableName().getTableName()));
}
else if (writerTarget instanceof CreateReference || writerTarget instanceof InsertReference || writerTarget instanceof UpdateReference) {
else if (writerTarget instanceof CreateReference || writerTarget instanceof TableWriterNode.CreateMaterializedReference || writerTarget instanceof InsertReference || writerTarget instanceof UpdateReference) {
throw new IllegalStateException(format("%s should not appear in final plan", writerTarget.getClass().getSimpleName()));
}
else {

View File

@ -16,24 +16,28 @@ package io.prestosql.sql.rewrite;
import io.airlift.log.Logger;
import io.prestosql.Session;
import io.prestosql.SystemSessionProperties;
import io.prestosql.connector.DataCenterUtility;
import io.prestosql.execution.warnings.WarningCollector;
import io.prestosql.heuristicindex.HeuristicIndexerManager;
import io.prestosql.metadata.Metadata;
import io.prestosql.metadata.MetadataUtil;
import io.prestosql.metadata.QualifiedObjectName;
import io.prestosql.metadata.TableHandle;
import io.prestosql.security.AccessControl;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.security.AccessDeniedException;
import io.prestosql.sql.analyzer.QueryExplainer;
import io.prestosql.sql.analyzer.SemanticException;
import io.prestosql.sql.parser.ParsingOptions;
import io.prestosql.sql.parser.SqlParser;
import io.prestosql.sql.tree.AliasedRelation;
import io.prestosql.sql.tree.AllColumns;
import io.prestosql.sql.tree.AstVisitor;
import io.prestosql.sql.tree.CreateIndex;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateTableAsSelect;
import io.prestosql.sql.tree.Except;
import io.prestosql.sql.tree.Expression;
@ -46,6 +50,7 @@ import io.prestosql.sql.tree.Node;
import io.prestosql.sql.tree.Query;
import io.prestosql.sql.tree.QueryBody;
import io.prestosql.sql.tree.QuerySpecification;
import io.prestosql.sql.tree.RefreshMaterializedView;
import io.prestosql.sql.tree.Relation;
import io.prestosql.sql.tree.SampledRelation;
import io.prestosql.sql.tree.Select;
@ -68,7 +73,9 @@ import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import static io.prestosql.metadata.MetadataUtil.createQualifiedObjectName;
import static io.prestosql.sql.ParsingUtil.createParsingOptions;
import static io.prestosql.sql.analyzer.SemanticErrorCode.MISSING_TABLE;
import static java.util.Objects.requireNonNull;
public class RowFilterColumnMaskingRewrite
@ -221,7 +228,7 @@ public class RowFilterColumnMaskingRewrite
if (!table.getName().getPrefix().isPresent() && (!session.getCatalog().isPresent() || !session.getSchema().isPresent())) {
return table;
}
QualifiedObjectName qualifiedObjectName = MetadataUtil.createQualifiedObjectName(session, table, table.getName());
QualifiedObjectName qualifiedObjectName = createQualifiedObjectName(session, table, table.getName());
// This section of code is used to load DC sub-catalogs dynamically
DataCenterUtility.loadDCCatalogForQueryFlow(session, metadata, qualifiedObjectName.getCatalogName());
@ -386,6 +393,42 @@ public class RowFilterColumnMaskingRewrite
createTableAsSelect.getComment());
}
@Override
protected Node visitCreateMaterializedView(CreateMaterializedView createMaterializedView, Void context)
{
Query query = (Query) visitQuery(createMaterializedView.getQuery(), context);
return createMaterializedView.getLocation().isPresent() ?
new CreateMaterializedView(createMaterializedView.getLocation().get(), createMaterializedView.getName(), query,
createMaterializedView.isNotExists(), createMaterializedView.getProperties(),
createMaterializedView.getColumnAliases(), createMaterializedView.getComment()) :
new CreateMaterializedView(createMaterializedView.getName(), query, createMaterializedView.isNotExists(),
createMaterializedView.getProperties(), createMaterializedView.getColumnAliases(),
createMaterializedView.getComment());
}
@Override
protected Node visitRefreshMaterializedView(RefreshMaterializedView refreshMaterializedView, Void context)
{
QualifiedObjectName tableName = createQualifiedObjectName(session, refreshMaterializedView, refreshMaterializedView.getTableName());
Optional<ConnectorViewDefinition> definition = metadata.getView(session, tableName);
if (!definition.isPresent()) {
throw new SemanticException(MISSING_TABLE, refreshMaterializedView, "Materialized View '%s' does not exist", tableName.toString());
}
ConnectorMaterializedViewDefinition mvDefinition = (ConnectorMaterializedViewDefinition) definition.get();
if (!mvDefinition.getCatalog().get().equals(
session.getSystemProperties().get(SystemSessionProperties.MATERIALIZED_VIEW_CATALOG_NAME))) {
throw new SemanticException(MISSING_TABLE, refreshMaterializedView,
"Please set session to original catalog, like set session materialized_view_catalog_name = '%s'",
mvDefinition.getCatalog().get());
}
metadata.dropMaterializedView(session, mvDefinition, tableName);
String sql = mvDefinition.getFullSql();
Statement statement = sqlParser.createStatement(sql, createParsingOptions(session));
CreateMaterializedView createMv = (CreateMaterializedView) statement;
return createMv;
}
@Override
protected Node visitInsert(Insert insert, Void context)
{

View File

@ -35,6 +35,7 @@ import io.prestosql.spi.HetuConstant;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.StandardErrorCode;
import io.prestosql.spi.connector.CatalogSchemaName;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.ConnectorViewDefinition;
import io.prestosql.spi.connector.SchemaTableName;
@ -54,6 +55,7 @@ import io.prestosql.sql.tree.ArrayConstructor;
import io.prestosql.sql.tree.AstVisitor;
import io.prestosql.sql.tree.BooleanLiteral;
import io.prestosql.sql.tree.ColumnDefinition;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateTable;
import io.prestosql.sql.tree.CreateView;
import io.prestosql.sql.tree.DoubleLiteral;
@ -136,6 +138,7 @@ import static io.prestosql.sql.analyzer.SemanticErrorCode.NOT_SUPPORTED;
import static io.prestosql.sql.analyzer.SemanticErrorCode.VIEW_PARSE_ERROR;
import static io.prestosql.sql.tree.BooleanLiteral.FALSE_LITERAL;
import static io.prestosql.sql.tree.BooleanLiteral.TRUE_LITERAL;
import static io.prestosql.sql.tree.ShowCreate.Type.MATERIALIZED;
import static io.prestosql.sql.tree.ShowCreate.Type.TABLE;
import static io.prestosql.sql.tree.ShowCreate.Type.VIEW;
import static java.lang.String.format;
@ -441,6 +444,29 @@ final class ShowQueriesRewrite
{
QualifiedObjectName objectName = createQualifiedObjectName(session, node, node.getName());
Optional<ConnectorViewDefinition> viewDefinition = metadata.getView(session, objectName);
StringBuilder header = new StringBuilder();
if (node.getType() == MATERIALIZED) {
if (!viewDefinition.isPresent()) {
if (metadata.getTableHandle(session, objectName).isPresent()) {
throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a table, not a materialized view", objectName);
}
throw new SemanticException(MISSING_TABLE, node, "Materialized View '%s' does not exist", objectName);
}
ConnectorMaterializedViewDefinition definition = (ConnectorMaterializedViewDefinition) viewDefinition.get();
Query query = parseView(definition.getOriginalSql(), objectName, node);
CreateMaterializedView createMaterializedView = new CreateMaterializedView(
QualifiedName.of(objectName.getCatalogName(), objectName.getSchemaName(), objectName.getObjectName()),
query,
false,
Collections.emptyList(),
Optional.empty(),
Optional.empty());
String sql = formatSql(createMaterializedView, Optional.of(parameters)).trim();
return singleValueQuery("Create Materialized View", sql);
}
if (node.getType() == VIEW) {
if (!viewDefinition.isPresent()) {
@ -459,7 +485,7 @@ final class ShowQueriesRewrite
return singleValueQuery("Create View", sql);
}
if (node.getType() == TABLE) {
if (node.getType() == TABLE || node.getType() == MATERIALIZED) {
if (viewDefinition.isPresent()) {
throw new SemanticException(NOT_SUPPORTED, node, "Relation '%s' is a view, not a table", objectName);
}
@ -473,10 +499,11 @@ final class ShowQueriesRewrite
Map<String, PropertyMetadata<?>> allColumnProperties = metadata.getColumnPropertyManager().getAllProperties().get(tableHandle.get().getCatalogName());
QualifiedObjectName finalObjectName = objectName;
List<TableElement> columns = connectorTableMetadata.getColumns().stream()
.filter(column -> !column.isHidden())
.map(column -> {
List<Property> propertyNodes = buildProperties(objectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties);
List<Property> propertyNodes = buildProperties(finalObjectName, Optional.of(column.getName()), INVALID_COLUMN_PROPERTY, column.getProperties(), allColumnProperties);
return new ColumnDefinition(new Identifier(column.getName()), column.getType().getDisplayName(), column.isNullable(), propertyNodes, Optional.ofNullable(column.getComment()));
})
.collect(toImmutableList());
@ -494,7 +521,7 @@ final class ShowQueriesRewrite
return singleValueQuery("Create Table", formatSql(createTable, Optional.of(parameters)).trim());
}
throw new UnsupportedOperationException("SHOW CREATE only supported for tables and views");
throw new UnsupportedOperationException("SHOW CREATE only supported for tables, views and materialized views");
}
private List<Property> buildProperties(

View File

@ -21,6 +21,7 @@ import io.airlift.http.server.HttpServerConfig;
import io.airlift.http.server.HttpServerInfo;
import io.airlift.node.NodeInfo;
import io.airlift.units.Duration;
import io.omnicache.jdbc.OmniCacheJdbcImpl;
import io.prestosql.GroupByHashPageIndexerFactory;
import io.prestosql.PagesIndexPageSorter;
import io.prestosql.Session;
@ -52,6 +53,7 @@ import io.prestosql.execution.CreateTableTask;
import io.prestosql.execution.CreateViewTask;
import io.prestosql.execution.DataDefinitionTask;
import io.prestosql.execution.DeallocateTask;
import io.prestosql.execution.DropMaterializedViewTask;
import io.prestosql.execution.DropTableTask;
import io.prestosql.execution.DropViewTask;
import io.prestosql.execution.Lifespan;
@ -161,6 +163,7 @@ import io.prestosql.sql.tree.Commit;
import io.prestosql.sql.tree.CreateTable;
import io.prestosql.sql.tree.CreateView;
import io.prestosql.sql.tree.Deallocate;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropTable;
import io.prestosql.sql.tree.DropView;
import io.prestosql.sql.tree.Prepare;
@ -374,7 +377,8 @@ public class LocalQueryRunner
null,
new ServerConfig(),
new NodeSchedulerConfig(),
heuristicIndexerManager);
heuristicIndexerManager,
new OmniCacheJdbcImpl());
GlobalSystemConnectorFactory globalSystemConnectorFactory = new GlobalSystemConnectorFactory(ImmutableSet.of(
new NodeSystemTable(nodeManager),
@ -448,6 +452,7 @@ public class LocalQueryRunner
.put(CreateView.class, new CreateViewTask(sqlParser, featuresConfig))
.put(DropTable.class, new DropTableTask())
.put(DropView.class, new DropViewTask())
.put(DropMaterializedView.class, new DropMaterializedViewTask())
.put(RenameColumn.class, new RenameColumnTask())
.put(RenameTable.class, new RenameTableTask())
.put(RenameIndex.class, new RenameIndexTask())
@ -908,7 +913,7 @@ public class LocalQueryRunner
LogicalPlanner logicalPlanner = new LogicalPlanner(session, optimizers, new PlanSanityChecker(true), idAllocator, metadata, new TypeAnalyzer(sqlParser, metadata), statsCalculator, costCalculator, warningCollector);
Analysis analysis = analyzer.analyze(preparedQuery.getStatement());
return logicalPlanner.plan(analysis, stage);
return logicalPlanner.plan(analysis, sql, stage);
}
private static List<Split> getNextBatch(SplitSource splitSource)

View File

@ -22,6 +22,7 @@ import io.prestosql.sql.tree.Call;
import io.prestosql.sql.tree.Comment;
import io.prestosql.sql.tree.Commit;
import io.prestosql.sql.tree.CreateIndex;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateRole;
import io.prestosql.sql.tree.CreateSchema;
import io.prestosql.sql.tree.CreateTable;
@ -34,6 +35,7 @@ import io.prestosql.sql.tree.DescribeOutput;
import io.prestosql.sql.tree.DropCache;
import io.prestosql.sql.tree.DropColumn;
import io.prestosql.sql.tree.DropIndex;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropRole;
import io.prestosql.sql.tree.DropSchema;
import io.prestosql.sql.tree.DropTable;
@ -44,6 +46,7 @@ import io.prestosql.sql.tree.GrantRoles;
import io.prestosql.sql.tree.Insert;
import io.prestosql.sql.tree.Prepare;
import io.prestosql.sql.tree.Query;
import io.prestosql.sql.tree.RefreshMaterializedView;
import io.prestosql.sql.tree.RenameColumn;
import io.prestosql.sql.tree.RenameIndex;
import io.prestosql.sql.tree.RenameSchema;
@ -95,6 +98,8 @@ public final class StatementUtils
builder.put(Analyze.class, QueryType.ANALYZE);
builder.put(CreateTableAsSelect.class, QueryType.INSERT);
builder.put(CreateMaterializedView.class, QueryType.INSERT);
builder.put(RefreshMaterializedView.class, QueryType.INSERT);
builder.put(Insert.class, QueryType.INSERT);
builder.put(Update.class, QueryType.UPDATE);
@ -131,6 +136,7 @@ public final class StatementUtils
builder.put(DropIndex.class, QueryType.DATA_DEFINITION);
builder.put(CreateView.class, QueryType.DATA_DEFINITION);
builder.put(DropView.class, QueryType.DATA_DEFINITION);
builder.put(DropMaterializedView.class, QueryType.DATA_DEFINITION);
builder.put(Use.class, QueryType.DATA_DEFINITION);
builder.put(SetSession.class, QueryType.DATA_DEFINITION);
builder.put(ResetSession.class, QueryType.DATA_DEFINITION);

View File

@ -154,7 +154,8 @@ abstract class SimulationSplit
task.splitComplete(this);
}
return Futures.immediateCheckedFuture(null);
//return Futures.immediateCheckedFuture(null);
return Futures.immediateFuture(null);
}
ListenableFuture<?> processResult = getProcessResult();

View File

@ -27,6 +27,7 @@ import io.prestosql.spi.connector.CatalogSchemaName;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorCapabilities;
import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition;
import io.prestosql.spi.connector.ConnectorMetadata;
import io.prestosql.spi.connector.ConnectorOutputMetadata;
import io.prestosql.spi.connector.ConnectorTableMetadata;
@ -36,6 +37,7 @@ import io.prestosql.spi.connector.ConstraintApplicationResult;
import io.prestosql.spi.connector.LimitApplicationResult;
import io.prestosql.spi.connector.ProjectionApplicationResult;
import io.prestosql.spi.connector.SampleType;
import io.prestosql.spi.connector.SchemaTableName;
import io.prestosql.spi.connector.SubQueryApplicationResult;
import io.prestosql.spi.connector.SystemTable;
import io.prestosql.spi.expression.ConnectorExpression;
@ -708,4 +710,28 @@ public abstract class AbstractMockMetadata
{
return true;
}
@Override
public void dropMaterializedView(Session session, ConnectorMaterializedViewDefinition definition, QualifiedObjectName name)
{
throw new UnsupportedOperationException();
}
@Override
public void refreshMaterializedView(Session session, TableHandle tableHandle)
{
throw new UnsupportedOperationException();
}
@Override
public OutputTableHandle beginCreateMaterializedView(Session session, String catalogName, ConnectorTableMetadata tableMetadata, ConnectorMaterializedViewDefinition definition, Optional<NewTableLayout> layout)
{
throw new UnsupportedOperationException();
}
@Override
public Optional<ConnectorOutputMetadata> finishCreateMaterializedView(Session session, OutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics, String originalTargetCatalog, SchemaTableName name)
{
throw new UnsupportedOperationException();
}
}

View File

@ -135,6 +135,21 @@ public class TestingRangerAccessControl
{
}
@Override
public void checkCanCreateMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName, Set<String> columnNames)
{
}
@Override
public void checkCanRefreshMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanDropMaterializedTable(TransactionId transactionId, Identity identity, QualifiedObjectName tableName)
{
}
@Override
public void checkCanShowTablesMetadata(TransactionId transactionId, Identity identity, CatalogSchemaName schema)
{

View File

@ -18,7 +18,7 @@
<parent>
<artifactId>presto-root</artifactId>
<groupId>io.hetu.core</groupId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-matching</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-memory-context</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-memory</artifactId>

View File

@ -4,7 +4,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-ml</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-mysql</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-orc</artifactId>
@ -126,10 +126,16 @@
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-main</artifactId>
<exclusions>
<exclusion>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
</dependency>
<dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-parser</artifactId>
<scope>test</scope>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-parquet</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-parser</artifactId>

View File

@ -48,6 +48,11 @@ statement
(COMMENT string)?
(WITH properties)? #createTable
| DROP TABLE (IF EXISTS)? qualifiedName #dropTable
| CREATE MATERIALIZED VIEW (IF NOT EXISTS)? qualifiedName columnAliases?
(COMMENT string)?
(WITH properties)? AS (query | '('query')') #createMaterializedView
| REFRESH MATERIALIZED VIEW (CONCURRENTLY)? qualifiedName #refreshMaterializedView
| DROP MATERIALIZED VIEW (IF NOT EXISTS)? qualifiedName #dropMaterializedView
| CACHE TABLE tableName=qualifiedName
(WHERE booleanExpression)?
(WITH properties)? #cacheTable
@ -109,6 +114,7 @@ statement
('(' explainOption (',' explainOption)* ')')? statement #explain
| SHOW CREATE TABLE qualifiedName #showCreateTable
| SHOW CREATE VIEW qualifiedName #showCreateView
| SHOW CREATE MATERIALIZED VIEW qualifiedName #showCreateMaterializedView
| SHOW TABLES ((FROM | IN) qualifiedName)?
(LIKE pattern=string (ESCAPE escape=string)?)? #showTables
| SHOW (SCHEMAS | DATABASES) ((FROM | IN) (cluster=identifier '.')? catalog=identifier)?
@ -541,7 +547,7 @@ nonReserved
| SHOW | SOME | START | STATS | SUBSTRING | SYSTEM
| TABLES | TABLESAMPLE | TEXT | TIES | TIME | TIMESTAMP | TO | TRANSACTION | TRY_CAST | TYPE
| UNBOUNDED | UNCOMMITTED | USE | USER
| VALIDATE | VERBOSE | VIEW
| VALIDATE | VERBOSE | VIEW | MATERIALIZED
| WORK | WRITE
| YEAR
| ZONE
@ -749,6 +755,8 @@ WORK: 'WORK';
WRITE: 'WRITE';
YEAR: 'YEAR';
ZONE: 'ZONE';
MATERIALIZED: 'MATERIALIZED';
REFRESH: 'REFRESH';
INDEX: 'INDEX';
BITMAP: 'BITMAP';
BLOOM: 'BLOOM';

View File

@ -28,6 +28,7 @@ import io.prestosql.sql.tree.ColumnDefinition;
import io.prestosql.sql.tree.Comment;
import io.prestosql.sql.tree.Commit;
import io.prestosql.sql.tree.CreateIndex;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateRole;
import io.prestosql.sql.tree.CreateSchema;
import io.prestosql.sql.tree.CreateTable;
@ -40,6 +41,7 @@ import io.prestosql.sql.tree.DescribeOutput;
import io.prestosql.sql.tree.DropCache;
import io.prestosql.sql.tree.DropColumn;
import io.prestosql.sql.tree.DropIndex;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropRole;
import io.prestosql.sql.tree.DropSchema;
import io.prestosql.sql.tree.DropTable;
@ -651,6 +653,18 @@ public final class SqlFormatter
return null;
}
@Override
protected Void visitDropMaterializedView(DropMaterializedView node, Integer context)
{
builder.append("DROP MATERIALIZED VIEW ");
if (node.isExists()) {
builder.append("IF EXISTS ");
}
builder.append(node.getTableName());
return null;
}
@Override
protected Void visitExplain(Explain node, Integer indent)
{
@ -932,6 +946,31 @@ public final class SqlFormatter
return null;
}
@Override
protected Void visitCreateMaterializedView(CreateMaterializedView node, Integer indent)
{
builder.append("CREATE MATERIALIZED VIEW ");
if (node.isNotExists()) {
builder.append("IF NOT EXISTS ");
}
builder.append(formatName(node.getName()));
if (node.getColumnAliases().isPresent()) {
String columnList = node.getColumnAliases().get().stream().map(element -> formatExpression(element, parameters)).collect(joining(", "));
builder.append(format("( %s )", columnList));
}
if (node.getComment().isPresent()) {
builder.append("\nCOMMENT " + formatStringLiteral(node.getComment().get()));
}
builder.append(formatPropertiesSingleLine(node.getProperties()));
builder.append(" AS \n");
process(node.getQuery(), indent);
return null;
}
@Override
protected Void visitCreateTable(CreateTable node, Integer indent)
{

View File

@ -41,6 +41,7 @@ import io.prestosql.sql.tree.Comment;
import io.prestosql.sql.tree.Commit;
import io.prestosql.sql.tree.ComparisonExpression;
import io.prestosql.sql.tree.CreateIndex;
import io.prestosql.sql.tree.CreateMaterializedView;
import io.prestosql.sql.tree.CreateRole;
import io.prestosql.sql.tree.CreateSchema;
import io.prestosql.sql.tree.CreateTable;
@ -60,6 +61,7 @@ import io.prestosql.sql.tree.DoubleLiteral;
import io.prestosql.sql.tree.DropCache;
import io.prestosql.sql.tree.DropColumn;
import io.prestosql.sql.tree.DropIndex;
import io.prestosql.sql.tree.DropMaterializedView;
import io.prestosql.sql.tree.DropRole;
import io.prestosql.sql.tree.DropSchema;
import io.prestosql.sql.tree.DropTable;
@ -126,6 +128,7 @@ import io.prestosql.sql.tree.QuantifiedComparisonExpression;
import io.prestosql.sql.tree.Query;
import io.prestosql.sql.tree.QueryBody;
import io.prestosql.sql.tree.QuerySpecification;
import io.prestosql.sql.tree.RefreshMaterializedView;
import io.prestosql.sql.tree.Relation;
import io.prestosql.sql.tree.RenameColumn;
import io.prestosql.sql.tree.RenameIndex;
@ -276,6 +279,52 @@ class AstBuilder
(Identifier) visit(context.identifier()));
}
@Override
public Node visitCreateMaterializedView(SqlBaseParser.CreateMaterializedViewContext context)
{
Optional<String> comment = Optional.empty();
if (context.COMMENT() != null) {
comment = Optional.of(((StringLiteral) visit(context.string())).getValue());
}
Optional<List<Identifier>> columnAliases = Optional.empty();
if (context.columnAliases() != null) {
columnAliases = Optional.of(visit(context.columnAliases().identifier(), Identifier.class));
}
List<Property> properties = ImmutableList.of();
if (context.properties() != null) {
properties = visit(context.properties().property(), Property.class);
}
return new CreateMaterializedView(
getLocation(context),
getQualifiedName(context.qualifiedName()),
(Query) visit(context.query()),
context.EXISTS() != null,
properties,
columnAliases,
comment);
}
@Override
public Node visitDropMaterializedView(SqlBaseParser.DropMaterializedViewContext context)
{
return new DropMaterializedView(getLocation(context), getQualifiedName(context.qualifiedName()), context.EXISTS() != null);
}
@Override
public Node visitRefreshMaterializedView(SqlBaseParser.RefreshMaterializedViewContext context)
{
return new RefreshMaterializedView(getLocation(context), getQualifiedName(context.qualifiedName()), context.CONCURRENTLY() != null);
}
@Override
public Node visitShowCreateMaterializedView(SqlBaseParser.ShowCreateMaterializedViewContext context)
{
return new ShowCreate(getLocation(context), ShowCreate.Type.MATERIALIZED, getQualifiedName(context.qualifiedName()));
}
@Override
public Node visitCreateTableAsSelect(SqlBaseParser.CreateTableAsSelectContext context)
{

View File

@ -642,6 +642,21 @@ public abstract class AstVisitor<R, C>
return visitStatement(node, context);
}
protected R visitCreateMaterializedView(CreateMaterializedView node, C context)
{
return visitStatement(node, context);
}
protected R visitDropMaterializedView(DropMaterializedView node, C context)
{
return visitStatement(node, context);
}
protected R visitRefreshMaterializedView(RefreshMaterializedView node, C context)
{
return visitStatement(node, context);
}
protected R visitInsert(Insert node, C context)
{
return visitStatement(node, context);

View File

@ -0,0 +1,153 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.tree;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import static com.google.common.base.MoreObjects.toStringHelper;
import static java.util.Objects.requireNonNull;
public class CreateMaterializedView
extends Statement
{
private final QualifiedName name;
private final Query query;
private final boolean notExists;
private final List<Property> properties;
private final Optional<List<Identifier>> columnAliases;
private final Optional<String> comment;
public static final String CATALOG_NAME_PROPERTY = "catalog";
public CreateMaterializedView(QualifiedName name, Query query, boolean notExists, List<Property> properties, Optional<List<Identifier>> columnAliases, Optional<String> comment)
{
this(Optional.empty(), name, query, notExists, properties, columnAliases, comment);
}
public CreateMaterializedView(NodeLocation location, QualifiedName name, Query query, boolean notExists, List<Property> properties, Optional<List<Identifier>> columnAliases, Optional<String> comment)
{
this(Optional.of(location), name, query, notExists, properties, columnAliases, comment);
}
private CreateMaterializedView(Optional<NodeLocation> location, QualifiedName name, Query query, boolean notExists, List<Property> properties, Optional<List<Identifier>> columnAliases, Optional<String> comment)
{
super(location);
this.name = requireNonNull(name, "name is null");
this.query = requireNonNull(query, "query is null");
this.notExists = notExists;
this.properties = new ArrayList<Property>(requireNonNull(properties, "properties is null"));
this.columnAliases = columnAliases;
this.comment = requireNonNull(comment, "comment is null");
}
public QualifiedName getName()
{
return name;
}
public Query getQuery()
{
return query;
}
public boolean isNotExists()
{
return notExists;
}
public List<Property> getProperties()
{
return properties;
}
public Optional<List<Identifier>> getColumnAliases()
{
return columnAliases;
}
public Optional<String> getComment()
{
return comment;
}
@Override
public <R, C> R accept(AstVisitor<R, C> visitor, C context)
{
return visitor.visitCreateMaterializedView(this, context);
}
@Override
public List<Node> getChildren()
{
return ImmutableList.<Node>builder()
.add(query)
.addAll(properties)
.build();
}
@Override
public int hashCode()
{
return Objects.hash(name, query, properties, columnAliases, comment);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
io.prestosql.sql.tree.CreateMaterializedView o = (io.prestosql.sql.tree.CreateMaterializedView) obj;
return Objects.equals(name, o.name)
&& Objects.equals(query, o.query)
&& Objects.equals(notExists, o.notExists)
&& Objects.equals(properties, o.properties)
&& Objects.equals(columnAliases, o.columnAliases)
&& Objects.equals(comment, o.comment);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("name", name)
.add("query", query)
.add("notExists", notExists)
.add("properties", properties)
.add("columnAliases", columnAliases)
.add("comment", comment)
.toString();
}
public String getTargetCatalogName(String configuredCatalogName)
{
for (Property property : properties) {
if (property.getName().toString().equals(io.prestosql.sql.tree.CreateMaterializedView.CATALOG_NAME_PROPERTY)) {
String name = property.getValue().toString().replaceAll("'", "");
properties.remove(property);
return name;
}
}
return configuredCatalogName;
}
}

View File

@ -0,0 +1,99 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.tree;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import static com.google.common.base.MoreObjects.toStringHelper;
public class DropMaterializedView
extends Statement
{
private final QualifiedName tableName;
private final boolean exists;
public DropMaterializedView(QualifiedName tableName, boolean exists)
{
this(Optional.empty(), tableName, exists);
}
public DropMaterializedView(NodeLocation location, QualifiedName tableName, boolean exists)
{
this(Optional.of(location), tableName, exists);
}
private DropMaterializedView(Optional<NodeLocation> location, QualifiedName tableName, boolean exists)
{
super(location);
this.tableName = tableName;
this.exists = exists;
}
public QualifiedName getTableName()
{
return tableName;
}
public boolean isExists()
{
return exists;
}
@Override
public <R, C> R accept(AstVisitor<R, C> visitor, C context)
{
return visitor.visitDropMaterializedView(this, context);
}
@Override
public List<Node> getChildren()
{
return ImmutableList.of();
}
@Override
public int hashCode()
{
return Objects.hash(tableName, exists);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
DropMaterializedView o = (DropMaterializedView) obj;
return Objects.equals(tableName, o.tableName)
&& (exists == o.exists);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("tableName", tableName)
.add("exists", exists)
.toString();
}
}

View File

@ -0,0 +1,94 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.prestosql.sql.tree;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import static com.google.common.base.MoreObjects.toStringHelper;
public class RefreshMaterializedView
extends Statement
{
private final QualifiedName tableName;
private final boolean concurrently;
public RefreshMaterializedView(NodeLocation location, QualifiedName tableName, boolean concurrently)
{
this(Optional.of(location), tableName, concurrently);
}
private RefreshMaterializedView(Optional<NodeLocation> location, QualifiedName tableName, boolean concurrently)
{
super(location);
this.tableName = tableName;
this.concurrently = concurrently;
}
public QualifiedName getTableName()
{
return tableName;
}
public boolean isConcurrently()
{
return concurrently;
}
@Override
public <R, C> R accept(AstVisitor<R, C> visitor, C context)
{
return visitor.visitRefreshMaterializedView(this, context);
}
@Override
public List<Node> getChildren()
{
return ImmutableList.of();
}
@Override
public int hashCode()
{
return Objects.hash(tableName, concurrently);
}
@Override
public boolean equals(Object obj)
{
if (this == obj) {
return true;
}
if ((obj == null) || (getClass() != obj.getClass())) {
return false;
}
RefreshMaterializedView o = (RefreshMaterializedView) obj;
return Objects.equals(tableName, o.tableName)
&& (concurrently == o.concurrently);
}
@Override
public String toString()
{
return toStringHelper(this)
.add("tableName", tableName)
.add("concurrently", concurrently)
.toString();
}
}

View File

@ -28,7 +28,8 @@ public class ShowCreate
public enum Type
{
TABLE,
VIEW
VIEW,
MATERIALIZED
}
private final Type type;

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-password-authenticators</artifactId>

View File

@ -5,7 +5,7 @@
<parent>
<groupId>io.hetu.core</groupId>
<artifactId>presto-root</artifactId>
<version>1.0.0-SNAPSHOT</version>
<version>1.2.0-SNAPSHOT</version>
</parent>
<artifactId>presto-plugin-toolkit</artifactId>

Some files were not shown because too many files have changed in this diff Show More