From 0ec44314ad9ec822ddacb6c96c3f54eb92e15015 Mon Sep 17 00:00:00 2001 From: lizheng920625 Date: Mon, 29 Mar 2021 09:21:25 +0800 Subject: [PATCH] Init commits for materialized view --- hetu-carbondata/pom.xml | 2 +- hetu-common/pom.xml | 2 +- hetu-datacenter/pom.xml | 2 +- hetu-filesystem-client/pom.xml | 2 +- hetu-hana/pom.xml | 2 +- hetu-hazelcast/pom.xml | 2 +- hetu-hbase/pom.xml | 8 +- hetu-heuristic-index/pom.xml | 2 +- hetu-hive-functions/pom.xml | 2 +- hetu-listener/pom.xml | 2 +- hetu-metastore/pom.xml | 2 +- hetu-mongodb/pom.xml | 2 +- hetu-mv/pom.xml | 81 +++++ .../TestLocalMvMetastore.java | 102 ++++++ .../TestMaterializedView.java | 94 ++++++ .../TestMaterializedViewConfig.java | 36 +++ .../TestMvConnectorMetadata.java | 242 ++++++++++++++ .../hetu/core/materializedview/TestUtils.java | 73 +++++ hetu-opengauss/pom.xml | 4 +- hetu-oracle/pom.xml | 2 +- hetu-seed-store/pom.xml | 2 +- hetu-server-rpm/pom.xml | 2 +- hetu-server/pom.xml | 2 +- hetu-server/src/main/provisio/hetu.xml | 6 + hetu-sql-migration-tool/pom.xml | 2 +- hetu-state-store/pom.xml | 2 +- hetu-transport/pom.xml | 2 +- hetu-vdm/pom.xml | 2 +- pom.xml | 7 +- presto-array/pom.xml | 2 +- presto-atop/pom.xml | 2 +- presto-base-jdbc/pom.xml | 2 +- presto-benchmark-driver/pom.xml | 2 +- presto-benchmark/pom.xml | 2 +- presto-benchto-benchmarks/pom.xml | 2 +- presto-cli/pom.xml | 2 +- presto-client/pom.xml | 2 +- presto-elasticsearch/pom.xml | 2 +- presto-example-http/pom.xml | 2 +- presto-geospatial-toolkit/pom.xml | 2 +- presto-geospatial/pom.xml | 2 +- presto-hive-hadoop2/pom.xml | 2 +- presto-hive/pom.xml | 2 +- presto-jdbc/pom.xml | 2 +- presto-jmx/pom.xml | 2 +- presto-kafka/pom.xml | 2 +- presto-local-file/pom.xml | 2 +- presto-main/pom.xml | 41 ++- .../io/prestosql/SystemSessionProperties.java | 31 ++ .../prestosql/connector/ConnectorManager.java | 32 +- .../prestosql/dispatcher/DispatchManager.java | 116 ++++++- .../execution/DropMaterializedViewTask.java | 74 +++++ .../execution/QueryStateMachine.java | 5 + .../execution/SqlQueryExecution.java | 7 +- .../java/io/prestosql/metadata/Metadata.java | 26 ++ .../prestosql/metadata/MetadataManager.java | 64 ++++ .../operator/TableWriterOperator.java | 6 +- .../query/CachedSqlQueryExecution.java | 2 +- .../prestosql/query/HetuLogicalPlanner.java | 4 +- .../io/prestosql/security/AccessControl.java | 21 ++ .../security/AccessControlManager.java | 18 ++ .../security/AllowAllAccessControl.java | 15 + .../security/DenyAllAccessControl.java | 15 + .../prestosql/server/CoordinatorModule.java | 3 + .../io/prestosql/server/ServerConfig.java | 15 + .../io/prestosql/server/ServerMainModule.java | 5 + .../sql/analyzer/FeaturesConfig.java | 15 + .../sql/analyzer/QueryExplainer.java | 2 +- .../sql/analyzer/StatementAnalyzer.java | 62 ++++ .../sql/planner/LocalExecutionPlanner.java | 5 + .../prestosql/sql/planner/LogicalPlanner.java | 92 +++++- .../sql/planner/OutputExtractor.java | 7 + .../optimizations/BeginTableWrite.java | 9 + .../sql/planner/plan/TableWriterNode.java | 90 ++++++ .../planner/planprinter/IoPlanPrinter.java | 11 +- .../RowFilterColumnMaskingRewrite.java | 47 ++- .../sql/rewrite/ShowQueriesRewrite.java | 33 +- .../prestosql/testing/LocalQueryRunner.java | 9 +- .../io/prestosql/util/StatementUtils.java | 6 + .../execution/executor/SimulationSplit.java | 3 +- .../metadata/AbstractMockMetadata.java | 26 ++ .../rewrite/TestingRangerAccessControl.java | 15 + presto-matching/pom.xml | 2 +- presto-memory-context/pom.xml | 2 +- presto-memory/pom.xml | 2 +- presto-ml/pom.xml | 2 +- presto-mysql/pom.xml | 2 +- presto-orc/pom.xml | 10 +- presto-parquet/pom.xml | 2 +- presto-parser/pom.xml | 2 +- .../antlr4/io/prestosql/sql/parser/SqlBase.g4 | 10 +- .../java/io/prestosql/sql/SqlFormatter.java | 39 +++ .../io/prestosql/sql/parser/AstBuilder.java | 49 +++ .../io/prestosql/sql/tree/AstVisitor.java | 15 + .../sql/tree/CreateMaterializedView.java | 153 +++++++++ .../sql/tree/DropMaterializedView.java | 99 ++++++ .../sql/tree/RefreshMaterializedView.java | 94 ++++++ .../io/prestosql/sql/tree/ShowCreate.java | 3 +- presto-password-authenticators/pom.xml | 2 +- presto-plugin-toolkit/pom.xml | 2 +- presto-postgresql/pom.xml | 2 +- presto-product-tests/pom.xml | 10 +- presto-proxy/pom.xml | 2 +- presto-rcfile/pom.xml | 2 +- presto-record-decoder/pom.xml | 2 +- presto-resource-group-managers/pom.xml | 2 +- presto-session-property-managers/pom.xml | 2 +- presto-spi/pom.xml | 2 +- .../ConnectorMaterializedViewDefinition.java | 102 ++++++ .../spi/connector/ConnectorMetadata.java | 36 +++ .../ClassLoaderSafeConnectorMetadata.java | 33 ++ presto-sqlserver/pom.xml | 2 +- presto-teradata-functions/pom.xml | 2 +- presto-testing-docker/pom.xml | 2 +- presto-testing-server-launcher/pom.xml | 2 +- presto-tests/pom.xml | 2 +- presto-thrift-api/pom.xml | 2 +- presto-thrift-testing-server/pom.xml | 2 +- presto-thrift/pom.xml | 2 +- presto-tpcds/pom.xml | 2 +- presto-tpch/pom.xml | 2 +- presto-verifier/pom.xml | 2 +- .../MaterializedViewModule.java | 71 +++++ .../MaterializedViewPlugin.java | 52 ++++ .../conf/MaterializedViewConfig.java | 51 +++ .../MaterializedViewColumnHandle.java | 110 +++++++ .../connector/MaterializedViewConnector.java | 62 ++++ .../MaterializedViewConnectorFactory.java | 89 ++++++ ...erializedViewConnectorHandlerResolver.java | 43 +++ .../connector/MaterializedViewHandle.java | 98 ++++++ .../MaterializedViewTransactionHandle.java | 24 ++ .../HetuMaterializedViewMetastore.java | 294 ++++++++++++++++++ .../LocalMaterializedViewMetastore.java | 236 ++++++++++++++ .../metadata/MaterializedView.java | 231 ++++++++++++++ .../MaterializedViewConnectorMetadata.java | 215 +++++++++++++ ...erializedViewConnectorMetadataFactory.java | 50 +++ .../metadata/MaterializedViewMetastore.java | 48 +++ .../MaterializedViewMetastoreFactory.java | 66 ++++ .../utils/MaterializedViewConstants.java | 52 ++++ .../utils/MaterializedViewDateUtils.java | 62 ++++ .../utils/MaterializedViewErrorCode.java | 49 +++ .../utils/MaterializedViewJsonUtils.java | 107 +++++++ .../utils/MaterializedViewMetadataSpec.java | 48 +++ .../utils/MaterializedViewStatus.java | 73 +++++ 144 files changed, 4389 insertions(+), 108 deletions(-) create mode 100644 hetu-mv/pom.xml create mode 100644 hetu-mv/src/test/java/io/hetu/core/materializedview/TestLocalMvMetastore.java create mode 100644 hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedView.java create mode 100644 hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedViewConfig.java create mode 100644 hetu-mv/src/test/java/io/hetu/core/materializedview/TestMvConnectorMetadata.java create mode 100644 hetu-mv/src/test/java/io/hetu/core/materializedview/TestUtils.java create mode 100644 presto-main/src/main/java/io/prestosql/execution/DropMaterializedViewTask.java create mode 100644 presto-parser/src/main/java/io/prestosql/sql/tree/CreateMaterializedView.java create mode 100644 presto-parser/src/main/java/io/prestosql/sql/tree/DropMaterializedView.java create mode 100644 presto-parser/src/main/java/io/prestosql/sql/tree/RefreshMaterializedView.java create mode 100644 presto-spi/src/main/java/io/prestosql/spi/connector/ConnectorMaterializedViewDefinition.java create mode 100644 src/main/java/io/hetu/core/materializedview/MaterializedViewModule.java create mode 100644 src/main/java/io/hetu/core/materializedview/MaterializedViewPlugin.java create mode 100644 src/main/java/io/hetu/core/materializedview/conf/MaterializedViewConfig.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewColumnHandle.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewConnector.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewConnectorFactory.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewConnectorHandlerResolver.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewHandle.java create mode 100644 src/main/java/io/hetu/core/materializedview/connector/MaterializedViewTransactionHandle.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/HetuMaterializedViewMetastore.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/LocalMaterializedViewMetastore.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/MaterializedView.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/MaterializedViewConnectorMetadata.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/MaterializedViewConnectorMetadataFactory.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/MaterializedViewMetastore.java create mode 100644 src/main/java/io/hetu/core/materializedview/metadata/MaterializedViewMetastoreFactory.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewConstants.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewDateUtils.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewErrorCode.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewJsonUtils.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewMetadataSpec.java create mode 100644 src/main/java/io/hetu/core/materializedview/utils/MaterializedViewStatus.java diff --git a/hetu-carbondata/pom.xml b/hetu-carbondata/pom.xml index 3b602bc11..c219d7e71 100755 --- a/hetu-carbondata/pom.xml +++ b/hetu-carbondata/pom.xml @@ -22,7 +22,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-carbondata diff --git a/hetu-common/pom.xml b/hetu-common/pom.xml index b0403d88f..85ba92d22 100644 --- a/hetu-common/pom.xml +++ b/hetu-common/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-common diff --git a/hetu-datacenter/pom.xml b/hetu-datacenter/pom.xml index 3105c2f60..cd966fbcb 100644 --- a/hetu-datacenter/pom.xml +++ b/hetu-datacenter/pom.xml @@ -4,7 +4,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-datacenter diff --git a/hetu-filesystem-client/pom.xml b/hetu-filesystem-client/pom.xml index fb30680b6..bd072e3ed 100644 --- a/hetu-filesystem-client/pom.xml +++ b/hetu-filesystem-client/pom.xml @@ -3,7 +3,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-hana/pom.xml b/hetu-hana/pom.xml index dc8c69e57..085541d90 100644 --- a/hetu-hana/pom.xml +++ b/hetu-hana/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-hana diff --git a/hetu-hazelcast/pom.xml b/hetu-hazelcast/pom.xml index 312b7830d..11002b9b4 100644 --- a/hetu-hazelcast/pom.xml +++ b/hetu-hazelcast/pom.xml @@ -5,7 +5,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-hazelcast diff --git a/hetu-hbase/pom.xml b/hetu-hbase/pom.xml index e815af796..943b0b572 100644 --- a/hetu-hbase/pom.xml +++ b/hetu-hbase/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-hbase @@ -297,6 +297,12 @@ io.hetu.core presto-tests + + + com.google.protobuf + protobuf-java + + test diff --git a/hetu-heuristic-index/pom.xml b/hetu-heuristic-index/pom.xml index 03f0b1a03..9f73f83d2 100644 --- a/hetu-heuristic-index/pom.xml +++ b/hetu-heuristic-index/pom.xml @@ -3,7 +3,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-hive-functions/pom.xml b/hetu-hive-functions/pom.xml index 35bb87841..4c31c0e57 100644 --- a/hetu-hive-functions/pom.xml +++ b/hetu-hive-functions/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-listener/pom.xml b/hetu-listener/pom.xml index f815a7f45..aa6e18806 100644 --- a/hetu-listener/pom.xml +++ b/hetu-listener/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-metastore/pom.xml b/hetu-metastore/pom.xml index f3f5c0d89..d155e671c 100644 --- a/hetu-metastore/pom.xml +++ b/hetu-metastore/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-mongodb/pom.xml b/hetu-mongodb/pom.xml index 41c490ecd..6f35e5f40 100644 --- a/hetu-mongodb/pom.xml +++ b/hetu-mongodb/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-mv/pom.xml b/hetu-mv/pom.xml new file mode 100644 index 000000000..a37bccb3c --- /dev/null +++ b/hetu-mv/pom.xml @@ -0,0 +1,81 @@ + + + + presto-root + io.hetu.core + 1.2.0-SNAPSHOT + + 4.0.0 + + hetu-mv + hetu-plugin + + + ${project.parent.basedir} + + + + + io.hetu.core + presto-spi + + + io.airlift + bootstrap + + + io.airlift + json + + + org.codehaus.jettison + jettison + + + com.fasterxml.jackson.core + jackson-annotations + + + io.airlift + configuration + + + com.google.guava + guava + + + com.google.inject + guice + + + com.fasterxml.jackson.core + jackson-databind + + + io.airlift + log + + + javax.inject + javax.inject + + + + org.testng + testng + test + + + io.hetu.core + presto-main + test + + + org.mockito + mockito-core + test + + + diff --git a/hetu-mv/src/test/java/io/hetu/core/materializedview/TestLocalMvMetastore.java b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestLocalMvMetastore.java new file mode 100644 index 000000000..2e2781e49 --- /dev/null +++ b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestLocalMvMetastore.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.materializedview; + +import io.hetu.core.materializedview.conf.MaterializedViewConfig; +import io.hetu.core.materializedview.metadata.LocalMaterializedViewMetastore; +import io.hetu.core.materializedview.metadata.MaterializedView; +import io.hetu.core.materializedview.utils.MaterializedViewStatus; +import io.prestosql.spi.connector.SchemaTableName; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.List; +import java.util.Set; + +import static io.hetu.core.materializedview.TestUtils.CURRENT_SCHEMA; +import static io.hetu.core.materializedview.TestUtils.DATA_SCHEMA; +import static io.hetu.core.materializedview.TestUtils.TEST_FILE; +import static io.hetu.core.materializedview.TestUtils.VIEW_NAME; +import static io.hetu.core.materializedview.TestUtils.createMaterializedView; +import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL; +import static io.hetu.core.materializedview.utils.MaterializedViewStatus.getStatusString; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestLocalMvMetastore +{ + private LocalMaterializedViewMetastore localMaterializedViewMetastore; + private final String newName = "test_view"; + + @BeforeClass + public void setUp() + { + MaterializedViewConfig config = new MaterializedViewConfig(); + config.setMetastoreType(TYPE_LOCAL); + config.setMetastorePath(TEST_FILE); + localMaterializedViewMetastore = new LocalMaterializedViewMetastore(config); + localMaterializedViewMetastore.addSchema(DATA_SCHEMA); + localMaterializedViewMetastore.addMaterializedView(DATA_SCHEMA, createMaterializedView(VIEW_NAME)); + } + + @Test + public void testSchema() + { + localMaterializedViewMetastore.addSchema(CURRENT_SCHEMA); + Set schemas = localMaterializedViewMetastore.getSchemas(); + assertEquals(schemas.size(), 2); + assertTrue(schemas.contains(CURRENT_SCHEMA)); + + localMaterializedViewMetastore.dropSchema(CURRENT_SCHEMA); + schemas = localMaterializedViewMetastore.getSchemas(); + assertEquals(schemas.size(), 1); + assertFalse(schemas.contains(CURRENT_SCHEMA)); + } + + @Test + public void testMaterializedView() + { + SchemaTableName viewName = new SchemaTableName(DATA_SCHEMA, newName); + + List views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA); + assertEquals(views.size(), 1); + + MaterializedView view = createMaterializedView(newName); + localMaterializedViewMetastore.addMaterializedView(DATA_SCHEMA, view); + views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA); + assertEquals(views.size(), 2); + assertTrue(views.contains(newName)); + MaterializedView actualView = localMaterializedViewMetastore.getMaterializedView(viewName); + assertEquals(actualView.toString(), view.toString()); + + localMaterializedViewMetastore.setViewStatus(viewName, MaterializedViewStatus.ENABLE); + actualView = localMaterializedViewMetastore.getMaterializedView(viewName); + assertEquals(actualView.getStatus(), getStatusString(MaterializedViewStatus.ENABLE)); + + localMaterializedViewMetastore.dropMaterializedView(DATA_SCHEMA, newName); + views = localMaterializedViewMetastore.getAllMaterializedView(DATA_SCHEMA); + assertEquals(views.size(), 1); + assertFalse(views.contains(newName)); + } + + @AfterClass + public void down() + { + new File(TEST_FILE).delete(); + } +} diff --git a/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedView.java b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedView.java new file mode 100644 index 000000000..466cd819a --- /dev/null +++ b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedView.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.materializedview; + +import io.hetu.core.materializedview.connector.MaterializedViewColumnHandle; +import io.hetu.core.materializedview.metadata.MaterializedView; +import io.hetu.core.materializedview.utils.MaterializedViewDateUtils; +import io.hetu.core.materializedview.utils.MaterializedViewStatus; +import io.prestosql.spi.type.TypeSignature; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.List; + +import static io.hetu.core.materializedview.TestUtils.CURRENT_CATALOG; +import static io.hetu.core.materializedview.TestUtils.CURRENT_SCHEMA; +import static io.hetu.core.materializedview.TestUtils.DATA_CATALOG; +import static io.hetu.core.materializedview.TestUtils.DATA_SCHEMA; +import static io.hetu.core.materializedview.TestUtils.DATE; +import static io.hetu.core.materializedview.TestUtils.FULL_SQL; +import static io.hetu.core.materializedview.TestUtils.META_CATALOG; +import static io.hetu.core.materializedview.TestUtils.OWNER; +import static io.hetu.core.materializedview.TestUtils.VIEW_NAME; +import static io.hetu.core.materializedview.TestUtils.VIEW_SQL; +import static io.hetu.core.materializedview.utils.MaterializedViewDateUtils.getStringFromDate; +import static io.hetu.core.materializedview.utils.MaterializedViewStatus.DISABLE; +import static io.hetu.core.materializedview.utils.MaterializedViewStatus.ENABLE; +import static io.hetu.core.materializedview.utils.MaterializedViewStatus.getStatusFromString; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class TestMaterializedView +{ + @Test + public void testMaterializedView() + { + MaterializedView materializedView = createMaterializedView(VIEW_NAME); + assertEquals(VIEW_NAME, materializedView.getViewName()); + assertEquals(VIEW_SQL, materializedView.getViewSql()); + assertEquals(FULL_SQL, materializedView.getFullSql()); + assertEquals(DATA_CATALOG, materializedView.getDataCatalog()); + assertEquals(DATA_SCHEMA, materializedView.getDataSchema()); + assertEquals(META_CATALOG, materializedView.getMetaCatalog()); + assertEquals(CURRENT_CATALOG, materializedView.getCurrentCatalog()); + assertEquals(CURRENT_SCHEMA, materializedView.getCurrentSchema()); + assertTrue(materializedView.getColumns().equals(createMaterializedViewColumns())); + assertEquals(OWNER, materializedView.getOwner()); + assertFalse(materializedView.isRunAsInvoker()); + assertEquals(materializedView.getLastRefreshTime(), getStringFromDate(DATE)); + assertEquals(getStatusFromString(materializedView.getStatus()), DISABLE); + + materializedView.setStatus(ENABLE); + assertEquals(getStatusFromString(materializedView.getStatus()), ENABLE); + } + + private MaterializedView createMaterializedView(String viewName) + { + return new MaterializedView( + viewName, + VIEW_SQL, + FULL_SQL, + DATA_CATALOG, + DATA_SCHEMA, + META_CATALOG, + CURRENT_CATALOG, + CURRENT_SCHEMA, + createMaterializedViewColumns(), + OWNER, + false, + MaterializedViewDateUtils.getStringFromDate(DATE), + MaterializedViewStatus.getStatusString(DISABLE)); + } + + private List createMaterializedViewColumns() + { + List columns = new ArrayList<>(); + columns.add(new MaterializedViewColumnHandle("column0", 0, TypeSignature.parseTypeSignature("char"))); + columns.add(new MaterializedViewColumnHandle("column1", 1, TypeSignature.parseTypeSignature("int"))); + return columns; + } +} diff --git a/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedViewConfig.java b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedViewConfig.java new file mode 100644 index 000000000..b2ae3d5ed --- /dev/null +++ b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMaterializedViewConfig.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.materializedview; + +import io.hetu.core.materializedview.conf.MaterializedViewConfig; +import org.testng.annotations.Test; + +import static io.hetu.core.materializedview.TestUtils.TEST_FILE; +import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL; +import static org.testng.Assert.assertEquals; + +public class TestMaterializedViewConfig +{ + @Test + public void testConfig() + { + MaterializedViewConfig config = new MaterializedViewConfig(); + config.setMetastoreType(TYPE_LOCAL); + config.setMetastorePath(TEST_FILE); + + assertEquals(config.getMetastoreType(), TYPE_LOCAL); + assertEquals(config.getMetastorePath(), TEST_FILE); + } +} diff --git a/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMvConnectorMetadata.java b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMvConnectorMetadata.java new file mode 100644 index 000000000..59f3c6867 --- /dev/null +++ b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestMvConnectorMetadata.java @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.materializedview; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.hetu.core.materializedview.conf.MaterializedViewConfig; +import io.hetu.core.materializedview.metadata.MaterializedView; +import io.hetu.core.materializedview.metadata.MaterializedViewConnectorMetadata; +import io.hetu.core.materializedview.metadata.MaterializedViewConnectorMetadataFactory; +import io.hetu.core.materializedview.metadata.MaterializedViewMetastore; +import io.hetu.core.materializedview.metadata.MaterializedViewMetastoreFactory; +import io.hetu.core.materializedview.utils.MaterializedViewStatus; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorMaterializedViewDefinition; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorTableHandle; +import io.prestosql.spi.connector.ConnectorTableMetadata; +import io.prestosql.spi.connector.ConnectorViewDefinition; +import io.prestosql.spi.connector.ConnectorViewDefinition.ViewColumn; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.metastore.HetuMetastore; +import io.prestosql.spi.type.testing.TestingTypeManager; +import io.prestosql.testing.TestingConnectorSession; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.hetu.core.materializedview.utils.MaterializedViewConstants.TYPE_LOCAL; +import static io.prestosql.spi.type.VarcharType.VARCHAR; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestMvConnectorMetadata +{ + private static final String SCHEMA_NAME1 = "test1"; + private static final String SCHEMA_NAME2 = "test2"; + private static final String TABLE_NAME = "table"; + private static final String COLUMN_NAME = "column"; + private static final String CATALOG_NAME = "memory"; + private static final String SCHEMA_NAME = "demo"; + private static final String META_CATALOG_NAME = "mv"; + private static final String CURRENT_CATALOG_NAME = "hive"; + private static final String CURRENT_SCHEMA_NAME = "default"; + private static final String EXECUTE_SQL = "select column from test1.table"; + private static final String FULL_SQL = "create materialized view mv.demo.view as select column from test1.table"; + private static final String STATUS = "ENABLE"; + private static final String OWNER = "root"; + private MaterializedViewConnectorMetadata metadata; + private MockMaterializedViewMetastore mockMvMetastore; + private ConnectorSession session; + + @BeforeClass + public void setUp() + { + MaterializedViewConfig config = new MaterializedViewConfig(); + config.setMetastoreType(TYPE_LOCAL); + + mockMvMetastore = new MockMaterializedViewMetastore(); + MaterializedViewMetastoreFactory mockMetastoreFactory = mock(MaterializedViewMetastoreFactory.class); + when(mockMetastoreFactory.create(any())).thenReturn(mockMvMetastore); + + HetuMetastore hetuMetastore = mock(HetuMetastore.class); + MaterializedViewConnectorMetadataFactory factory = new MaterializedViewConnectorMetadataFactory(mockMetastoreFactory, new TestingTypeManager(), hetuMetastore); + metadata = factory.create(); + session = new TestingConnectorSession(ImmutableList.of()); + + metadata.createSchema(session, SCHEMA_NAME1, ImmutableMap.of()); + } + + @Test + public void testSchema() + { + //listSchemaNames + //createSchema + //dropSchema + metadata.createSchema(session, SCHEMA_NAME2, ImmutableMap.of()); + List schemas = metadata.listSchemaNames(session); + assertEquals(schemas.size(), 2); + assertTrue(schemas.contains(SCHEMA_NAME1)); + assertTrue(schemas.contains(SCHEMA_NAME2)); + metadata.dropSchema(session, SCHEMA_NAME2); + schemas = metadata.listSchemaNames(session); + assertEquals(schemas.size(), 1); + assertTrue(schemas.contains(SCHEMA_NAME1)); + } + + @Test + public void testViews() + { + ColumnMetadata columnMetadata = new ColumnMetadata(COLUMN_NAME, VARCHAR); + ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(new SchemaTableName(SCHEMA_NAME1, TABLE_NAME), ImmutableList.of(columnMetadata)); + ViewColumn viewColumn = new ViewColumn(COLUMN_NAME, VARCHAR.getTypeSignature()); + ConnectorMaterializedViewDefinition definition = new ConnectorMaterializedViewDefinition( + EXECUTE_SQL, + Optional.of(CATALOG_NAME), + Optional.of(SCHEMA_NAME), + ImmutableList.of(viewColumn), + Optional.of(OWNER), + false, + FULL_SQL, + META_CATALOG_NAME, + CURRENT_CATALOG_NAME, + CURRENT_SCHEMA_NAME, + STATUS); + metadata.beginCreateMaterializedView(tableMetadata, definition); + SchemaTableName schemaTableName = new SchemaTableName(SCHEMA_NAME1, TABLE_NAME); + metadata.finishCreateMaterializedView(schemaTableName); + + Optional viewDefinition = metadata.getView(session, schemaTableName); + assertTrue(viewDefinition.isPresent()); + ConnectorMaterializedViewDefinition materializedViewDefinition = (ConnectorMaterializedViewDefinition) viewDefinition.get(); + assertEquals(materializedViewDefinition.toString(), definition.toString()); + + ConnectorTableHandle tableHandle = metadata.getTableHandle(session, schemaTableName); + ConnectorTableMetadata getTableMetadata = metadata.getTableMetadata(session, tableHandle); + assertEquals(getTableMetadata.toString(), tableMetadata.toString()); + Map columnHandles = metadata.getColumnHandles(session, tableHandle); + ColumnMetadata getColumnMetadata = metadata.getColumnMetadata(session, tableHandle, columnHandles.get(COLUMN_NAME)); + assertEquals(getColumnMetadata, columnMetadata); + + List tables = metadata.listTables(session, Optional.empty()); + assertEquals(tables.size(), 1); + assertEquals(tables.get(0), schemaTableName); + + metadata.dropMaterializedView(session, tableHandle); + tables = metadata.listTables(session, Optional.empty()); + assertEquals(tables.size(), 0); + } + + class MockMaterializedViewMetastore + implements MaterializedViewMetastore + { + private Map> views; + + @Override + public void init() + { + views = new HashMap<>(); + } + + @Override + public void addMaterializedView(String schema, MaterializedView materializedView) + { + synchronized (views) { + Map viewMap = views.get(schema); + viewMap.put(materializedView.getViewName(), materializedView); + } + } + + @Override + public void dropMaterializedView(String schema, String view) + { + synchronized (views) { + Map viewMap = views.get(schema); + viewMap.remove(view); + } + } + + @Override + public void addSchema(String schemaName) + { + synchronized (views) { + if (views.containsKey(schemaName)) { + return; + } + Map viewMap = new HashMap<>(); + views.put(schemaName, viewMap); + } + } + + @Override + public void dropSchema(String schemaName) + { + synchronized (views) { + views.remove(schemaName); + } + } + + @Override + public void setViewStatus(SchemaTableName viewName, MaterializedViewStatus status) + { + synchronized (views) { + Map viewMap = views.get(viewName.getSchemaName()); + MaterializedView materializedView = viewMap.get(viewName.getTableName()); + materializedView.setStatus(status); + } + } + + @Override + public Set getSchemas() + { + synchronized (views) { + return views.keySet(); + } + } + + @Override + public List getAllMaterializedView(String schemaName) + { + List viewList = new ArrayList<>(); + synchronized (views) { + Map viewMap = views.get(schemaName); + for (MaterializedView view : viewMap.values()) { + viewList.add(view.getViewName()); + } + } + return viewList; + } + + @Override + public MaterializedView getMaterializedView(SchemaTableName viewName) + { + synchronized (views) { + Map viewMap = views.get(viewName.getSchemaName()); + return viewMap.get(viewName.getTableName()); + } + } + } +} diff --git a/hetu-mv/src/test/java/io/hetu/core/materializedview/TestUtils.java b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestUtils.java new file mode 100644 index 000000000..4eb5fd493 --- /dev/null +++ b/hetu-mv/src/test/java/io/hetu/core/materializedview/TestUtils.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.materializedview; + +import io.hetu.core.materializedview.connector.MaterializedViewColumnHandle; +import io.hetu.core.materializedview.metadata.MaterializedView; +import io.hetu.core.materializedview.utils.MaterializedViewDateUtils; +import io.hetu.core.materializedview.utils.MaterializedViewStatus; +import io.prestosql.spi.type.TypeSignature; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import static io.hetu.core.materializedview.utils.MaterializedViewStatus.DISABLE; + +public class TestUtils +{ + public static final String TEST_FILE = "testLocalMvMetadata.ini"; + + public static final String VIEW_NAME = "mv.test.view"; + public static final String FULL_SQL = "create materialized view mv.test.view as select * from hive.default.test"; + public static final String VIEW_SQL = "select * from hive.default.test"; + public static final String DATA_CATALOG = "memory"; + public static final String DATA_SCHEMA = "test"; + public static final String META_CATALOG = "mv"; + public static final String CURRENT_CATALOG = "hive"; + public static final String CURRENT_SCHEMA = "default"; + public static final String OWNER = "test"; + public static final Date DATE = new Date(); + + private TestUtils() + { + } + + public static MaterializedView createMaterializedView(String viewName) + { + return new MaterializedView( + viewName, + VIEW_SQL, + FULL_SQL, + DATA_CATALOG, + DATA_SCHEMA, + META_CATALOG, + CURRENT_CATALOG, + CURRENT_SCHEMA, + createMaterializedViewColumns(), + OWNER, + false, + MaterializedViewDateUtils.getStringFromDate(DATE), + MaterializedViewStatus.getStatusString(DISABLE)); + } + + public static List createMaterializedViewColumns() + { + List columns = new ArrayList<>(); + columns.add(new MaterializedViewColumnHandle("column0", 0, TypeSignature.parseTypeSignature("char"))); + columns.add(new MaterializedViewColumnHandle("column1", 1, TypeSignature.parseTypeSignature("int"))); + return columns; + } +} diff --git a/hetu-opengauss/pom.xml b/hetu-opengauss/pom.xml index 1289b5af6..5b7e176b3 100644 --- a/hetu-opengauss/pom.xml +++ b/hetu-opengauss/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 @@ -153,7 +153,7 @@ io.hetu.core presto-postgresql - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT compile diff --git a/hetu-oracle/pom.xml b/hetu-oracle/pom.xml index 4d595ac20..a76e074e6 100644 --- a/hetu-oracle/pom.xml +++ b/hetu-oracle/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-oracle diff --git a/hetu-seed-store/pom.xml b/hetu-seed-store/pom.xml index 76fe35524..d451c406e 100644 --- a/hetu-seed-store/pom.xml +++ b/hetu-seed-store/pom.xml @@ -3,7 +3,7 @@ presto-root io.hetu.core - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT 4.0.0 diff --git a/hetu-server-rpm/pom.xml b/hetu-server-rpm/pom.xml index c94c8dc01..045b555e7 100644 --- a/hetu-server-rpm/pom.xml +++ b/hetu-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-server-rpm diff --git a/hetu-server/pom.xml b/hetu-server/pom.xml index 07357f6a2..870c8bc4b 100644 --- a/hetu-server/pom.xml +++ b/hetu-server/pom.xml @@ -5,7 +5,7 @@ io.hetu.core presto-root - 1.1.1-SNAPSHOT + 1.2.0-SNAPSHOT hetu-server diff --git a/hetu-server/src/main/provisio/hetu.xml b/hetu-server/src/main/provisio/hetu.xml index 542e19133..d1392b74a 100644 --- a/hetu-server/src/main/provisio/hetu.xml +++ b/hetu-server/src/main/provisio/hetu.xml @@ -138,6 +138,12 @@ + + + + + +