From e326d183769341f193865bd131727d3e6f622bbb Mon Sep 17 00:00:00 2001 From: guojunfei <970763131@qq.com> Date: Mon, 7 Sep 2020 18:39:57 +0800 Subject: [PATCH 1/2] remove HBaseHdfsMetastore, move HBaseLocalMetastore to test, and just use in test, add HBaseHetuMetastore UT --- hetu-hbase/pom.xml | 69 +++-- .../core/plugin/hbase/conf/HBaseConfig.java | 36 --- .../hbase/connector/HBaseConnection.java | 18 +- .../hbase/metadata/HBaseMetastoreFactory.java | 10 +- .../hbase/metadata/HdfsHBaseMetastore.java | 274 ------------------ .../plugin/hbase/{test => }/TestHBase.java | 15 +- .../hbase/{test => }/TestHBasePlugin.java | 3 +- .../plugin/hbase/{test => }/TestQuery.java | 14 +- .../core/plugin/hbase/client/TestUtils.java | 11 - .../{test => conf}/TestHBaseClientConf.java | 15 +- .../TestHBaseTableProperties.java | 3 +- .../TestHBaseClientConnection.java | 8 +- .../TestHBaseColumnHandle.java | 7 +- .../TestHBaseConnector.java | 20 +- .../TestHBaseTableHandle.java | 5 +- .../TestHBaseTransactionHandle.java | 5 +- .../hbase/metadata}/JsonHBaseTableUtils.java | 4 +- .../hbase/metadata/LocalHBaseMetastore.java | 9 +- .../TestHBaseMetastoreFactory.java | 34 +-- .../{test => metadata}/TestHBaseTable.java | 5 +- .../TestHBaseTableUtils.java} | 57 +--- .../metadata/TestHetuHBaseMetastore.java | 153 ++++++++++ .../TestHBaseKerberosAuthentication.java | 28 +- .../{test => }/split/TestHBaseSplit.java | 3 +- .../split/TestHbaseSplitManager.java | 10 +- .../hbase/test/TestHdfsHBaseMetastore.java | 229 --------------- .../AbstractTestHBaseRowSerializer.java | 3 +- .../serializers/TestStringRowSerializer.java | 4 +- 28 files changed, 290 insertions(+), 762 deletions(-) delete mode 100644 hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HdfsHBaseMetastore.java rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/TestHBase.java (96%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/TestHBasePlugin.java (93%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/TestQuery.java (95%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => conf}/TestHBaseClientConf.java (82%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => conf}/TestHBaseTableProperties.java (97%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => connector}/TestHBaseClientConnection.java (85%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => connector}/TestHBaseColumnHandle.java (94%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => connector}/TestHBaseConnector.java (96%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => connector}/TestHBaseTableHandle.java (95%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => connector}/TestHBaseTransactionHandle.java (91%) rename hetu-hbase/src/{main/java/io/hetu/core/plugin/hbase/utils => test/java/io/hetu/core/plugin/hbase/metadata}/JsonHBaseTableUtils.java (98%) rename hetu-hbase/src/{main => test}/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java (96%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => metadata}/TestHBaseMetastoreFactory.java (68%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => metadata}/TestHBaseTable.java (95%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test/TestJsonHBaseTableUtils.java => metadata/TestHBaseTableUtils.java} (79%) create mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => security}/TestHBaseKerberosAuthentication.java (86%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/split/TestHBaseSplit.java (95%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/split/TestHbaseSplitManager.java (87%) delete mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHdfsHBaseMetastore.java rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/utils/serializers/AbstractTestHBaseRowSerializer.java (99%) rename hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/{test => }/utils/serializers/TestStringRowSerializer.java (88%) diff --git a/hetu-hbase/pom.xml b/hetu-hbase/pom.xml index b51ed7d00..acf43135d 100644 --- a/hetu-hbase/pom.xml +++ b/hetu-hbase/pom.xml @@ -41,6 +41,13 @@ + + org.weakref + jmxutils + ${dep.jmxutils.version} + compile + + io.prestosql.hadoop hadoop-apache @@ -147,35 +154,12 @@ ${version.commons-lang3} - - org.mockito - mockito-all - ${version.mockito-all} - test - - - junit - junit - ${version.junit} - test - - - hamcrest-core - org.hamcrest - - - com.google.protobuf protobuf-java ${version.protobuf-java} - - org.testng - testng - ${version.testing} - test - + org.apache.hbase hbase-common @@ -318,6 +302,43 @@ com.fasterxml.jackson.core runtime + + + org.testng + testng + test + + + + io.hetu.core + presto-tests + test + + + + io.hetu.core + presto-plugin-toolkit + test + + + + io.hetu.core + hetu-metastore + test + + + + io.airlift + testing-mysql-server + test + + + + org.mockito + mockito-all + ${version.mockito-all} + test + diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java index 6a59d2863..259d76239 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java +++ b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java @@ -31,11 +31,8 @@ public class HBaseConfig private String zkQuorum; private String zkClientPort; private String metastoreType; - private String metastoreUrl; private String defaultValue = "NULL"; private String jaasConfPath; // java.security.auth.login.config: jaas.conf - private String coreSitePath; // core-site.xml file path - private String hdfsSitePath; // hdfs-site.xml file path private String hbaseSitePath; // hbase-site.xml file path private String krb5ConfPath; // java.security.krb5.conf: krb5.conf private String userKeytabPath; // user.keytab file path @@ -86,28 +83,6 @@ public class HBaseConfig this.jaasConfPath = jaasConfPath; } - public String getCoreSitePath() - { - return coreSitePath; - } - - @Config("hbase.core.site.path") - public void setCoreSitePath(String coreSitePath) - { - this.coreSitePath = coreSitePath; - } - - public String getHdfsSitePath() - { - return hdfsSitePath; - } - - @Config("hbase.hdfs.site.path") - public void setHdfsSitePath(String hdfsSitePath) - { - this.hdfsSitePath = hdfsSitePath; - } - public String getHbaseSitePath() { return hbaseSitePath; @@ -185,17 +160,6 @@ public class HBaseConfig this.metastoreType = metastoreType; } - public String getMetastoreUrl() - { - return metastoreUrl; - } - - @Config("hbase.metastore.uri") - public void setMetastoreUrl(String metastoreUrl) - { - this.metastoreUrl = metastoreUrl; - } - public String getKerberos() { return kerberos; diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/connector/HBaseConnection.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/connector/HBaseConnection.java index 0b522b9f4..999015e50 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/connector/HBaseConnection.java +++ b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/connector/HBaseConnection.java @@ -104,10 +104,6 @@ public class HBaseConnection * Configuration */ protected Configuration cfg; - /** - * catalogFile - */ - protected String catalogFile; private UserGroupInformation ugi; private HBaseMetastore hBaseMetastore; @@ -125,6 +121,19 @@ public class HBaseConnection authenticate(); } + /** + * constructor for test + * + * @param hBaseMetastore HBaseMetaStore + * @param config HBaseConfig + */ + protected HBaseConnection(HBaseMetastore hBaseMetastore, HBaseConfig config) + { + this.hbaseConfig = config; + this.hBaseMetastore = hBaseMetastore; + authenticate(); + } + /** * getDefaultValue * @@ -183,7 +192,6 @@ public class HBaseConnection cfg.set("hbase.zookeeper.property.clientPort", hbaseConfig.getZkClientPort()); try { - catalogFile = hbaseConfig.getMetastoreUrl(); init(); hBaseMetastore.init(); } diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HBaseMetastoreFactory.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HBaseMetastoreFactory.java index f81466d8c..7f5b28589 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HBaseMetastoreFactory.java +++ b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HBaseMetastoreFactory.java @@ -27,8 +27,6 @@ import io.prestosql.spi.metastore.HetuMetastore; */ public class HBaseMetastoreFactory { - private static final String TYPE_LOCAL = "local"; - private static final String TYPE_HDFS = "hdfs"; private static final String TYPE_HETUMETASTORE = "hetuMetastore"; private final HBaseConfig hbaseConfig; private final HetuMetastore hetuMetastore; @@ -47,13 +45,7 @@ public class HBaseMetastoreFactory public HBaseMetastore create() { String type = hbaseConfig.getMetastoreType(); - if (type == null || TYPE_LOCAL.equals(type)) { - return new LocalHBaseMetastore(hbaseConfig); - } - else if (TYPE_HDFS.equals(type)) { - return new HdfsHBaseMetastore(hbaseConfig); - } - else if (TYPE_HETUMETASTORE.equals(type)) { + if (type == null || TYPE_HETUMETASTORE.equals(type)) { return new HetuHBaseMetastore(hetuMetastore); } else { diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HdfsHBaseMetastore.java b/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HdfsHBaseMetastore.java deleted file mode 100644 index f2ae01ffd..000000000 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HdfsHBaseMetastore.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.hbase.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; -import io.hetu.core.plugin.hbase.utils.Constants; -import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils; -import io.hetu.core.plugin.hbase.utils.Utils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.security.UserGroupInformation; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -import java.io.BufferedWriter; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import static java.util.Objects.requireNonNull; - -/** - * HdfsHBaseMetastore: store file on hdfs - * - * @since 2020-03-30 - */ -public class HdfsHBaseMetastore - implements HBaseMetastore -{ - private static final Logger LOG = Logger.get(HdfsHBaseMetastore.class); - private static final String ID = "HDFS"; - - private final Map hbaseTables = new ConcurrentHashMap<>(); - private String filePath; - private HBaseConfig hbaseConfig; - private FileSystem fs; - private Configuration hdfsConfig; - - public HdfsHBaseMetastore(HBaseConfig config) - { - this.hbaseConfig = config; - this.filePath = hbaseConfig.getMetastoreUrl(); - authenticate(); - this.fs = getFs(); - } - - @Override - public void init() - { - JSONObject jsonObject = readJsonFromHdfs(filePath); - JsonHBaseTableUtils.loadHBaseTablesFromJson(hbaseTables, jsonObject); - } - - /** - * get hdfs filesystem - * - * @return filesystem - */ - private FileSystem getFs() - { - FileSystem fileSystem = null; - try { - fileSystem = FileSystem.get(hdfsConfig); - } - catch (IOException e) { - LOG.error("Get hadoop file System error, cause by %s", e.getMessage()); - } - return fileSystem; - } - - private void authenticate() - { - hdfsConfig = new Configuration(); - - try { - if (!Utils.isFileExist(hbaseConfig.getHdfsSitePath())) { - throw new FileNotFoundException(hbaseConfig.getHdfsSitePath()); - } - if (!Utils.isFileExist(hbaseConfig.getCoreSitePath())) { - throw new FileNotFoundException(hbaseConfig.getCoreSitePath()); - } - hdfsConfig.addResource(new Path(hbaseConfig.getHdfsSitePath())); - hdfsConfig.addResource(new Path(hbaseConfig.getCoreSitePath())); - - if (Constants.HDFS_AUTHENTICATION_KERBEROS.equals(hbaseConfig.getKerberos())) { - String keyTabPath = requireNonNull(hbaseConfig.getUserKeytabPath()); - String krb5ConfPath = requireNonNull(hbaseConfig.getKrb5ConfPath()); - - if (!Utils.isFileExist(keyTabPath)) { - throw new FileNotFoundException(keyTabPath); - } - if (!Utils.isFileExist(krb5ConfPath)) { - throw new FileNotFoundException(krb5ConfPath); - } - - String hdfsPrincipal = requireNonNull(hbaseConfig.getPrincipalUsername()); - System.setProperty("java.security.krb5.conf", krb5ConfPath); - UserGroupInformation.setConfiguration(hdfsConfig); - UserGroupInformation.loginUserFromKeytab(hdfsPrincipal, keyTabPath); - } - } - catch (IOException exception) { - if (exception instanceof FileNotFoundException) { - LOG.error("File not exist, cause by %s", exception.getMessage()); - } - else { - LOG.error("Kerberos authentication failed, caused by %s", exception.getMessage()); - } - } - } - - @Override - public String getId() - { - return this.ID; - } - - @Override - public void addHBaseTable(HBaseTable hBaseTable) - { - // hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified - synchronized (hbaseTables) { - if (hbaseTables.containsKey(hBaseTable.getFullTableName())) { - hbaseTables.remove(hBaseTable.getFullTableName()); - } - - hbaseTables.put(hBaseTable.getFullTableName(), hBaseTable); - putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - - @Override - public void renameHBaseTable(HBaseTable newTable, String oldTable) - { - // hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified - synchronized (hbaseTables) { - if (hbaseTables.containsKey(oldTable)) { - hbaseTables.remove(oldTable); - } - - hbaseTables.put(newTable.getFullTableName(), newTable); - putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - - @Override - public void dropHBaseTable(HBaseTable hBaseTable) - { - synchronized (hbaseTables) { - if (hbaseTables.containsKey(hBaseTable.getFullTableName())) { - hbaseTables.remove(hBaseTable.getFullTableName()); - putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - } - - @Override - public synchronized Map getAllHBaseTables() - { - return ImmutableMap.copyOf(hbaseTables); - } - - @Override - public HBaseTable getHBaseTable(String tableName) - { - return hbaseTables.get(tableName); - } - - /** - * readJsonFromHdfs - * - * @param file file - * @return JSONObject - */ - public JSONObject readJsonFromHdfs(String file) - { - InputStream input = null; - JSONObject rs = null; - StringBuilder catalogs = new StringBuilder(); - try { - Path path = new Path(file); - if (!fs.exists(path)) { - return rs; - } - input = fs.open(path); - while (input.available() > 0) { - catalogs.append((char) input.read()); - } - } - catch (IOException e) { - LOG.error("readJsonFromHdfs : some wrong...file [%s].", file); - } - finally { - if (input != null) { - try { - input.close(); - } - catch (IOException e) { - LOG.error("readJsonFromHdfs : something wrong...file [%s].", file); - } - } - } - - try { - if (catalogs.toString().length() > 0) { - rs = new JSONObject(catalogs.toString()); - } - } - catch (JSONException e) { - LOG.error("readJsonFromHdfs : to json wrong...cause by [%s].", e.getMessage()); - } - return rs; - } - - /** - * putJsonToFile - * - * @param file file - * @param jsonObject jsonObject - */ - public void putJsonToHdfs(String file, JSONObject jsonObject) - { - String jsonStr = ""; - OutputStream out = null; - try { - jsonStr = jsonObject.toString(Constants.NUMBER4); - } - catch (JSONException e) { - LOG.error("putJsonToHdfs : json to string wrong...cause by [%s]", e.getMessage()); - } - - Path path = new Path(file); - try { - out = fs.create(path, true); - BufferedWriter bufferOut = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)); - bufferOut.write(jsonStr); - bufferOut.flush(); - bufferOut.close(); - } - catch (IOException e) { - LOG.error("write file[%s] failed... ", e.getMessage()); - } - finally { - if (out != null) { - try { - out.close(); - } - catch (IOException e) { - LOG.error("write file[%s] failed... ", e.getMessage()); - } - } - } - } -} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBase.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java similarity index 96% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBase.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java index 1f9bdc864..68be9d3da 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBase.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase; import io.hetu.core.plugin.hbase.client.TestUtils; import io.hetu.core.plugin.hbase.client.TestingConnectorSession; @@ -23,7 +23,10 @@ import io.hetu.core.plugin.hbase.connector.HBaseConnectorId; import io.hetu.core.plugin.hbase.connector.HBaseConnectorMetadataFactory; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.connector.HBaseTransactionHandle; +import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata; +import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; +import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider; import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider; import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider; @@ -73,6 +76,7 @@ import static org.testng.Assert.assertEquals; */ public class TestHBase { + private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseConnectorMetadata hcm; @@ -89,11 +93,10 @@ public class TestHBase { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini"); - TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl()); + TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); table = TestUtils.createHBaseTableHandle(); schemaTableName = new SchemaTableName("hbase", "test_table"); - hconn = new TestHBaseClientConnection(hCConf); + hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); hconn.getConn(); session = new TestingConnectorSession("root"); hcm = new HBaseConnectorMetadata(hconn); @@ -304,7 +307,7 @@ public class TestHBase /** * testListTables */ - public void testListTables() + private void testListTables() { List tables = hcm.listTables(session, Optional.of("hbase")); assertEquals(tables.size(), 1); @@ -481,6 +484,6 @@ public class TestHBase @AfterClass public void clear() { - TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl()); + TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBasePlugin.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBasePlugin.java similarity index 93% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBasePlugin.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBasePlugin.java index f75d8fbaa..5a3e9c83c 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBasePlugin.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBasePlugin.java @@ -12,9 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase; -import io.hetu.core.plugin.hbase.HBasePlugin; import org.testng.annotations.Test; import java.lang.reflect.Method; diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestQuery.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java similarity index 95% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestQuery.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java index 79e4a2b36..1ba56c0c6 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestQuery.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase; import io.airlift.slice.Slice; import io.airlift.slice.Slices; @@ -23,6 +23,9 @@ import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; import io.hetu.core.plugin.hbase.connector.HBaseConnection; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; +import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; +import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; +import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; import io.hetu.core.plugin.hbase.query.HBaseGetRecordCursor; import io.hetu.core.plugin.hbase.query.HBaseRecordCursor; import io.hetu.core.plugin.hbase.query.HBaseRecordSet; @@ -63,6 +66,8 @@ import static org.testng.Assert.assertEquals; */ public class TestQuery { + private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; + private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private ConnectorSession session; @@ -77,9 +82,8 @@ public class TestQuery { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini"); - TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl()); - hconn = new TestHBaseClientConnection(hCConf); + TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); + hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); hconn.getConn(); session = new TestingConnectorSession("root"); split = @@ -338,6 +342,6 @@ public class TestQuery @AfterClass public void clear() { - TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl()); + TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestUtils.java index fdedf12c5..b09094d11 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestUtils.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestUtils.java @@ -14,11 +14,9 @@ */ package io.hetu.core.plugin.hbase.client; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.metadata.HBaseTable; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; import io.prestosql.spi.connector.ColumnHandle; import io.prestosql.spi.connector.ColumnMetadata; import io.prestosql.spi.connector.ConnectorTableMetadata; @@ -210,15 +208,6 @@ public class TestUtils return properties; } - /** - * createHBaseTable - */ - public static Map createHBaseTable(String file) - { - LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(new HBaseConfig()); - return loadHBaseTableFromFile(lHBMetastore.readJsonFromFile(file)); - } - /** * loadHBaseTableFromFile * diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConf.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseClientConf.java similarity index 82% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConf.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseClientConf.java index 33bda1833..54cce8c95 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConf.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseClientConf.java @@ -12,13 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.conf; -import io.hetu.core.plugin.hbase.conf.HBaseColumnProperties; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; import org.testng.annotations.Test; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestHBaseClientConf @@ -47,9 +45,6 @@ public class TestHBaseClientConf hcc.setMetastoreType("type"); assertEquals("type", hcc.getMetastoreType()); - hcc.setMetastoreUrl("file"); - assertEquals("file", hcc.getMetastoreUrl()); - hcc.setKerberos("file"); assertEquals("file", hcc.getKerberos()); @@ -65,12 +60,6 @@ public class TestHBaseClientConf hcc.setJaasConfPath("/etc/hetu/"); assertEquals("/etc/hetu/", hcc.getJaasConfPath()); - hcc.setCoreSitePath("/etc/hetu/"); - assertEquals("/etc/hetu/", hcc.getCoreSitePath()); - - hcc.setHdfsSitePath("/etc/hetu/"); - assertEquals("/etc/hetu/", hcc.getHdfsSitePath()); - hcc.setHbaseSitePath("/etc/hetu/"); assertEquals("/etc/hetu/", hcc.getHbaseSitePath()); diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableProperties.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseTableProperties.java similarity index 97% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableProperties.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseTableProperties.java index 129f8c6c1..3c5eeb70f 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableProperties.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseTableProperties.java @@ -12,9 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.conf; -import io.hetu.core.plugin.hbase.conf.HBaseTableProperties; import org.apache.commons.lang3.tuple.Pair; import org.testng.annotations.Test; diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConnection.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseClientConnection.java similarity index 85% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConnection.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseClientConnection.java index 3bfbe6844..7f3a77bee 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConnection.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseClientConnection.java @@ -12,11 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.connector; import io.hetu.core.plugin.hbase.client.TestHBaseConnection; import io.hetu.core.plugin.hbase.conf.HBaseConfig; -import io.hetu.core.plugin.hbase.connector.HBaseConnection; +import io.hetu.core.plugin.hbase.metadata.HBaseMetastore; import org.apache.hadoop.hbase.client.HBaseAdmin; import java.io.IOException; @@ -29,9 +29,9 @@ import java.io.IOException; public class TestHBaseClientConnection extends HBaseConnection { - public TestHBaseClientConnection(HBaseConfig conf) + public TestHBaseClientConnection(HBaseConfig conf, HBaseMetastore metastore) { - super(conf); + super(metastore, conf); } /** diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseColumnHandle.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseColumnHandle.java similarity index 94% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseColumnHandle.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseColumnHandle.java index 17952df58..0a77054c6 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseColumnHandle.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseColumnHandle.java @@ -12,16 +12,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.connector; -import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; import io.prestosql.spi.type.Type; import io.prestosql.spi.type.VarcharType; -import org.junit.Test; +import org.testng.annotations.Test; import java.util.Optional; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestHBaseColumnHandle diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseConnector.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java similarity index 96% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseConnector.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java index 252dc6641..d4e7fd710 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseConnector.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java @@ -12,20 +12,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.connector; import io.hetu.core.plugin.hbase.client.TestUtils; import io.hetu.core.plugin.hbase.client.TestingConnectorSession; import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.conf.HBaseTableProperties; -import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; -import io.hetu.core.plugin.hbase.connector.HBaseConnection; -import io.hetu.core.plugin.hbase.connector.HBaseConnector; -import io.hetu.core.plugin.hbase.connector.HBaseConnectorId; -import io.hetu.core.plugin.hbase.connector.HBaseConnectorMetadataFactory; -import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata; import io.hetu.core.plugin.hbase.metadata.HBaseTable; +import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; +import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider; import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider; import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider; @@ -78,6 +74,7 @@ import static org.testng.Assert.assertEquals; */ public class TestHBaseConnector { + private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseConnectorMetadata hcm; @@ -93,10 +90,9 @@ public class TestHBaseConnector { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini"); - TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl()); + TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); schemaTableName = new SchemaTableName("hbase", "test_table"); - hconn = new TestHBaseClientConnection(hCConf); + hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); hconn.getConn(); session = new TestingConnectorSession("root"); hcm = new HBaseConnectorMetadata(hconn); @@ -358,7 +354,7 @@ public class TestHBaseConnector hCnnConf.setZkClientPort("2181"); hCnnConf.setZkQuorum("zk1"); hCnnConf.setRetryNumber(1); - HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf); + HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf, new LocalHBaseMetastore(null)); try { hConn.createSchema("schema", null); @@ -610,6 +606,6 @@ public class TestHBaseConnector @AfterClass public void clear() { - TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl()); + TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableHandle.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTableHandle.java similarity index 95% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableHandle.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTableHandle.java index 306f057aa..936a8f492 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableHandle.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTableHandle.java @@ -12,14 +12,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.connector; -import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import org.testng.annotations.Test; import java.util.Optional; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestHBaseTableHandle diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTransactionHandle.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTransactionHandle.java similarity index 91% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTransactionHandle.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTransactionHandle.java index fe1a2e14d..d9ddb0a29 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTransactionHandle.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTransactionHandle.java @@ -12,14 +12,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.connector; -import io.hetu.core.plugin.hbase.connector.HBaseTransactionHandle; import org.testng.annotations.Test; import java.util.UUID; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestHBaseTransactionHandle diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/utils/JsonHBaseTableUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java similarity index 98% rename from hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/utils/JsonHBaseTableUtils.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java index 8cf5ea8b5..07c5ff409 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/utils/JsonHBaseTableUtils.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java @@ -12,11 +12,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.utils; +package io.hetu.core.plugin.hbase.metadata; import io.airlift.log.Logger; import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; -import io.hetu.core.plugin.hbase.metadata.HBaseTable; +import io.hetu.core.plugin.hbase.utils.Utils; import io.prestosql.spi.connector.ColumnHandle; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; diff --git a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java similarity index 96% rename from hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java index 53faedd75..a0124781e 100644 --- a/hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java @@ -16,9 +16,7 @@ package io.hetu.core.plugin.hbase.metadata; import com.google.common.collect.ImmutableMap; import io.airlift.log.Logger; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.utils.Constants; -import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; @@ -33,6 +31,7 @@ import java.util.concurrent.ConcurrentHashMap; /** * LocalHBaseMetastore: store file on local filesystem + * This Metastore is just for test * * @since 2020-03-30 */ @@ -42,11 +41,11 @@ public class LocalHBaseMetastore private static final Logger LOG = Logger.get(LocalHBaseMetastore.class); private static final String ID = "LOCAL"; private final Map hbaseTables = new ConcurrentHashMap<>(); - private String filePath; + private final String filePath; - public LocalHBaseMetastore(HBaseConfig config) + public LocalHBaseMetastore(String filePath) { - filePath = config.getMetastoreUrl(); + this.filePath = filePath; } @Override diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseMetastoreFactory.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseMetastoreFactory.java similarity index 68% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseMetastoreFactory.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseMetastoreFactory.java index 19f9e13e9..c1b990c00 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseMetastoreFactory.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseMetastoreFactory.java @@ -12,20 +12,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.metadata; import io.hetu.core.plugin.hbase.client.TestUtils; import io.hetu.core.plugin.hbase.conf.HBaseConfig; -import io.hetu.core.plugin.hbase.metadata.HBaseMetastoreFactory; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; import io.hetu.core.plugin.hbase.utils.HBaseErrorCode; import io.prestosql.spi.PrestoException; import io.prestosql.spi.connector.ConnectorTableMetadata; import io.prestosql.spi.connector.SchemaTableName; import org.testng.annotations.Test; -import java.util.UUID; - import static org.testng.Assert.assertEquals; /** @@ -35,34 +31,6 @@ import static org.testng.Assert.assertEquals; */ public class TestHBaseMetastoreFactory { - /** - * testLocalHBaseMetastoreGetId - */ - @Test - public void testLocalHBaseMetastoreGetId() - { - LocalHBaseMetastore lhBMetastore = new LocalHBaseMetastore(new HBaseConfig()); - assertEquals("LOCAL", lhBMetastore.getId()); - } - - /** - * testHBaseMetastoreFactoryHdfs - */ - @Test - public void testHBaseMetastoreFactoryHdfs() - { - HBaseConfig hBConf = new HBaseConfig(); - hBConf.setMetastoreType("hdfs"); - hBConf.setHdfsSitePath("./hdfs-site.xml" + UUID.randomUUID()); - hBConf.setCoreSitePath("./core-site.xml" + UUID.randomUUID()); - hBConf.setKerberos("KERBEROS"); - hBConf.setUserKeytabPath("./user.keytab" + UUID.randomUUID()); - hBConf.setKrb5ConfPath("./krb5.conf" + UUID.randomUUID()); - hBConf.setPrincipalUsername("root"); - HBaseMetastoreFactory hBMetaFactory = new HBaseMetastoreFactory(hBConf); - hBMetaFactory.create(); - } - /** * testHBaseMetastoreFactoryHetuMetastore */ diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTable.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTable.java similarity index 95% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTable.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTable.java index 134e647e1..1bd43ca19 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTable.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTable.java @@ -12,17 +12,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.metadata; import io.hetu.core.plugin.hbase.client.TestUtils; -import io.hetu.core.plugin.hbase.metadata.HBaseTable; import io.prestosql.spi.connector.SchemaTableName; import org.codehaus.jettison.json.JSONException; import org.testng.annotations.Test; import java.util.Optional; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestHBaseTable diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestJsonHBaseTableUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java similarity index 79% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestJsonHBaseTableUtils.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java index 9a7eb6efe..bf55ec147 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestJsonHBaseTableUtils.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java @@ -12,29 +12,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.metadata; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; -import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; -import org.testng.annotations.Test; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import static org.junit.Assert.assertEquals; +import static org.testng.Assert.assertEquals; /** * TestJsonHBaseTableUtils * * @since 2020-03-20 */ -public class TestJsonHBaseTableUtils +public class TestHBaseTableUtils { - private static LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(new HBaseConfig()); + private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; + private static LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH); private static String jsonStr = "{\n" + " \"hbase.test_table\": {\n" @@ -97,39 +93,7 @@ public class TestJsonHBaseTableUtils + " ]\n" + " }}"; - /** - * testFileUtils - * - * @throws NullPointerException Exception - */ - @Test - public void testFileUtils() - throws Exception - { - String file = "./tmpcatalog.ini"; - try { - JSONObject json = new JSONObject(jsonStr); - JsonHBaseTableUtils.loadHBaseTablesFromJson(new HashMap<>(), json); - JSONObject json2 = lHBMetastore.readJsonFromFile(file); - - assertEquals(json.toString(), json2.toString()); - - File pfile = new File(file); - pfile.delete(); - } - catch (NullPointerException e) { - assertEquals(e.toString(), "java.lang.NullPointerException"); - } - } - - /** - * testLoadHBaseTablesFromJsonNull - */ - @Test - public void testLoadHBaseTablesFromJsonNull() - { - JsonHBaseTableUtils.loadHBaseTablesFromJson(new HashMap<>(), null); - } + private TestHBaseTableUtils() {} /** * preFile @@ -172,13 +136,4 @@ public class TestJsonHBaseTableUtils } } } - - /** - * testReadException - */ - @Test - public void testReadException() - { - assertEquals(null, lHBMetastore.readJsonFromFile("xxx")); - } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java new file mode 100644 index 000000000..cf8a0795f --- /dev/null +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.hbase.metadata; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.testing.mysql.TestingMySqlServer; +import io.hetu.core.metastore.jdbc.JdbcMetastoreModule; +import io.hetu.core.plugin.hbase.client.TestUtils; +import io.prestosql.plugin.base.jmx.MBeanServerModule; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.metastore.HetuMetastore; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; +import org.weakref.jmx.guice.MBeanModule; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static io.prestosql.spi.metastore.HetuErrorCode.HETU_METASTORE_CODE; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class TestHetuHBaseMetastore +{ + private static final String TEST_DATABASES = "hbase"; + private static final String TEST_MYSQL_USER = "user"; + private static final String TEST_MYSQL_PASSWORD = "testpass"; + + private TestingMySqlServer mySqlServer; + private HetuHBaseMetastore metaStore; + + @BeforeClass + public void setUp() + throws Exception + { + this.mySqlServer = new TestingMySqlServer(TEST_MYSQL_USER, TEST_MYSQL_PASSWORD, TEST_DATABASES); + Map properties = new ImmutableMap.Builder() + .put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES)) + .put("hetu.metastore.db.user", TEST_MYSQL_USER) + .put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD) + .build(); + try { + Bootstrap app = new Bootstrap( + new MBeanModule(), + new MBeanServerModule(), + new JdbcMetastoreModule()); + + Injector injector = app + .strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(properties) + .initialize(); + + this.metaStore = new HetuHBaseMetastore(injector.getInstance(HetuMetastore.class)); + metaStore.init(); + } + catch (Exception ex) { + throwIfUnchecked(ex); + throw new PrestoException(HETU_METASTORE_CODE, + "init hetu hbase metastore module failed."); + } + } + + @AfterClass(alwaysRun = true) + public void destroy() + { + mySqlServer.close(); + } + + @Test + public void testGetId() + { + assertEquals(metaStore.getId(), "Hetu"); + } + + @Test + public void testOPHBaseTable() + { + Optional serializerClassName = + Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer"); + + HBaseTable testHBaseTable1 = + new HBaseTable( + "hbase", + "testTable1", + TestUtils.createColumnList(), + "rowkey", + false, + serializerClassName, + Optional.of(""), + Optional.of("testTable")); + + metaStore.addHBaseTable(testHBaseTable1); + + HBaseTable testHBaseTable2 = + new HBaseTable( + "hbase", + "testTable2", + TestUtils.createColumnList(), + "rowkey", + false, + serializerClassName, + Optional.of(""), + Optional.of("testTable2")); + + metaStore.addHBaseTable(testHBaseTable2); + // test get all tables + Map hBaseTables = metaStore.getAllHBaseTables(); + assertTrue(hBaseTables.containsKey("hbase.testTable1")); + assertTrue(hBaseTables.containsKey("hbase.testTable2")); + // test get one of tables + HBaseTable table = metaStore.getHBaseTable("hbase.testTable1"); + assertNotNull(table); + assertEquals("hbase.testTable1", table.getFullTableName()); + // test rename table + HBaseTable testHBaseTable3 = + new HBaseTable( + "hbase", + "testTable3", + TestUtils.createColumnList(), + "rowkey", + false, + serializerClassName, + Optional.of(""), + Optional.of("testTable3")); + metaStore.renameHBaseTable(testHBaseTable3, "hbase.testTable1"); + hBaseTables = metaStore.getAllHBaseTables(); + assertFalse(hBaseTables.containsKey("hbase.testTable1")); + assertTrue(hBaseTables.containsKey("hbase.testTable3")); + // test drop table + metaStore.dropHBaseTable(testHBaseTable3); + hBaseTables = metaStore.getAllHBaseTables(); + assertFalse(hBaseTables.containsKey("hbase.testTable3")); + } +} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseKerberosAuthentication.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java similarity index 86% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseKerberosAuthentication.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java index f9a51c56f..414c48603 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseKerberosAuthentication.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java @@ -12,9 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test; +package io.hetu.core.plugin.hbase.security; -import io.hetu.core.plugin.hbase.security.HBaseKerberosAuthentication; +import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.testng.annotations.Test; @@ -87,7 +87,7 @@ public class TestHBaseKerberosAuthentication } // krb5ConfPath is not a file - TestJsonHBaseTableUtils.createFile("user.keytab.test"); + TestHBaseTableUtils.createFile("user.keytab.test"); try { HBaseKerberosAuthentication.authenticateAndReturnUGI( "root", "./user.keytab.test", "./", new HBaseConfiguration()); @@ -97,7 +97,7 @@ public class TestHBaseKerberosAuthentication assertEquals(true, e.toString().contains("java.io.IOException: krb5ConfFile"), e.toString()); assertEquals(true, e.toString().contains("is not a file"), e.toString()); } - TestJsonHBaseTableUtils.delFile("user.keytab"); + TestHBaseTableUtils.delFile("user.keytab"); } /** @@ -108,8 +108,8 @@ public class TestHBaseKerberosAuthentication throws Exception { // krb5ConfPath is not a file - TestJsonHBaseTableUtils.createFile("user.keytab.test"); - TestJsonHBaseTableUtils.createFile("krb5.conf.test"); + TestHBaseTableUtils.createFile("user.keytab.test"); + TestHBaseTableUtils.createFile("krb5.conf.test"); try { HBaseKerberosAuthentication.authenticateAndReturnUGI( "root", "./user.keytab.test", "./krb5.conf.test", new HBaseConfiguration()); @@ -117,8 +117,8 @@ public class TestHBaseKerberosAuthentication catch (IllegalStateException | NullPointerException e) { throw new IllegalStateException("testAuthenticateAndReturnUGINormal: failed"); } - TestJsonHBaseTableUtils.delFile("user.keytab.test"); - TestJsonHBaseTableUtils.delFile("krb5.conf.test"); + TestHBaseTableUtils.delFile("user.keytab.test"); + TestHBaseTableUtils.delFile("krb5.conf.test"); } /** @@ -141,8 +141,8 @@ public class TestHBaseKerberosAuthentication // userKeytabFile not exist try { - TestJsonHBaseTableUtils.delFile("krb5.conf.test"); - TestJsonHBaseTableUtils.delFile("jaas.conf.test"); + TestHBaseTableUtils.delFile("krb5.conf.test"); + TestHBaseTableUtils.delFile("jaas.conf.test"); HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test"); throw new IOException("testAuthenticate : failed"); } @@ -152,8 +152,8 @@ public class TestHBaseKerberosAuthentication } // Complete process - TestJsonHBaseTableUtils.createFile("krb5.conf.test"); - TestJsonHBaseTableUtils.createFile("jaas.conf.test"); + TestHBaseTableUtils.createFile("krb5.conf.test"); + TestHBaseTableUtils.createFile("jaas.conf.test"); try { HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test"); throw new IOException("testAuthenticate : failed"); @@ -163,7 +163,7 @@ public class TestHBaseKerberosAuthentication e.toString(), "java.io.IOException: AppConfigurationEntry named jaas.conf.test does not have value of keyTab."); } - TestJsonHBaseTableUtils.delFile("krb5.conf.test"); - TestJsonHBaseTableUtils.delFile("jaas.conf.test"); + TestHBaseTableUtils.delFile("krb5.conf.test"); + TestHBaseTableUtils.delFile("jaas.conf.test"); } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHBaseSplit.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHBaseSplit.java similarity index 95% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHBaseSplit.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHBaseSplit.java index f99b16d7a..56e4b661c 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHBaseSplit.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHBaseSplit.java @@ -12,10 +12,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test.split; +package io.hetu.core.plugin.hbase.split; import io.hetu.core.plugin.hbase.client.TestUtils; -import io.hetu.core.plugin.hbase.split.HBaseSplit; import io.prestosql.spi.HostAddress; import org.testng.annotations.Test; diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHbaseSplitManager.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java similarity index 87% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHbaseSplitManager.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java index 412749467..a0dd888de 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHbaseSplitManager.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java @@ -12,14 +12,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test.split; +package io.hetu.core.plugin.hbase.split; import io.hetu.core.plugin.hbase.client.TestUtils; import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.connector.HBaseConnection; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; -import io.hetu.core.plugin.hbase.split.HBaseSplitManager; -import io.hetu.core.plugin.hbase.test.TestHBaseClientConnection; +import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; +import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -34,6 +34,7 @@ import static org.testng.Assert.assertEquals; */ public class TestHbaseSplitManager { + private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseSplitManager hsm; @@ -46,8 +47,7 @@ public class TestHbaseSplitManager { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini"); - hconn = new TestHBaseClientConnection(hCConf); + hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); hconn.getConn(); hsm = new HBaseSplitManager(hconn); } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHdfsHBaseMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHdfsHBaseMetastore.java deleted file mode 100644 index d9c31df83..000000000 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHdfsHBaseMetastore.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.hbase.test; - -import io.hetu.core.plugin.hbase.client.TestUtils; -import io.hetu.core.plugin.hbase.conf.HBaseConfig; -import io.hetu.core.plugin.hbase.metadata.HBaseTable; -import io.hetu.core.plugin.hbase.metadata.HdfsHBaseMetastore; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; -import org.codehaus.jettison.json.JSONObject; -import org.mockito.Mockito; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.FileNotFoundException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.Optional; - -import static org.testng.Assert.assertEquals; - -/** - * TestHdfsHBaseMetastore - * - * @since 2020-03-20 - */ -public class TestHdfsHBaseMetastore -{ - private HdfsHBaseMetastore hdfsHBaseMetastore; - private HBaseConfig hCConf = new HBaseConfig(); - - /** - * setUp - */ - @BeforeClass - public void setUp() - { - hCConf.setZkClientPort("2181"); - hCConf.setZkQuorum("zk1"); - hCConf.setMetastoreType("hdfs"); - hCConf.setHdfsSitePath("./hdfs-site.xml.test"); - hCConf.setCoreSitePath("./core-site.xml.test"); - hCConf.setKerberos("KERBEROS"); - hCConf.setUserKeytabPath("./user.keytab.test"); - hCConf.setKrb5ConfPath("./krb5.conf.test"); - hCConf.setPrincipalUsername("root"); - hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini"); - TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl()); - hdfsHBaseMetastore = new HdfsHBaseMetastore(hCConf); - } - - /** - * testInit - */ - @Test - public void testInit() - { - hdfsHBaseMetastore.init(); - } - - /** - * testAuthenticate - * - * @throws FileNotFoundException InvocationTargetException Exception - */ - @Test - public void testAuthenticate() - throws Exception - { - Method method = HdfsHBaseMetastore.class.getDeclaredMethod("authenticate"); - method.setAccessible(true); - try { - TestJsonHBaseTableUtils.createFile("./hdfs-site.xml.test"); - method.invoke(hdfsHBaseMetastore); - } - catch (FileNotFoundException e) { - assertEquals(e.getMessage(), null); - } - - try { - TestJsonHBaseTableUtils.createFile("./core-site.xml.test"); - method.invoke(hdfsHBaseMetastore); - } - catch (FileNotFoundException e) { - assertEquals(e.getMessage(), null); - } - - try { - TestJsonHBaseTableUtils.createFile("./user.keytab.test"); - method.invoke(hdfsHBaseMetastore); - } - catch (FileNotFoundException e) { - assertEquals(e.getMessage(), null); - } - - try { - TestJsonHBaseTableUtils.createFile("./krb5.conf.test"); - method.invoke(hdfsHBaseMetastore); - } - catch (InvocationTargetException e) { - assertEquals(e.getMessage(), null); - } - - TestJsonHBaseTableUtils.delFile("./hdfs-site.xml.test"); - TestJsonHBaseTableUtils.delFile("./core-site.xml.test"); - TestJsonHBaseTableUtils.delFile("./user.keytab.test"); - TestJsonHBaseTableUtils.delFile("./krb5.conf.test"); - } - - /** - * testGetId - */ - @Test - public void testGetId() - { - assertEquals("HDFS", hdfsHBaseMetastore.getId()); - } - - /** - * testOPHBaseTable - */ - @Test - public void testOPHBaseTable() - { - HdfsHBaseMetastore hDFSHBaseMetastore = Mockito.spy(new HdfsHBaseMetastore(hCConf)); - Optional serializerClassName = - Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer"); - HBaseTable hBaseTable = - new HBaseTable( - "hbase", - "oldtable", - TestUtils.createColumnList(), - "rowkey", - false, - serializerClassName, - Optional.of(""), - Optional.of("oldtable")); - Mockito.doNothing().when(hDFSHBaseMetastore).putJsonToHdfs(Mockito.anyString(), Mockito.any(JSONObject.class)); - hDFSHBaseMetastore.addHBaseTable(hBaseTable); - - HBaseTable hBaseNewTable = - new HBaseTable( - "hbase", - "newtable", - TestUtils.createColumnList(), - "rowkey", - false, - serializerClassName, - Optional.of(""), - Optional.of("newtable")); - - hDFSHBaseMetastore.addHBaseTable(hBaseNewTable); - // add an existing hBaseTable - hDFSHBaseMetastore.addHBaseTable(hBaseNewTable); - Map hbaseTables = hDFSHBaseMetastore.getAllHBaseTables(); - assertEquals(true, hbaseTables.containsKey("hbase.oldtable")); - assertEquals(true, hbaseTables.containsKey("hbase.newtable")); - - HBaseTable hBaseOldTable = hDFSHBaseMetastore.getHBaseTable("hbase.oldtable"); - assertEquals("hbase.oldtable", hBaseOldTable.getFullTableName()); - - hDFSHBaseMetastore.renameHBaseTable(hBaseNewTable, "hbase.oldtable"); - hbaseTables = hDFSHBaseMetastore.getAllHBaseTables(); - assertEquals(false, hbaseTables.containsKey("hbase.oldtable")); - assertEquals(true, hbaseTables.containsKey("hbase.newtable")); - - hDFSHBaseMetastore.dropHBaseTable(hBaseNewTable); - hbaseTables = hDFSHBaseMetastore.getAllHBaseTables(); - assertEquals(false, hbaseTables.containsKey("hbase.oldtable")); - assertEquals(false, hbaseTables.containsKey("hbase.newtable")); - } - - /** - * testLocalHBaseMetastore - */ - @Test - public void testLocalHBaseMetastore() - { - HBaseConfig conf = new HBaseConfig(); - conf.setMetastoreUrl("LOCAL"); - conf.setMetastoreUrl("./localHBaseMetastore.ini"); - LocalHBaseMetastore localHBaseMetastore = new LocalHBaseMetastore(conf); - - Optional serializerClassName = - Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer"); - HBaseTable hBaseTable = - new HBaseTable( - "hbase", - "oldtable", - TestUtils.createColumnList(), - "rowkey", - false, - serializerClassName, - Optional.of(""), - Optional.of("oldtable")); - localHBaseMetastore.addHBaseTable(hBaseTable); - localHBaseMetastore.dropHBaseTable(hBaseTable); - - TestJsonHBaseTableUtils.delFile(conf.getMetastoreUrl()); - } - - /** - * clear - */ - @AfterClass - public void clear() - { - TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl()); - TestJsonHBaseTableUtils.delFile("./.hbasetablecatalogtmp.ini.crc"); - TestJsonHBaseTableUtils.delFile("./hdfs-site.xml.test"); - TestJsonHBaseTableUtils.delFile("./core-site.xml.test"); - TestJsonHBaseTableUtils.delFile("./user.keytab.test"); - TestJsonHBaseTableUtils.delFile("./krb5.conf.test"); - } -} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/AbstractTestHBaseRowSerializer.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/AbstractTestHBaseRowSerializer.java similarity index 99% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/AbstractTestHBaseRowSerializer.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/AbstractTestHBaseRowSerializer.java index 8886ddc73..4fd4779da 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/AbstractTestHBaseRowSerializer.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/AbstractTestHBaseRowSerializer.java @@ -12,11 +12,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test.utils.serializers; +package io.hetu.core.plugin.hbase.utils.serializers; import io.airlift.slice.Slice; import io.hetu.core.plugin.hbase.utils.TestSliceUtils; -import io.hetu.core.plugin.hbase.utils.serializers.HBaseRowSerializer; import io.prestosql.spi.PrestoException; import io.prestosql.spi.block.Block; import io.prestosql.spi.block.ShortArrayBlock; diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/TestStringRowSerializer.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/TestStringRowSerializer.java similarity index 88% rename from hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/TestStringRowSerializer.java rename to hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/TestStringRowSerializer.java index 1df6b441c..1138dc63e 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/TestStringRowSerializer.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/TestStringRowSerializer.java @@ -12,9 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.hetu.core.plugin.hbase.test.utils.serializers; - -import io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer; +package io.hetu.core.plugin.hbase.utils.serializers; /** * TestStringRowSerializer From d83edd7e62a8a57f55311a7a59c59627f3612abd Mon Sep 17 00:00:00 2001 From: guojunfei <970763131@qq.com> Date: Tue, 8 Sep 2020 17:58:09 +0800 Subject: [PATCH 2/2] remove hbase local file metastore --- .../io/hetu/core/plugin/hbase/TestHBase.java | 27 ++- .../io/hetu/core/plugin/hbase/TestQuery.java | 29 ++- .../hbase/connector/TestHBaseConnector.java | 52 ++--- .../hbase/metadata/JsonHBaseTableUtils.java | 125 ---------- .../hbase/metadata/LocalHBaseMetastore.java | 216 ------------------ .../hbase/metadata/TestHBaseTableUtils.java | 139 ----------- .../metadata/TestHetuHBaseMetastore.java | 48 +--- .../metadata/TestingHBaseTableUtils.java | 54 +++++ .../hbase/metadata/TestingHetuMetastore.java | 115 ++++++++++ .../TestHBaseKerberosAuthentication.java | 26 +-- .../hbase/split/TestHbaseSplitManager.java | 14 +- 11 files changed, 237 insertions(+), 608 deletions(-) delete mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java delete mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java delete mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java create mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHBaseTableUtils.java create mode 100644 hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java index 68be9d3da..cea469c60 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java @@ -25,8 +25,7 @@ import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.connector.HBaseTransactionHandle; import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; -import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; +import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore; import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider; import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider; import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider; @@ -76,7 +75,6 @@ import static org.testng.Assert.assertEquals; */ public class TestHBase { - private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseConnectorMetadata hcm; @@ -84,6 +82,7 @@ public class TestHBase private SchemaTableName schemaTableName; private ConnectorTableHandle table; private HBaseConnector hConnector; + private TestingHetuMetastore hetuMetastore; /** * setUp @@ -93,10 +92,10 @@ public class TestHBase { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); + hetuMetastore = new TestingHetuMetastore(); table = TestUtils.createHBaseTableHandle(); schemaTableName = new SchemaTableName("hbase", "test_table"); - hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); + hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore()); hconn.getConn(); session = new TestingConnectorSession("root"); hcm = new HBaseConnectorMetadata(hconn); @@ -110,6 +109,15 @@ public class TestHBase null); } + /** + * clear + */ + @AfterClass + public void clear() + { + hetuMetastore.close(); + } + /** * testGetHAdmin */ @@ -477,13 +485,4 @@ public class TestHBase cps.abort(); } } - - /** - * clear - */ - @AfterClass - public void clear() - { - TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); - } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java index 1ba56c0c6..6b6b1deb9 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java @@ -24,8 +24,7 @@ import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; import io.hetu.core.plugin.hbase.connector.HBaseConnection; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; -import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; +import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore; import io.hetu.core.plugin.hbase.query.HBaseGetRecordCursor; import io.hetu.core.plugin.hbase.query.HBaseRecordCursor; import io.hetu.core.plugin.hbase.query.HBaseRecordSet; @@ -66,12 +65,11 @@ import static org.testng.Assert.assertEquals; */ public class TestQuery { - private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; - private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private ConnectorSession session; private HBaseRecordSet recordSet; + private TestingHetuMetastore hetuMetastore; private HBaseSplit split; /** @@ -82,8 +80,8 @@ public class TestQuery { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); - hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); + hetuMetastore = new TestingHetuMetastore(); + hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore()); hconn.getConn(); session = new TestingConnectorSession("root"); split = @@ -101,6 +99,15 @@ public class TestQuery hconn, session, split, TestUtils.createHBaseTableHandle(), TestUtils.createColumnList()); } + /** + * clear + */ + @AfterClass + public void clear() + { + hetuMetastore.close(); + } + /** * testHBaseRecordSetCursorIsBatchGet */ @@ -244,7 +251,6 @@ public class TestQuery */ @Test public void testHBaseRecordCursor() - throws Exception { List columnHandles = new ArrayList<>(); columnHandles.add(TestUtils.createHBaseColumnRowId("rowkey")); @@ -335,13 +341,4 @@ public class TestQuery // Check read data assertEquals("nick_name_1 12 2019-06-11 20", serializer.getColumnValues().get("rowkey")); } - - /** - * clear - */ - @AfterClass - public void clear() - { - TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); - } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java index d4e7fd710..2c7a0999f 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java @@ -20,8 +20,7 @@ import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.conf.HBaseTableProperties; import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata; import io.hetu.core.plugin.hbase.metadata.HBaseTable; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; -import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; +import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore; import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider; import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider; import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider; @@ -74,13 +73,13 @@ import static org.testng.Assert.assertEquals; */ public class TestHBaseConnector { - private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseConnectorMetadata hcm; private SchemaTableName schemaTableName; private HBaseConnector hConnector; private ConnectorSession session; + private TestingHetuMetastore hetuMetastore; /** * setUp @@ -90,9 +89,9 @@ public class TestHBaseConnector { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - TestHBaseTableUtils.preFile(TEST_METASTORE_FILE_PATH); + hetuMetastore = new TestingHetuMetastore(); schemaTableName = new SchemaTableName("hbase", "test_table"); - hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); + hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore()); hconn.getConn(); session = new TestingConnectorSession("root"); hcm = new HBaseConnectorMetadata(hconn); @@ -106,6 +105,15 @@ public class TestHBaseConnector new HBaseTableProperties()); } + /** + * clear + */ + @AfterClass + public void clear() + { + hetuMetastore.close(); + } + /** * testDropSchema */ @@ -189,29 +197,6 @@ public class TestHBaseConnector assertEquals("test_table", map.get("hbase").get(0).getTableName()); } - /** - * testCreateNewTable - */ - @Test - public void testCreateNewTable() - { - hcm.createTable(session, TestUtils.createConnectorTableMeta(), false); - } - - /** - * testCreateExistTable - */ - @Test - public void testCreateExistTable() - { - Map properties = TestUtils.createProperties(); - properties.put("hbase_table_name", "hbase:test_table"); - ConnectorTableMetadata ctm = - new ConnectorTableMetadata( - new SchemaTableName("hbase", "test_table"), TestUtils.createColumnMetadataList(), properties); - hcm.createTable(session, ctm, false); - } - /** * testApplyFilter */ @@ -354,7 +339,7 @@ public class TestHBaseConnector hCnnConf.setZkClientPort("2181"); hCnnConf.setZkQuorum("zk1"); hCnnConf.setRetryNumber(1); - HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf, new LocalHBaseMetastore(null)); + HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf, null); try { hConn.createSchema("schema", null); @@ -599,13 +584,4 @@ public class TestHBaseConnector assertEquals(hConnector.getTableProperties().size(), 7); assertEquals(hConnector.getColumnProperties().size(), 2); } - - /** - * clear - */ - @AfterClass - public void clear() - { - TestHBaseTableUtils.delFile(TEST_METASTORE_FILE_PATH); - } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java deleted file mode 100644 index 07c5ff409..000000000 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/JsonHBaseTableUtils.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.hbase.metadata; - -import io.airlift.log.Logger; -import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; -import io.hetu.core.plugin.hbase.utils.Utils; -import io.prestosql.spi.connector.ColumnHandle; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -/** - * JsonHBaseTableUtils - * - * @since 2020-03-30 - */ -public class JsonHBaseTableUtils -{ - private static final Logger LOG = Logger.get(JsonHBaseTableUtils.class); - - private JsonHBaseTableUtils() {} - - /** - * loadHBaseTablesFromJson - * - * @param hbaseTables hbaseTables map - * @param jsonObject jsonObject - */ - public static void loadHBaseTablesFromJson(Map hbaseTables, JSONObject jsonObject) - { - if (jsonObject == null) { - return; - } - - Iterator iterator = jsonObject.keys(); - String key; - List columns; - JSONObject jsonHtable; - Map columnHandleMap; - - try { - while (iterator.hasNext()) { - columnHandleMap = new ConcurrentHashMap<>(); - key = iterator.next(); - jsonHtable = jsonObject.getJSONObject(key); - columns = new ArrayList<>(); - JSONArray jaColumns = jsonHtable.getJSONArray("columns"); - if (jaColumns.length() > 0) { - for (int i = 0; i < jaColumns.length(); i++) { - JSONObject jsonColumns = jaColumns.getJSONObject(i); - HBaseColumnHandle columnHandle = - new HBaseColumnHandle( - jsonColumns.getString("name"), - Optional.of(jsonColumns.getString("family")), - Optional.of(jsonColumns.getString("qualifer")), - Utils.createTypeByName(jsonColumns.getString("type")), - jsonColumns.getInt("ordinal"), - jsonColumns.getString("comment"), - jsonColumns.getBoolean("indexed")); - columns.add(columnHandle); - columnHandleMap.put(columnHandle.getName(), columnHandle); - } - } - - HBaseTable hbaseTable = - new HBaseTable( - jsonHtable.getString("schema"), - jsonHtable.getString("table"), - columns, - jsonHtable.getString("rowId"), - jsonHtable.getBoolean("external"), - Optional.of(jsonHtable.getString("serializerClassName")), - Optional.of(jsonHtable.getString("indexColumns")), - Optional.of(jsonHtable.getString("hbaseTableName"))); - hbaseTable.setColumnsToMap(columnHandleMap); - hbaseTables.put(key, hbaseTable); - } - } - catch (JSONException e) { - LOG.error("loadHbaseTableFromFile faild... cause by %s", e); - } - } - - /** - * hbaseTablesMapToJson - * - * @param hbaseTablesMap hbaseTables Map - * @return JSONObject - */ - public static JSONObject hbaseTablesMapToJson(Map hbaseTablesMap) - { - JSONObject hbaseTablesJson = new JSONObject(); - - try { - for (Map.Entry entry : hbaseTablesMap.entrySet()) { - hbaseTablesJson.put(entry.getKey(), entry.getValue().parseHbaseTableToJson()); - } - } - catch (JSONException e) { - LOG.error("hbaseTablesMapToJson faild... cause by %s", e); - } - - return hbaseTablesJson; - } -} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java deleted file mode 100644 index a0124781e..000000000 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.hbase.metadata; - -import com.google.common.collect.ImmutableMap; -import io.airlift.log.Logger; -import io.hetu.core.plugin.hbase.utils.Constants; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.Reader; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * LocalHBaseMetastore: store file on local filesystem - * This Metastore is just for test - * - * @since 2020-03-30 - */ -public class LocalHBaseMetastore - implements HBaseMetastore -{ - private static final Logger LOG = Logger.get(LocalHBaseMetastore.class); - private static final String ID = "LOCAL"; - private final Map hbaseTables = new ConcurrentHashMap<>(); - private final String filePath; - - public LocalHBaseMetastore(String filePath) - { - this.filePath = filePath; - } - - @Override - public void init() - { - JSONObject jsonObject = readJsonFromFile(filePath); - JsonHBaseTableUtils.loadHBaseTablesFromJson(hbaseTables, jsonObject); - } - - @Override - public String getId() - { - return this.ID; - } - - @Override - public void addHBaseTable(HBaseTable hBaseTable) - { - // hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified - synchronized (hbaseTables) { - if (hbaseTables.containsKey(hBaseTable.getFullTableName())) { - hbaseTables.remove(hBaseTable.getFullTableName()); - } - - hbaseTables.put(hBaseTable.getFullTableName(), hBaseTable); - putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - - @Override - public void renameHBaseTable(HBaseTable newTable, String oldTable) - { - // hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified - synchronized (hbaseTables) { - if (hbaseTables.containsKey(oldTable)) { - hbaseTables.remove(oldTable); - } - - hbaseTables.put(newTable.getFullTableName(), newTable); - putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - - @Override - public void dropHBaseTable(HBaseTable hBaseTable) - { - synchronized (hbaseTables) { - if (hbaseTables.containsKey(hBaseTable.getFullTableName())) { - hbaseTables.remove(hBaseTable.getFullTableName()); - putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables)); - } - } - } - - @Override - public synchronized Map getAllHBaseTables() - { - return ImmutableMap.copyOf(hbaseTables); - } - - @Override - public HBaseTable getHBaseTable(String tableName) - { - return hbaseTables.get(tableName); - } - - /** - * readJsonFromFile - * - * @param file file - * @return JSONObject - */ - public JSONObject readJsonFromFile(String file) - { - Reader in = null; - JSONObject rs = null; - StringBuilder catalogs = new StringBuilder(); - try { - File metaFile = new File(file); - if (!metaFile.exists()) { - return rs; - } - char[] buffer = new char[Constants.NUMBER1024]; - in = new FileReader(file); - int len = in.read(buffer); - while (len != Constants.NUMBER_NEGATIVE_1) { - if (len == Constants.NUMBER1024) { - catalogs.append(buffer); - } - else { - catalogs.append(buffer, 0, len); - } - len = in.read(buffer); - } - } - catch (IOException e) { - LOG.error("readJsonFromFile : some wrong...file [%s]. Cause by %s", file, e.getMessage()); - } - finally { - if (in != null) { - try { - in.close(); - } - catch (IOException e) { - LOG.error("readJsonFromFile : something wrong...file [%s]. Cause by %s", file, e.getMessage()); - } - } - } - - try { - if (catalogs.toString().length() > 0) { - rs = new JSONObject(catalogs.toString()); - } - } - catch (JSONException e) { - LOG.error("readJsonFromFile : to json wrong...file[%s]. Cause by %s", file, e.getMessage()); - } - return rs; - } - - /** - * putJsonToFile - * - * @param file file - * @param jsonObject jsonObject - */ - public void putJsonToFile(String file, JSONObject jsonObject) - { - PrintWriter out = null; - String jsonStr = ""; - - try { - jsonStr = jsonObject.toString(Constants.NUMBER4); - } - catch (JSONException e) { - LOG.error("putJsonToFile : json to string wrong...cause by", e.getMessage()); - } - - // check file exist and file permission - File metaFile = new File(file); - if (!metaFile.exists()) { - try { - if (!metaFile.createNewFile()) { - throw new IOException("when create file, it return false"); - } - } - catch (IOException e) { - LOG.error("create file[%s] failed... check permission is available", e); - } - } - - try { - // overwrite - out = new PrintWriter(new FileWriter(file, false)); - // write file by json format - out.write(jsonStr); - out.flush(); - } - catch (IOException e) { - LOG.error("putJsonToFile : some wrong...cause by", e.getMessage()); - } - finally { - if (out != null) { - out.close(); - } - } - } -} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java deleted file mode 100644 index bf55ec147..000000000 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTableUtils.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.hetu.core.plugin.hbase.metadata; - -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -import java.io.File; -import java.io.IOException; - -import static org.testng.Assert.assertEquals; - -/** - * TestJsonHBaseTableUtils - * - * @since 2020-03-20 - */ -public class TestHBaseTableUtils -{ - private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; - private static LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH); - private static String jsonStr = - "{\n" - + " \"hbase.test_table\": {\n" - + " \"schema\": \"hbase\",\n" - + " \"external\": false,\n" - + " \"rowIdOrdinal\": 0,\n" - + " \"serializerClassName\":" - + " \"io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer\",\n" - + " \"indexed\": false,\n" - + " \"rowId\": \"rowkey\",\n" - + " \"table\": \"test_table\",\n" - + " \"indexColumns\": \"\",\n" - + " \"hbaseTableName\": \"hbase:test_table\",\n" - + " \"columns\": [\n" - + " {\n" - + " \"name\": \"rowkey\",\n" - + " \"family\": \"\",\n" - + " \"qualifer\": \"\",\n" - + " \"type\": \"io.prestosql.spi.type.VarcharType\",\n" - + " \"ordinal\": 0,\n" - + " \"comment\": \"HBase row ID\",\n" - + " \"indexed\": false\n" - + " },\n" - + " {\n" - + " \"name\": \"name\",\n" - + " \"family\": \"name\",\n" - + " \"qualifer\": \"nick_name\",\n" - + " \"type\": \"io.prestosql.spi.type.VarcharType\",\n" - + " \"ordinal\": 1,\n" - + " \"comment\": \"HBase column name:nick_name. Indexed: false\",\n" - + " \"indexed\": false\n" - + " },\n" - + " {\n" - + " \"name\": \"age\",\n" - + " \"family\": \"age\",\n" - + " \"qualifer\": \"lit_age\",\n" - + " \"type\": \"io.prestosql.spi.type.BigintType\",\n" - + " \"ordinal\": 2,\n" - + " \"comment\": \"HBase column age:lit_age. Indexed: false\",\n" - + " \"indexed\": false\n" - + " },\n" - + " {\n" - + " \"name\": \"gender\",\n" - + " \"family\": \"gender\",\n" - + " \"qualifer\": \"gender\",\n" - + " \"type\": \"io.prestosql.spi.type.DateType\",\n" - + " \"ordinal\": 3,\n" - + " \"comment\": \"HBase column gender:gender. Indexed: false\",\n" - + " \"indexed\": false\n" - + " },\n" - + " {\n" - + " \"name\": \"t\",\n" - + " \"family\": \"t\",\n" - + " \"qualifer\": \"t\",\n" - + " \"type\": \"io.prestosql.spi.type.BigintType\",\n" - + " \"ordinal\": 4,\n" - + " \"comment\": \"HBase column t:t. Indexed: false\",\n" - + " \"indexed\": false\n" - + " }\n" - + " ]\n" - + " }}"; - - private TestHBaseTableUtils() {} - - /** - * preFile - * - * @throws JSONException Exception - */ - public static void preFile(String file) - { - try { - JSONObject json = new JSONObject(jsonStr); - lHBMetastore.putJsonToFile(file, json); - } - catch (JSONException e) { - assertEquals(e.toString().substring(0, 36), "putJsonToFile : json to string wrong"); - } - } - - /** - * delFile - */ - public static void delFile(String file) - { - File pfile = new File(file); - pfile.delete(); - } - - /** - * createFile - */ - public static void createFile(String filename) - throws Exception - { - File file = new File(filename); - if (!file.exists()) { - try { - file.createNewFile(); - } - catch (IOException e) { - throw new IOException("createFile " + filename + " : failed"); - } - } - } -} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java index cf8a0795f..197c69de5 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java @@ -14,25 +14,14 @@ */ package io.hetu.core.plugin.hbase.metadata; -import com.google.common.collect.ImmutableMap; -import com.google.inject.Injector; -import io.airlift.bootstrap.Bootstrap; -import io.airlift.testing.mysql.TestingMySqlServer; -import io.hetu.core.metastore.jdbc.JdbcMetastoreModule; import io.hetu.core.plugin.hbase.client.TestUtils; -import io.prestosql.plugin.base.jmx.MBeanServerModule; -import io.prestosql.spi.PrestoException; -import io.prestosql.spi.metastore.HetuMetastore; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import org.weakref.jmx.guice.MBeanModule; import java.util.Map; import java.util.Optional; -import static com.google.common.base.Throwables.throwIfUnchecked; -import static io.prestosql.spi.metastore.HetuErrorCode.HETU_METASTORE_CODE; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertFalse; import static org.testng.Assert.assertNotNull; @@ -40,49 +29,20 @@ import static org.testng.Assert.assertTrue; public class TestHetuHBaseMetastore { - private static final String TEST_DATABASES = "hbase"; - private static final String TEST_MYSQL_USER = "user"; - private static final String TEST_MYSQL_PASSWORD = "testpass"; - - private TestingMySqlServer mySqlServer; + private TestingHetuMetastore hetuMetastore; private HetuHBaseMetastore metaStore; @BeforeClass public void setUp() - throws Exception { - this.mySqlServer = new TestingMySqlServer(TEST_MYSQL_USER, TEST_MYSQL_PASSWORD, TEST_DATABASES); - Map properties = new ImmutableMap.Builder() - .put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES)) - .put("hetu.metastore.db.user", TEST_MYSQL_USER) - .put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD) - .build(); - try { - Bootstrap app = new Bootstrap( - new MBeanModule(), - new MBeanServerModule(), - new JdbcMetastoreModule()); - - Injector injector = app - .strictConfig() - .doNotInitializeLogging() - .setRequiredConfigurationProperties(properties) - .initialize(); - - this.metaStore = new HetuHBaseMetastore(injector.getInstance(HetuMetastore.class)); - metaStore.init(); - } - catch (Exception ex) { - throwIfUnchecked(ex); - throw new PrestoException(HETU_METASTORE_CODE, - "init hetu hbase metastore module failed."); - } + hetuMetastore = new TestingHetuMetastore(); + metaStore = hetuMetastore.getHetuMetastore(); } @AfterClass(alwaysRun = true) public void destroy() { - mySqlServer.close(); + hetuMetastore.close(); } @Test diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHBaseTableUtils.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHBaseTableUtils.java new file mode 100644 index 000000000..4d9fe8708 --- /dev/null +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHBaseTableUtils.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.hbase.metadata; + +import java.io.File; +import java.io.IOException; + +/** + * TestJsonHBaseTableUtils + * + * @since 2020-03-20 + */ +public class TestingHBaseTableUtils +{ + private TestingHBaseTableUtils() {} + + /** + * delFile + */ + public static void delFile(String file) + { + File pfile = new File(file); + pfile.delete(); + } + + /** + * createFile + */ + public static void createFile(String filename) + throws Exception + { + File file = new File(filename); + if (!file.exists()) { + try { + file.createNewFile(); + } + catch (IOException e) { + throw new IOException("createFile " + filename + " : failed"); + } + } + } +} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java new file mode 100644 index 000000000..4f5142302 --- /dev/null +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java @@ -0,0 +1,115 @@ +/* + * Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.hetu.core.plugin.hbase.metadata; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; +import io.airlift.testing.mysql.TestingMySqlServer; +import io.hetu.core.metastore.jdbc.JdbcMetastoreModule; +import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle; +import io.prestosql.plugin.base.jmx.MBeanServerModule; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.metastore.HetuMetastore; +import io.prestosql.spi.type.BigintType; +import io.prestosql.spi.type.DateType; +import io.prestosql.spi.type.VarcharType; +import org.weakref.jmx.guice.MBeanModule; + +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static io.prestosql.spi.metastore.HetuErrorCode.HETU_METASTORE_CODE; + +public class TestingHetuMetastore + implements AutoCloseable +{ + private static final String TEST_DATABASES = "hbase"; + private static final String TEST_MYSQL_USER = "user"; + private static final String TEST_MYSQL_PASSWORD = "testpass"; + private static final HBaseTable TEST_TABLE = new HBaseTable( + "hbase", "test_table", + ImmutableList.of( + new HBaseColumnHandle("rowkey", Optional.of(""), Optional.of(""), VarcharType.VARCHAR, + 0, "HBase row ID", false), + new HBaseColumnHandle("name", Optional.of("name"), Optional.of("nick_name"), VarcharType.VARCHAR, + 1, "HBase column name:nick_name. Indexed: false", false), + new HBaseColumnHandle("age", Optional.of("age"), Optional.of("lit_age"), BigintType.BIGINT, + 2, "HBase column age:lit_age. Indexed: false", false), + new HBaseColumnHandle("gender", Optional.of("gender"), Optional.of("gender"), DateType.DATE, + 3, "HBase column gender:gender. Indexed:false", false), + new HBaseColumnHandle("t", Optional.of("t"), Optional.of("t"), BigintType.BIGINT, + 4, "HBase column t:t. Indexed: false", false)), + "rowkey", + false, + Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer"), + Optional.empty(), + Optional.of("hbase:test_table")); + + private final TestingMySqlServer mySqlServer; + private final HetuHBaseMetastore metaStore; + + public TestingHetuMetastore() + { + try { + this.mySqlServer = new TestingMySqlServer(TEST_MYSQL_USER, TEST_MYSQL_PASSWORD, TEST_DATABASES); + Map properties = new ImmutableMap.Builder() + .put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES)) + .put("hetu.metastore.db.user", TEST_MYSQL_USER) + .put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD) + .build(); + + Bootstrap app = new Bootstrap( + new MBeanModule(), + new MBeanServerModule(), + new JdbcMetastoreModule()); + + Injector injector = app + .strictConfig() + .doNotInitializeLogging() + .setRequiredConfigurationProperties(properties) + .initialize(); + + this.metaStore = new HetuHBaseMetastore(injector.getInstance(HetuMetastore.class)); + metaStore.init(); + Map map = TEST_TABLE.getColumns().stream().collect(Collectors.toMap( + HBaseColumnHandle::getColumnName, + Function.identity())); + TEST_TABLE.setColumnsToMap(map); + metaStore.addHBaseTable(TEST_TABLE); + } + catch (Exception ex) { + throwIfUnchecked(ex); + throw new PrestoException(HETU_METASTORE_CODE, + "init hetu hbase metastore module failed."); + } + } + + public HetuHBaseMetastore getHetuMetastore() + { + return metaStore; + } + + @Override + public void close() + { + mySqlServer.close(); + } +} diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java index 414c48603..9ba37409d 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java @@ -14,7 +14,7 @@ */ package io.hetu.core.plugin.hbase.security; -import io.hetu.core.plugin.hbase.metadata.TestHBaseTableUtils; +import io.hetu.core.plugin.hbase.metadata.TestingHBaseTableUtils; import org.apache.hadoop.hbase.HBaseConfiguration; import org.testng.annotations.Test; @@ -87,7 +87,7 @@ public class TestHBaseKerberosAuthentication } // krb5ConfPath is not a file - TestHBaseTableUtils.createFile("user.keytab.test"); + TestingHBaseTableUtils.createFile("user.keytab.test"); try { HBaseKerberosAuthentication.authenticateAndReturnUGI( "root", "./user.keytab.test", "./", new HBaseConfiguration()); @@ -97,7 +97,7 @@ public class TestHBaseKerberosAuthentication assertEquals(true, e.toString().contains("java.io.IOException: krb5ConfFile"), e.toString()); assertEquals(true, e.toString().contains("is not a file"), e.toString()); } - TestHBaseTableUtils.delFile("user.keytab"); + TestingHBaseTableUtils.delFile("user.keytab"); } /** @@ -108,8 +108,8 @@ public class TestHBaseKerberosAuthentication throws Exception { // krb5ConfPath is not a file - TestHBaseTableUtils.createFile("user.keytab.test"); - TestHBaseTableUtils.createFile("krb5.conf.test"); + TestingHBaseTableUtils.createFile("user.keytab.test"); + TestingHBaseTableUtils.createFile("krb5.conf.test"); try { HBaseKerberosAuthentication.authenticateAndReturnUGI( "root", "./user.keytab.test", "./krb5.conf.test", new HBaseConfiguration()); @@ -117,8 +117,8 @@ public class TestHBaseKerberosAuthentication catch (IllegalStateException | NullPointerException e) { throw new IllegalStateException("testAuthenticateAndReturnUGINormal: failed"); } - TestHBaseTableUtils.delFile("user.keytab.test"); - TestHBaseTableUtils.delFile("krb5.conf.test"); + TestingHBaseTableUtils.delFile("user.keytab.test"); + TestingHBaseTableUtils.delFile("krb5.conf.test"); } /** @@ -141,8 +141,8 @@ public class TestHBaseKerberosAuthentication // userKeytabFile not exist try { - TestHBaseTableUtils.delFile("krb5.conf.test"); - TestHBaseTableUtils.delFile("jaas.conf.test"); + TestingHBaseTableUtils.delFile("krb5.conf.test"); + TestingHBaseTableUtils.delFile("jaas.conf.test"); HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test"); throw new IOException("testAuthenticate : failed"); } @@ -152,8 +152,8 @@ public class TestHBaseKerberosAuthentication } // Complete process - TestHBaseTableUtils.createFile("krb5.conf.test"); - TestHBaseTableUtils.createFile("jaas.conf.test"); + TestingHBaseTableUtils.createFile("krb5.conf.test"); + TestingHBaseTableUtils.createFile("jaas.conf.test"); try { HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test"); throw new IOException("testAuthenticate : failed"); @@ -163,7 +163,7 @@ public class TestHBaseKerberosAuthentication e.toString(), "java.io.IOException: AppConfigurationEntry named jaas.conf.test does not have value of keyTab."); } - TestHBaseTableUtils.delFile("krb5.conf.test"); - TestHBaseTableUtils.delFile("jaas.conf.test"); + TestingHBaseTableUtils.delFile("krb5.conf.test"); + TestingHBaseTableUtils.delFile("jaas.conf.test"); } } diff --git a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java index a0dd888de..0acd66a36 100644 --- a/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java +++ b/hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java @@ -19,7 +19,8 @@ import io.hetu.core.plugin.hbase.conf.HBaseConfig; import io.hetu.core.plugin.hbase.connector.HBaseConnection; import io.hetu.core.plugin.hbase.connector.HBaseTableHandle; import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection; -import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore; +import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore; +import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -34,10 +35,10 @@ import static org.testng.Assert.assertEquals; */ public class TestHbaseSplitManager { - private static final String TEST_METASTORE_FILE_PATH = "./hbasetablecatalogtmp.ini"; private HBaseConnection hconn; private HBaseConfig hCConf = new HBaseConfig(); private HBaseSplitManager hsm; + private TestingHetuMetastore hetuMetastore; /** * setUp @@ -47,11 +48,18 @@ public class TestHbaseSplitManager { hCConf.setZkClientPort("2181"); hCConf.setZkQuorum("zk1"); - hconn = new TestHBaseClientConnection(hCConf, new LocalHBaseMetastore(TEST_METASTORE_FILE_PATH)); + hetuMetastore = new TestingHetuMetastore(); + hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore()); hconn.getConn(); hsm = new HBaseSplitManager(hconn); } + @AfterClass + public void destroy() + { + hetuMetastore.close(); + } + /** * testSplitManager *