jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!204 Remove hbase local&hdfs metastore module and add HBaseHetuMetastore's UT 

Merge pull request !204 from guojunfei399/master
This commit is contained in:
i-robot 2020-09-09 17:12:58 +08:00 committed by Gitee
parent 8365c8bb6c d83edd7e62
commit f8a2048422
30 changed files with 435 additions and 1278 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
hetu-hbase/pom.xml
View File

@ -41,6 +41,13 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.weakref</groupId>
<artifactId>jmxutils</artifactId>
<version>${dep.jmxutils.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>io.prestosql.hadoop</groupId>
<artifactId>hadoop-apache</artifactId>
@ -147,35 +154,12 @@
<version>${version.commons-lang3}</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${version.mockito-all}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${version.junit}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<artifactId>hamcrest-core</artifactId>
<groupId>org.hamcrest</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${version.protobuf-java}</version>
</dependency>
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<version>${version.testing}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
@ -318,6 +302,43 @@
<groupId>com.fasterxml.jackson.core</groupId>
<scope>runtime</scope>
</dependency>
<!-- for testing -->
<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-tests</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>presto-plugin-toolkit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.hetu.core</groupId>
<artifactId>hetu-metastore</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.airlift</groupId>
<artifactId>testing-mysql-server</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${version.mockito-all}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>

36
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/conf/HBaseConfig.java
View File

@ -31,11 +31,8 @@ public class HBaseConfig
private String zkQuorum;
private String zkClientPort;
private String metastoreType;
private String metastoreUrl;
private String defaultValue = "NULL";
private String jaasConfPath; // java.security.auth.login.config: jaas.conf
private String coreSitePath; // core-site.xml file path
private String hdfsSitePath; // hdfs-site.xml file path
private String hbaseSitePath; // hbase-site.xml file path
private String krb5ConfPath; // java.security.krb5.conf: krb5.conf
private String userKeytabPath; // user.keytab file path
@ -86,28 +83,6 @@ public class HBaseConfig
this.jaasConfPath = jaasConfPath;
}
public String getCoreSitePath()
{
return coreSitePath;
}
@Config("hbase.core.site.path")
public void setCoreSitePath(String coreSitePath)
{
this.coreSitePath = coreSitePath;
}
public String getHdfsSitePath()
{
return hdfsSitePath;
}
@Config("hbase.hdfs.site.path")
public void setHdfsSitePath(String hdfsSitePath)
{
this.hdfsSitePath = hdfsSitePath;
}
public String getHbaseSitePath()
{
return hbaseSitePath;
@ -185,17 +160,6 @@ public class HBaseConfig
this.metastoreType = metastoreType;
}
public String getMetastoreUrl()
{
return metastoreUrl;
}
@Config("hbase.metastore.uri")
public void setMetastoreUrl(String metastoreUrl)
{
this.metastoreUrl = metastoreUrl;
}
public String getKerberos()
{
return kerberos;

18
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/connector/HBaseConnection.java
View File

@ -104,10 +104,6 @@ public class HBaseConnection
* Configuration
*/
protected Configuration cfg;
/**
* catalogFile
*/
protected String catalogFile;
private UserGroupInformation ugi;
private HBaseMetastore hBaseMetastore;
@ -125,6 +121,19 @@ public class HBaseConnection
authenticate();
}
/**
* constructor for test
*
* @param hBaseMetastore HBaseMetaStore
* @param config HBaseConfig
*/
protected HBaseConnection(HBaseMetastore hBaseMetastore, HBaseConfig config)
{
this.hbaseConfig = config;
this.hBaseMetastore = hBaseMetastore;
authenticate();
}
/**
* getDefaultValue
*
@ -183,7 +192,6 @@ public class HBaseConnection
cfg.set("hbase.zookeeper.property.clientPort", hbaseConfig.getZkClientPort());
try {
catalogFile = hbaseConfig.getMetastoreUrl();
init();
hBaseMetastore.init();
}

10
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HBaseMetastoreFactory.java
View File

@ -27,8 +27,6 @@ import io.prestosql.spi.metastore.HetuMetastore;
*/
public class HBaseMetastoreFactory
{
private static final String TYPE_LOCAL = "local";
private static final String TYPE_HDFS = "hdfs";
private static final String TYPE_HETUMETASTORE = "hetuMetastore";
private final HBaseConfig hbaseConfig;
private final HetuMetastore hetuMetastore;
@ -47,13 +45,7 @@ public class HBaseMetastoreFactory
public HBaseMetastore create()
{
String type = hbaseConfig.getMetastoreType();
if (type == null || TYPE_LOCAL.equals(type)) {
return new LocalHBaseMetastore(hbaseConfig);
}
else if (TYPE_HDFS.equals(type)) {
return new HdfsHBaseMetastore(hbaseConfig);
}
else if (TYPE_HETUMETASTORE.equals(type)) {
if (type == null || TYPE_HETUMETASTORE.equals(type)) {
return new HetuHBaseMetastore(hetuMetastore);
}
else {

274
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/HdfsHBaseMetastore.java
View File

@ -1,274 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.metadata;
import com.google.common.collect.ImmutableMap;
import io.airlift.log.Logger;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.utils.Constants;
import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils;
import io.hetu.core.plugin.hbase.utils.Utils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import static java.util.Objects.requireNonNull;
/**
* HdfsHBaseMetastore: store file on hdfs
*
* @since 2020-03-30
*/
public class HdfsHBaseMetastore
implements HBaseMetastore
{
private static final Logger LOG = Logger.get(HdfsHBaseMetastore.class);
private static final String ID = "HDFS";
private final Map<String, HBaseTable> hbaseTables = new ConcurrentHashMap<>();
private String filePath;
private HBaseConfig hbaseConfig;
private FileSystem fs;
private Configuration hdfsConfig;
public HdfsHBaseMetastore(HBaseConfig config)
{
this.hbaseConfig = config;
this.filePath = hbaseConfig.getMetastoreUrl();
authenticate();
this.fs = getFs();
}
@Override
public void init()
{
JSONObject jsonObject = readJsonFromHdfs(filePath);
JsonHBaseTableUtils.loadHBaseTablesFromJson(hbaseTables, jsonObject);
}
/**
* get hdfs filesystem
*
* @return filesystem
*/
private FileSystem getFs()
{
FileSystem fileSystem = null;
try {
fileSystem = FileSystem.get(hdfsConfig);
}
catch (IOException e) {
LOG.error("Get hadoop file System error, cause by %s", e.getMessage());
}
return fileSystem;
}
private void authenticate()
{
hdfsConfig = new Configuration();
try {
if (!Utils.isFileExist(hbaseConfig.getHdfsSitePath())) {
throw new FileNotFoundException(hbaseConfig.getHdfsSitePath());
}
if (!Utils.isFileExist(hbaseConfig.getCoreSitePath())) {
throw new FileNotFoundException(hbaseConfig.getCoreSitePath());
}
hdfsConfig.addResource(new Path(hbaseConfig.getHdfsSitePath()));
hdfsConfig.addResource(new Path(hbaseConfig.getCoreSitePath()));
if (Constants.HDFS_AUTHENTICATION_KERBEROS.equals(hbaseConfig.getKerberos())) {
String keyTabPath = requireNonNull(hbaseConfig.getUserKeytabPath());
String krb5ConfPath = requireNonNull(hbaseConfig.getKrb5ConfPath());
if (!Utils.isFileExist(keyTabPath)) {
throw new FileNotFoundException(keyTabPath);
}
if (!Utils.isFileExist(krb5ConfPath)) {
throw new FileNotFoundException(krb5ConfPath);
}
String hdfsPrincipal = requireNonNull(hbaseConfig.getPrincipalUsername());
System.setProperty("java.security.krb5.conf", krb5ConfPath);
UserGroupInformation.setConfiguration(hdfsConfig);
UserGroupInformation.loginUserFromKeytab(hdfsPrincipal, keyTabPath);
}
}
catch (IOException exception) {
if (exception instanceof FileNotFoundException) {
LOG.error("File not exist, cause by %s", exception.getMessage());
}
else {
LOG.error("Kerberos authentication failed, caused by %s", exception.getMessage());
}
}
}
@Override
public String getId()
{
return this.ID;
}
@Override
public void addHBaseTable(HBaseTable hBaseTable)
{
// hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified
synchronized (hbaseTables) {
if (hbaseTables.containsKey(hBaseTable.getFullTableName())) {
hbaseTables.remove(hBaseTable.getFullTableName());
}
hbaseTables.put(hBaseTable.getFullTableName(), hBaseTable);
putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
@Override
public void renameHBaseTable(HBaseTable newTable, String oldTable)
{
// hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified
synchronized (hbaseTables) {
if (hbaseTables.containsKey(oldTable)) {
hbaseTables.remove(oldTable);
}
hbaseTables.put(newTable.getFullTableName(), newTable);
putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
@Override
public void dropHBaseTable(HBaseTable hBaseTable)
{
synchronized (hbaseTables) {
if (hbaseTables.containsKey(hBaseTable.getFullTableName())) {
hbaseTables.remove(hBaseTable.getFullTableName());
putJsonToHdfs(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
}
@Override
public synchronized Map<String, HBaseTable> getAllHBaseTables()
{
return ImmutableMap.copyOf(hbaseTables);
}
@Override
public HBaseTable getHBaseTable(String tableName)
{
return hbaseTables.get(tableName);
}
/**
* readJsonFromHdfs
*
* @param file file
* @return JSONObject
*/
public JSONObject readJsonFromHdfs(String file)
{
InputStream input = null;
JSONObject rs = null;
StringBuilder catalogs = new StringBuilder();
try {
Path path = new Path(file);
if (!fs.exists(path)) {
return rs;
}
input = fs.open(path);
while (input.available() > 0) {
catalogs.append((char) input.read());
}
}
catch (IOException e) {
LOG.error("readJsonFromHdfs : some wrong...file [%s].", file);
}
finally {
if (input != null) {
try {
input.close();
}
catch (IOException e) {
LOG.error("readJsonFromHdfs : something wrong...file [%s].", file);
}
}
}
try {
if (catalogs.toString().length() > 0) {
rs = new JSONObject(catalogs.toString());
}
}
catch (JSONException e) {
LOG.error("readJsonFromHdfs : to json wrong...cause by [%s].", e.getMessage());
}
return rs;
}
/**
* putJsonToFile
*
* @param file file
* @param jsonObject jsonObject
*/
public void putJsonToHdfs(String file, JSONObject jsonObject)
{
String jsonStr = "";
OutputStream out = null;
try {
jsonStr = jsonObject.toString(Constants.NUMBER4);
}
catch (JSONException e) {
LOG.error("putJsonToHdfs : json to string wrong...cause by [%s]", e.getMessage());
}
Path path = new Path(file);
try {
out = fs.create(path, true);
BufferedWriter bufferOut = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
bufferOut.write(jsonStr);
bufferOut.flush();
bufferOut.close();
}
catch (IOException e) {
LOG.error("write file[%s] failed... ", e.getMessage());
}
finally {
if (out != null) {
try {
out.close();
}
catch (IOException e) {
LOG.error("write file[%s] failed... ", e.getMessage());
}
}
}
}
}

217
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/metadata/LocalHBaseMetastore.java
View File

@ -1,217 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.metadata;
import com.google.common.collect.ImmutableMap;
import io.airlift.log.Logger;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.utils.Constants;
import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Reader;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* LocalHBaseMetastore: store file on local filesystem
*
* @since 2020-03-30
*/
public class LocalHBaseMetastore
implements HBaseMetastore
{
private static final Logger LOG = Logger.get(LocalHBaseMetastore.class);
private static final String ID = "LOCAL";
private final Map<String, HBaseTable> hbaseTables = new ConcurrentHashMap<>();
private String filePath;
public LocalHBaseMetastore(HBaseConfig config)
{
filePath = config.getMetastoreUrl();
}
@Override
public void init()
{
JSONObject jsonObject = readJsonFromFile(filePath);
JsonHBaseTableUtils.loadHBaseTablesFromJson(hbaseTables, jsonObject);
}
@Override
public String getId()
{
return this.ID;
}
@Override
public void addHBaseTable(HBaseTable hBaseTable)
{
// hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified
synchronized (hbaseTables) {
if (hbaseTables.containsKey(hBaseTable.getFullTableName())) {
hbaseTables.remove(hBaseTable.getFullTableName());
}
hbaseTables.put(hBaseTable.getFullTableName(), hBaseTable);
putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
@Override
public void renameHBaseTable(HBaseTable newTable, String oldTable)
{
// hbaseTableJson must be synchronize, when save it to file ,it couldn't be modified
synchronized (hbaseTables) {
if (hbaseTables.containsKey(oldTable)) {
hbaseTables.remove(oldTable);
}
hbaseTables.put(newTable.getFullTableName(), newTable);
putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
@Override
public void dropHBaseTable(HBaseTable hBaseTable)
{
synchronized (hbaseTables) {
if (hbaseTables.containsKey(hBaseTable.getFullTableName())) {
hbaseTables.remove(hBaseTable.getFullTableName());
putJsonToFile(filePath, JsonHBaseTableUtils.hbaseTablesMapToJson(hbaseTables));
}
}
}
@Override
public synchronized Map<String, HBaseTable> getAllHBaseTables()
{
return ImmutableMap.copyOf(hbaseTables);
}
@Override
public HBaseTable getHBaseTable(String tableName)
{
return hbaseTables.get(tableName);
}
/**
* readJsonFromFile
*
* @param file file
* @return JSONObject
*/
public JSONObject readJsonFromFile(String file)
{
Reader in = null;
JSONObject rs = null;
StringBuilder catalogs = new StringBuilder();
try {
File metaFile = new File(file);
if (!metaFile.exists()) {
return rs;
}
char[] buffer = new char[Constants.NUMBER1024];
in = new FileReader(file);
int len = in.read(buffer);
while (len != Constants.NUMBER_NEGATIVE_1) {
if (len == Constants.NUMBER1024) {
catalogs.append(buffer);
}
else {
catalogs.append(buffer, 0, len);
}
len = in.read(buffer);
}
}
catch (IOException e) {
LOG.error("readJsonFromFile : some wrong...file [%s]. Cause by %s", file, e.getMessage());
}
finally {
if (in != null) {
try {
in.close();
}
catch (IOException e) {
LOG.error("readJsonFromFile : something wrong...file [%s]. Cause by %s", file, e.getMessage());
}
}
}
try {
if (catalogs.toString().length() > 0) {
rs = new JSONObject(catalogs.toString());
}
}
catch (JSONException e) {
LOG.error("readJsonFromFile : to json wrong...file[%s]. Cause by %s", file, e.getMessage());
}
return rs;
}
/**
* putJsonToFile
*
* @param file file
* @param jsonObject jsonObject
*/
public void putJsonToFile(String file, JSONObject jsonObject)
{
PrintWriter out = null;
String jsonStr = "";
try {
jsonStr = jsonObject.toString(Constants.NUMBER4);
}
catch (JSONException e) {
LOG.error("putJsonToFile : json to string wrong...cause by", e.getMessage());
}
// check file exist and file permission
File metaFile = new File(file);
if (!metaFile.exists()) {
try {
if (!metaFile.createNewFile()) {
throw new IOException("when create file, it return false");
}
}
catch (IOException e) {
LOG.error("create file[%s] failed... check permission is available", e);
}
}
try {
// overwrite
out = new PrintWriter(new FileWriter(file, false));
// write file by json format
out.write(jsonStr);
out.flush();
}
catch (IOException e) {
LOG.error("putJsonToFile : some wrong...cause by", e.getMessage());
}
finally {
if (out != null) {
out.close();
}
}
}
}

125
hetu-hbase/src/main/java/io/hetu/core/plugin/hbase/utils/JsonHBaseTableUtils.java
View File

@ -1,125 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.utils;
import io.airlift.log.Logger;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.hetu.core.plugin.hbase.metadata.HBaseTable;
import io.prestosql.spi.connector.ColumnHandle;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
* JsonHBaseTableUtils
*
* @since 2020-03-30
*/
public class JsonHBaseTableUtils
{
private static final Logger LOG = Logger.get(JsonHBaseTableUtils.class);
private JsonHBaseTableUtils() {}
/**
* loadHBaseTablesFromJson
*
* @param hbaseTables hbaseTables map
* @param jsonObject jsonObject
*/
public static void loadHBaseTablesFromJson(Map<String, HBaseTable> hbaseTables, JSONObject jsonObject)
{
if (jsonObject == null) {
return;
}
Iterator<String> iterator = jsonObject.keys();
String key;
List<HBaseColumnHandle> columns;
JSONObject jsonHtable;
Map<String, ColumnHandle> columnHandleMap;
try {
while (iterator.hasNext()) {
columnHandleMap = new ConcurrentHashMap<>();
key = iterator.next();
jsonHtable = jsonObject.getJSONObject(key);
columns = new ArrayList<>();
JSONArray jaColumns = jsonHtable.getJSONArray("columns");
if (jaColumns.length() > 0) {
for (int i = 0; i < jaColumns.length(); i++) {
JSONObject jsonColumns = jaColumns.getJSONObject(i);
HBaseColumnHandle columnHandle =
new HBaseColumnHandle(
jsonColumns.getString("name"),
Optional.of(jsonColumns.getString("family")),
Optional.of(jsonColumns.getString("qualifer")),
Utils.createTypeByName(jsonColumns.getString("type")),
jsonColumns.getInt("ordinal"),
jsonColumns.getString("comment"),
jsonColumns.getBoolean("indexed"));
columns.add(columnHandle);
columnHandleMap.put(columnHandle.getName(), columnHandle);
}
}
HBaseTable hbaseTable =
new HBaseTable(
jsonHtable.getString("schema"),
jsonHtable.getString("table"),
columns,
jsonHtable.getString("rowId"),
jsonHtable.getBoolean("external"),
Optional.of(jsonHtable.getString("serializerClassName")),
Optional.of(jsonHtable.getString("indexColumns")),
Optional.of(jsonHtable.getString("hbaseTableName")));
hbaseTable.setColumnsToMap(columnHandleMap);
hbaseTables.put(key, hbaseTable);
}
}
catch (JSONException e) {
LOG.error("loadHbaseTableFromFile faild... cause by %s", e);
}
}
/**
* hbaseTablesMapToJson
*
* @param hbaseTablesMap hbaseTables Map
* @return JSONObject
*/
public static JSONObject hbaseTablesMapToJson(Map<String, HBaseTable> hbaseTablesMap)
{
JSONObject hbaseTablesJson = new JSONObject();
try {
for (Map.Entry<String, HBaseTable> entry : hbaseTablesMap.entrySet()) {
hbaseTablesJson.put(entry.getKey(), entry.getValue().parseHbaseTableToJson());
}
}
catch (JSONException e) {
LOG.error("hbaseTablesMapToJson faild... cause by %s", e);
}
return hbaseTablesJson;
}
}

30
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBase.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBase.java
View File

@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.client.TestingConnectorSession;
@ -23,7 +23,9 @@ import io.hetu.core.plugin.hbase.connector.HBaseConnectorId;
import io.hetu.core.plugin.hbase.connector.HBaseConnectorMetadataFactory;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import io.hetu.core.plugin.hbase.connector.HBaseTransactionHandle;
import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection;
import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata;
import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore;
import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider;
import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider;
import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider;
@ -80,6 +82,7 @@ public class TestHBase
private SchemaTableName schemaTableName;
private ConnectorTableHandle table;
private HBaseConnector hConnector;
private TestingHetuMetastore hetuMetastore;
/**
* setUp
@ -89,11 +92,10 @@ public class TestHBase
{
hCConf.setZkClientPort("2181");
hCConf.setZkQuorum("zk1");
hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini");
TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl());
hetuMetastore = new TestingHetuMetastore();
table = TestUtils.createHBaseTableHandle();
schemaTableName = new SchemaTableName("hbase", "test_table");
hconn = new TestHBaseClientConnection(hCConf);
hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore());
hconn.getConn();
session = new TestingConnectorSession("root");
hcm = new HBaseConnectorMetadata(hconn);
@ -107,6 +109,15 @@ public class TestHBase
null);
}
/**
* clear
*/
@AfterClass
public void clear()
{
hetuMetastore.close();
}
/**
* testGetHAdmin
*/
@ -304,7 +315,7 @@ public class TestHBase
/**
* testListTables
*/
public void testListTables()
private void testListTables()
{
List<SchemaTableName> tables = hcm.listTables(session, Optional.of("hbase"));
assertEquals(tables.size(), 1);
@ -474,13 +485,4 @@ public class TestHBase
cps.abort();
}
}
/**
* clear
*/
@AfterClass
public void clear()
{
TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl());
}
}

3
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBasePlugin.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestHBasePlugin.java
View File

@ -12,9 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase;
import io.hetu.core.plugin.hbase.HBasePlugin;
import org.testng.annotations.Test;
import java.lang.reflect.Method;

29
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestQuery.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/TestQuery.java
View File

@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase;
import io.airlift.slice.Slice;
import io.airlift.slice.Slices;
@ -23,6 +23,8 @@ import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.hetu.core.plugin.hbase.connector.HBaseConnection;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection;
import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore;
import io.hetu.core.plugin.hbase.query.HBaseGetRecordCursor;
import io.hetu.core.plugin.hbase.query.HBaseRecordCursor;
import io.hetu.core.plugin.hbase.query.HBaseRecordSet;
@ -67,6 +69,7 @@ public class TestQuery
private HBaseConfig hCConf = new HBaseConfig();
private ConnectorSession session;
private HBaseRecordSet recordSet;
private TestingHetuMetastore hetuMetastore;
private HBaseSplit split;
/**
@ -77,9 +80,8 @@ public class TestQuery
{
hCConf.setZkClientPort("2181");
hCConf.setZkQuorum("zk1");
hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini");
TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl());
hconn = new TestHBaseClientConnection(hCConf);
hetuMetastore = new TestingHetuMetastore();
hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore());
hconn.getConn();
session = new TestingConnectorSession("root");
split =
@ -97,6 +99,15 @@ public class TestQuery
hconn, session, split, TestUtils.createHBaseTableHandle(), TestUtils.createColumnList());
}
/**
* clear
*/
@AfterClass
public void clear()
{
hetuMetastore.close();
}
/**
* testHBaseRecordSetCursorIsBatchGet
*/
@ -240,7 +251,6 @@ public class TestQuery
*/
@Test
public void testHBaseRecordCursor()
throws Exception
{
List<HBaseColumnHandle> columnHandles = new ArrayList<>();
columnHandles.add(TestUtils.createHBaseColumnRowId("rowkey"));
@ -331,13 +341,4 @@ public class TestQuery
// Check read data
assertEquals("nick_name_1 12 2019-06-11 20", serializer.getColumnValues().get("rowkey"));
}
/**
* clear
*/
@AfterClass
public void clear()
{
TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl());
}
}

11
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/client/TestUtils.java
View File

@ -14,11 +14,9 @@
*/
package io.hetu.core.plugin.hbase.client;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import io.hetu.core.plugin.hbase.metadata.HBaseTable;
import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.connector.ColumnMetadata;
import io.prestosql.spi.connector.ConnectorTableMetadata;
@ -210,15 +208,6 @@ public class TestUtils
return properties;
}
/**
* createHBaseTable
*/
public static Map<String, HBaseTable> createHBaseTable(String file)
{
LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(new HBaseConfig());
return loadHBaseTableFromFile(lHBMetastore.readJsonFromFile(file));
}
/**
* loadHBaseTableFromFile
*

15
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConf.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseClientConf.java
View File

@ -12,13 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.conf;
import io.hetu.core.plugin.hbase.conf.HBaseColumnProperties;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import org.testng.annotations.Test;
import static org.junit.Assert.assertEquals;
import static org.testng.Assert.assertEquals;
/**
* TestHBaseClientConf
@ -47,9 +45,6 @@ public class TestHBaseClientConf
hcc.setMetastoreType("type");
assertEquals("type", hcc.getMetastoreType());
hcc.setMetastoreUrl("file");
assertEquals("file", hcc.getMetastoreUrl());
hcc.setKerberos("file");
assertEquals("file", hcc.getKerberos());
@ -65,12 +60,6 @@ public class TestHBaseClientConf
hcc.setJaasConfPath("/etc/hetu/");
assertEquals("/etc/hetu/", hcc.getJaasConfPath());
hcc.setCoreSitePath("/etc/hetu/");
assertEquals("/etc/hetu/", hcc.getCoreSitePath());
hcc.setHdfsSitePath("/etc/hetu/");
assertEquals("/etc/hetu/", hcc.getHdfsSitePath());
hcc.setHbaseSitePath("/etc/hetu/");
assertEquals("/etc/hetu/", hcc.getHbaseSitePath());

3
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableProperties.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/conf/TestHBaseTableProperties.java
View File

@ -12,9 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.conf;
import io.hetu.core.plugin.hbase.conf.HBaseTableProperties;
import org.apache.commons.lang3.tuple.Pair;
import org.testng.annotations.Test;

8
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseClientConnection.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseClientConnection.java
View File

@ -12,11 +12,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.connector;
import io.hetu.core.plugin.hbase.client.TestHBaseConnection;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.connector.HBaseConnection;
import io.hetu.core.plugin.hbase.metadata.HBaseMetastore;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import java.io.IOException;
@ -29,9 +29,9 @@ import java.io.IOException;
public class TestHBaseClientConnection
extends HBaseConnection
{
public TestHBaseClientConnection(HBaseConfig conf)
public TestHBaseClientConnection(HBaseConfig conf, HBaseMetastore metastore)
{
super(conf);
super(metastore, conf);
}
/**

7
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseColumnHandle.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseColumnHandle.java
View File

@ -12,16 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.connector;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.prestosql.spi.type.Type;
import io.prestosql.spi.type.VarcharType;
import org.junit.Test;
import org.testng.annotations.Test;
import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.testng.Assert.assertEquals;
/**
* TestHBaseColumnHandle

58
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseConnector.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseConnector.java
View File

@ -12,20 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.connector;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.client.TestingConnectorSession;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.conf.HBaseTableProperties;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.hetu.core.plugin.hbase.connector.HBaseConnection;
import io.hetu.core.plugin.hbase.connector.HBaseConnector;
import io.hetu.core.plugin.hbase.connector.HBaseConnectorId;
import io.hetu.core.plugin.hbase.connector.HBaseConnectorMetadataFactory;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import io.hetu.core.plugin.hbase.metadata.HBaseConnectorMetadata;
import io.hetu.core.plugin.hbase.metadata.HBaseTable;
import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore;
import io.hetu.core.plugin.hbase.query.HBasePageSinkProvider;
import io.hetu.core.plugin.hbase.query.HBasePageSourceProvider;
import io.hetu.core.plugin.hbase.query.HBaseRecordSetProvider;
@ -84,6 +79,7 @@ public class TestHBaseConnector
private SchemaTableName schemaTableName;
private HBaseConnector hConnector;
private ConnectorSession session;
private TestingHetuMetastore hetuMetastore;
/**
* setUp
@ -93,10 +89,9 @@ public class TestHBaseConnector
{
hCConf.setZkClientPort("2181");
hCConf.setZkQuorum("zk1");
hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini");
TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl());
hetuMetastore = new TestingHetuMetastore();
schemaTableName = new SchemaTableName("hbase", "test_table");
hconn = new TestHBaseClientConnection(hCConf);
hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore());
hconn.getConn();
session = new TestingConnectorSession("root");
hcm = new HBaseConnectorMetadata(hconn);
@ -110,6 +105,15 @@ public class TestHBaseConnector
new HBaseTableProperties());
}
/**
* clear
*/
@AfterClass
public void clear()
{
hetuMetastore.close();
}
/**
* testDropSchema
*/
@ -193,29 +197,6 @@ public class TestHBaseConnector
assertEquals("test_table", map.get("hbase").get(0).getTableName());
}
/**
* testCreateNewTable
*/
@Test
public void testCreateNewTable()
{
hcm.createTable(session, TestUtils.createConnectorTableMeta(), false);
}
/**
* testCreateExistTable
*/
@Test
public void testCreateExistTable()
{
Map<String, Object> properties = TestUtils.createProperties();
properties.put("hbase_table_name", "hbase:test_table");
ConnectorTableMetadata ctm =
new ConnectorTableMetadata(
new SchemaTableName("hbase", "test_table"), TestUtils.createColumnMetadataList(), properties);
hcm.createTable(session, ctm, false);
}
/**
* testApplyFilter
*/
@ -358,7 +339,7 @@ public class TestHBaseConnector
hCnnConf.setZkClientPort("2181");
hCnnConf.setZkQuorum("zk1");
hCnnConf.setRetryNumber(1);
HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf);
HBaseConnection hConn = new TestHBaseClientConnection(hCnnConf, null);
try {
hConn.createSchema("schema", null);
@ -603,13 +584,4 @@ public class TestHBaseConnector
assertEquals(hConnector.getTableProperties().size(), 7);
assertEquals(hConnector.getColumnProperties().size(), 2);
}
/**
* clear
*/
@AfterClass
public void clear()
{
TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl());
}
}

5
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTableHandle.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTableHandle.java
View File

@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.connector;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import org.testng.annotations.Test;
import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.testng.Assert.assertEquals;
/**
* TestHBaseTableHandle

5
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTransactionHandle.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/connector/TestHBaseTransactionHandle.java
View File

@ -12,14 +12,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.connector;
import io.hetu.core.plugin.hbase.connector.HBaseTransactionHandle;
import org.testng.annotations.Test;
import java.util.UUID;
import static org.junit.Assert.assertEquals;
import static org.testng.Assert.assertEquals;
/**
* TestHBaseTransactionHandle

34
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseMetastoreFactory.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseMetastoreFactory.java
View File

@ -12,20 +12,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.metadata;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.metadata.HBaseMetastoreFactory;
import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore;
import io.hetu.core.plugin.hbase.utils.HBaseErrorCode;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ConnectorTableMetadata;
import io.prestosql.spi.connector.SchemaTableName;
import org.testng.annotations.Test;
import java.util.UUID;
import static org.testng.Assert.assertEquals;
/**
@ -35,34 +31,6 @@ import static org.testng.Assert.assertEquals;
*/
public class TestHBaseMetastoreFactory
{
/**
* testLocalHBaseMetastoreGetId
*/
@Test
public void testLocalHBaseMetastoreGetId()
{
LocalHBaseMetastore lhBMetastore = new LocalHBaseMetastore(new HBaseConfig());
assertEquals("LOCAL", lhBMetastore.getId());
}
/**
* testHBaseMetastoreFactoryHdfs
*/
@Test
public void testHBaseMetastoreFactoryHdfs()
{
HBaseConfig hBConf = new HBaseConfig();
hBConf.setMetastoreType("hdfs");
hBConf.setHdfsSitePath("./hdfs-site.xml" + UUID.randomUUID());
hBConf.setCoreSitePath("./core-site.xml" + UUID.randomUUID());
hBConf.setKerberos("KERBEROS");
hBConf.setUserKeytabPath("./user.keytab" + UUID.randomUUID());
hBConf.setKrb5ConfPath("./krb5.conf" + UUID.randomUUID());
hBConf.setPrincipalUsername("root");
HBaseMetastoreFactory hBMetaFactory = new HBaseMetastoreFactory(hBConf);
hBMetaFactory.create();
}
/**
* testHBaseMetastoreFactoryHetuMetastore
*/

5
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseTable.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHBaseTable.java
View File

@ -12,17 +12,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.metadata;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.metadata.HBaseTable;
import io.prestosql.spi.connector.SchemaTableName;
import org.codehaus.jettison.json.JSONException;
import org.testng.annotations.Test;
import java.util.Optional;
import static org.junit.Assert.assertEquals;
import static org.testng.Assert.assertEquals;
/**
* TestHBaseTable

113
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestHetuHBaseMetastore.java Normal file
View File

@ -0,0 +1,113 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.metadata;
import io.hetu.core.plugin.hbase.client.TestUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.util.Map;
import java.util.Optional;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
public class TestHetuHBaseMetastore
{
private TestingHetuMetastore hetuMetastore;
private HetuHBaseMetastore metaStore;
@BeforeClass
public void setUp()
{
hetuMetastore = new TestingHetuMetastore();
metaStore = hetuMetastore.getHetuMetastore();
}
@AfterClass(alwaysRun = true)
public void destroy()
{
hetuMetastore.close();
}
@Test
public void testGetId()
{
assertEquals(metaStore.getId(), "Hetu");
}
@Test
public void testOPHBaseTable()
{
Optional<String> serializerClassName =
Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer");
HBaseTable testHBaseTable1 =
new HBaseTable(
"hbase",
"testTable1",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("testTable"));
metaStore.addHBaseTable(testHBaseTable1);
HBaseTable testHBaseTable2 =
new HBaseTable(
"hbase",
"testTable2",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("testTable2"));
metaStore.addHBaseTable(testHBaseTable2);
// test get all tables
Map<String, HBaseTable> hBaseTables = metaStore.getAllHBaseTables();
assertTrue(hBaseTables.containsKey("hbase.testTable1"));
assertTrue(hBaseTables.containsKey("hbase.testTable2"));
// test get one of tables
HBaseTable table = metaStore.getHBaseTable("hbase.testTable1");
assertNotNull(table);
assertEquals("hbase.testTable1", table.getFullTableName());
// test rename table
HBaseTable testHBaseTable3 =
new HBaseTable(
"hbase",
"testTable3",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("testTable3"));
metaStore.renameHBaseTable(testHBaseTable3, "hbase.testTable1");
hBaseTables = metaStore.getAllHBaseTables();
assertFalse(hBaseTables.containsKey("hbase.testTable1"));
assertTrue(hBaseTables.containsKey("hbase.testTable3"));
// test drop table
metaStore.dropHBaseTable(testHBaseTable3);
hBaseTables = metaStore.getAllHBaseTables();
assertFalse(hBaseTables.containsKey("hbase.testTable3"));
}
}

54
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHBaseTableUtils.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.metadata;
import java.io.File;
import java.io.IOException;
/**
* TestJsonHBaseTableUtils
*
* @since 2020-03-20
*/
public class TestingHBaseTableUtils
{
private TestingHBaseTableUtils() {}
/**
* delFile
*/
public static void delFile(String file)
{
File pfile = new File(file);
pfile.delete();
}
/**
* createFile
*/
public static void createFile(String filename)
throws Exception
{
File file = new File(filename);
if (!file.exists()) {
try {
file.createNewFile();
}
catch (IOException e) {
throw new IOException("createFile " + filename + " : failed");
}
}
}
}

115
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/metadata/TestingHetuMetastore.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.metadata;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Injector;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.testing.mysql.TestingMySqlServer;
import io.hetu.core.metastore.jdbc.JdbcMetastoreModule;
import io.hetu.core.plugin.hbase.connector.HBaseColumnHandle;
import io.prestosql.plugin.base.jmx.MBeanServerModule;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.connector.ColumnHandle;
import io.prestosql.spi.metastore.HetuMetastore;
import io.prestosql.spi.type.BigintType;
import io.prestosql.spi.type.DateType;
import io.prestosql.spi.type.VarcharType;
import org.weakref.jmx.guice.MBeanModule;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.google.common.base.Throwables.throwIfUnchecked;
import static io.prestosql.spi.metastore.HetuErrorCode.HETU_METASTORE_CODE;
public class TestingHetuMetastore
implements AutoCloseable
{
private static final String TEST_DATABASES = "hbase";
private static final String TEST_MYSQL_USER = "user";
private static final String TEST_MYSQL_PASSWORD = "testpass";
private static final HBaseTable TEST_TABLE = new HBaseTable(
"hbase", "test_table",
ImmutableList.of(
new HBaseColumnHandle("rowkey", Optional.of(""), Optional.of(""), VarcharType.VARCHAR,
0, "HBase row ID", false),
new HBaseColumnHandle("name", Optional.of("name"), Optional.of("nick_name"), VarcharType.VARCHAR,
1, "HBase column name:nick_name. Indexed: false", false),
new HBaseColumnHandle("age", Optional.of("age"), Optional.of("lit_age"), BigintType.BIGINT,
2, "HBase column age:lit_age. Indexed: false", false),
new HBaseColumnHandle("gender", Optional.of("gender"), Optional.of("gender"), DateType.DATE,
3, "HBase column gender:gender. Indexed:false", false),
new HBaseColumnHandle("t", Optional.of("t"), Optional.of("t"), BigintType.BIGINT,
4, "HBase column t:t. Indexed: false", false)),
"rowkey",
false,
Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer"),
Optional.empty(),
Optional.of("hbase:test_table"));
private final TestingMySqlServer mySqlServer;
private final HetuHBaseMetastore metaStore;
public TestingHetuMetastore()
{
try {
this.mySqlServer = new TestingMySqlServer(TEST_MYSQL_USER, TEST_MYSQL_PASSWORD, TEST_DATABASES);
Map<String, String> properties = new ImmutableMap.Builder<String, String>()
.put("hetu.metastore.db.url", mySqlServer.getJdbcUrl(TEST_DATABASES))
.put("hetu.metastore.db.user", TEST_MYSQL_USER)
.put("hetu.metastore.db.password", TEST_MYSQL_PASSWORD)
.build();
Bootstrap app = new Bootstrap(
new MBeanModule(),
new MBeanServerModule(),
new JdbcMetastoreModule());
Injector injector = app
.strictConfig()
.doNotInitializeLogging()
.setRequiredConfigurationProperties(properties)
.initialize();
this.metaStore = new HetuHBaseMetastore(injector.getInstance(HetuMetastore.class));
metaStore.init();
Map<String, ColumnHandle> map = TEST_TABLE.getColumns().stream().collect(Collectors.toMap(
HBaseColumnHandle::getColumnName,
Function.identity()));
TEST_TABLE.setColumnsToMap(map);
metaStore.addHBaseTable(TEST_TABLE);
}
catch (Exception ex) {
throwIfUnchecked(ex);
throw new PrestoException(HETU_METASTORE_CODE,
"init hetu hbase metastore module failed.");
}
}
public HetuHBaseMetastore getHetuMetastore()
{
return metaStore;
}
@Override
public void close()
{
mySqlServer.close();
}
}

28
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHBaseKerberosAuthentication.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/security/TestHBaseKerberosAuthentication.java
View File

@ -12,9 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
package io.hetu.core.plugin.hbase.security;
import io.hetu.core.plugin.hbase.security.HBaseKerberosAuthentication;
import io.hetu.core.plugin.hbase.metadata.TestingHBaseTableUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.testng.annotations.Test;
@ -87,7 +87,7 @@ public class TestHBaseKerberosAuthentication
}
// krb5ConfPath is not a file
TestJsonHBaseTableUtils.createFile("user.keytab.test");
TestingHBaseTableUtils.createFile("user.keytab.test");
try {
HBaseKerberosAuthentication.authenticateAndReturnUGI(
"root", "./user.keytab.test", "./", new HBaseConfiguration());
@ -97,7 +97,7 @@ public class TestHBaseKerberosAuthentication
assertEquals(true, e.toString().contains("java.io.IOException: krb5ConfFile"), e.toString());
assertEquals(true, e.toString().contains("is not a file"), e.toString());
}
TestJsonHBaseTableUtils.delFile("user.keytab");
TestingHBaseTableUtils.delFile("user.keytab");
}
/**
@ -108,8 +108,8 @@ public class TestHBaseKerberosAuthentication
throws Exception
{
// krb5ConfPath is not a file
TestJsonHBaseTableUtils.createFile("user.keytab.test");
TestJsonHBaseTableUtils.createFile("krb5.conf.test");
TestingHBaseTableUtils.createFile("user.keytab.test");
TestingHBaseTableUtils.createFile("krb5.conf.test");
try {
HBaseKerberosAuthentication.authenticateAndReturnUGI(
"root", "./user.keytab.test", "./krb5.conf.test", new HBaseConfiguration());
@ -117,8 +117,8 @@ public class TestHBaseKerberosAuthentication
catch (IllegalStateException | NullPointerException e) {
throw new IllegalStateException("testAuthenticateAndReturnUGINormal: failed");
}
TestJsonHBaseTableUtils.delFile("user.keytab.test");
TestJsonHBaseTableUtils.delFile("krb5.conf.test");
TestingHBaseTableUtils.delFile("user.keytab.test");
TestingHBaseTableUtils.delFile("krb5.conf.test");
}
/**
@ -141,8 +141,8 @@ public class TestHBaseKerberosAuthentication
// userKeytabFile not exist
try {
TestJsonHBaseTableUtils.delFile("krb5.conf.test");
TestJsonHBaseTableUtils.delFile("jaas.conf.test");
TestingHBaseTableUtils.delFile("krb5.conf.test");
TestingHBaseTableUtils.delFile("jaas.conf.test");
HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test");
throw new IOException("testAuthenticate : failed");
}
@ -152,8 +152,8 @@ public class TestHBaseKerberosAuthentication
}
// Complete process
TestJsonHBaseTableUtils.createFile("krb5.conf.test");
TestJsonHBaseTableUtils.createFile("jaas.conf.test");
TestingHBaseTableUtils.createFile("krb5.conf.test");
TestingHBaseTableUtils.createFile("jaas.conf.test");
try {
HBaseKerberosAuthentication.setJaasConf("jaas.conf.test", "root", "krb5.conf.test");
throw new IOException("testAuthenticate : failed");
@ -163,7 +163,7 @@ public class TestHBaseKerberosAuthentication
e.toString(),
"java.io.IOException: AppConfigurationEntry named jaas.conf.test does not have value of keyTab.");
}
TestJsonHBaseTableUtils.delFile("krb5.conf.test");
TestJsonHBaseTableUtils.delFile("jaas.conf.test");
TestingHBaseTableUtils.delFile("krb5.conf.test");
TestingHBaseTableUtils.delFile("jaas.conf.test");
}
}

3
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHBaseSplit.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHBaseSplit.java
View File

@ -12,10 +12,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test.split;
package io.hetu.core.plugin.hbase.split;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.split.HBaseSplit;
import io.prestosql.spi.HostAddress;
import org.testng.annotations.Test;

18
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/split/TestHbaseSplitManager.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/split/TestHbaseSplitManager.java
View File

@ -12,14 +12,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test.split;
package io.hetu.core.plugin.hbase.split;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.connector.HBaseConnection;
import io.hetu.core.plugin.hbase.connector.HBaseTableHandle;
import io.hetu.core.plugin.hbase.split.HBaseSplitManager;
import io.hetu.core.plugin.hbase.test.TestHBaseClientConnection;
import io.hetu.core.plugin.hbase.connector.TestHBaseClientConnection;
import io.hetu.core.plugin.hbase.metadata.TestingHetuMetastore;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@ -37,6 +38,7 @@ public class TestHbaseSplitManager
private HBaseConnection hconn;
private HBaseConfig hCConf = new HBaseConfig();
private HBaseSplitManager hsm;
private TestingHetuMetastore hetuMetastore;
/**
* setUp
@ -46,12 +48,18 @@ public class TestHbaseSplitManager
{
hCConf.setZkClientPort("2181");
hCConf.setZkQuorum("zk1");
hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini");
hconn = new TestHBaseClientConnection(hCConf);
hetuMetastore = new TestingHetuMetastore();
hconn = new TestHBaseClientConnection(hCConf, hetuMetastore.getHetuMetastore());
hconn.getConn();
hsm = new HBaseSplitManager(hconn);
}
@AfterClass
public void destroy()
{
hetuMetastore.close();
}
/**
* testSplitManager
*

229
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestHdfsHBaseMetastore.java
View File

@ -1,229 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
import io.hetu.core.plugin.hbase.client.TestUtils;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.metadata.HBaseTable;
import io.hetu.core.plugin.hbase.metadata.HdfsHBaseMetastore;
import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore;
import org.codehaus.jettison.json.JSONObject;
import org.mockito.Mockito;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import java.io.FileNotFoundException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Map;
import java.util.Optional;
import static org.testng.Assert.assertEquals;
/**
* TestHdfsHBaseMetastore
*
* @since 2020-03-20
*/
public class TestHdfsHBaseMetastore
{
private HdfsHBaseMetastore hdfsHBaseMetastore;
private HBaseConfig hCConf = new HBaseConfig();
/**
* setUp
*/
@BeforeClass
public void setUp()
{
hCConf.setZkClientPort("2181");
hCConf.setZkQuorum("zk1");
hCConf.setMetastoreType("hdfs");
hCConf.setHdfsSitePath("./hdfs-site.xml.test");
hCConf.setCoreSitePath("./core-site.xml.test");
hCConf.setKerberos("KERBEROS");
hCConf.setUserKeytabPath("./user.keytab.test");
hCConf.setKrb5ConfPath("./krb5.conf.test");
hCConf.setPrincipalUsername("root");
hCConf.setMetastoreUrl("./hbasetablecatalogtmp.ini");
TestJsonHBaseTableUtils.preFile(hCConf.getMetastoreUrl());
hdfsHBaseMetastore = new HdfsHBaseMetastore(hCConf);
}
/**
* testInit
*/
@Test
public void testInit()
{
hdfsHBaseMetastore.init();
}
/**
* testAuthenticate
*
* @throws FileNotFoundException InvocationTargetException Exception
*/
@Test
public void testAuthenticate()
throws Exception
{
Method method = HdfsHBaseMetastore.class.getDeclaredMethod("authenticate");
method.setAccessible(true);
try {
TestJsonHBaseTableUtils.createFile("./hdfs-site.xml.test");
method.invoke(hdfsHBaseMetastore);
}
catch (FileNotFoundException e) {
assertEquals(e.getMessage(), null);
}
try {
TestJsonHBaseTableUtils.createFile("./core-site.xml.test");
method.invoke(hdfsHBaseMetastore);
}
catch (FileNotFoundException e) {
assertEquals(e.getMessage(), null);
}
try {
TestJsonHBaseTableUtils.createFile("./user.keytab.test");
method.invoke(hdfsHBaseMetastore);
}
catch (FileNotFoundException e) {
assertEquals(e.getMessage(), null);
}
try {
TestJsonHBaseTableUtils.createFile("./krb5.conf.test");
method.invoke(hdfsHBaseMetastore);
}
catch (InvocationTargetException e) {
assertEquals(e.getMessage(), null);
}
TestJsonHBaseTableUtils.delFile("./hdfs-site.xml.test");
TestJsonHBaseTableUtils.delFile("./core-site.xml.test");
TestJsonHBaseTableUtils.delFile("./user.keytab.test");
TestJsonHBaseTableUtils.delFile("./krb5.conf.test");
}
/**
* testGetId
*/
@Test
public void testGetId()
{
assertEquals("HDFS", hdfsHBaseMetastore.getId());
}
/**
* testOPHBaseTable
*/
@Test
public void testOPHBaseTable()
{
HdfsHBaseMetastore hDFSHBaseMetastore = Mockito.spy(new HdfsHBaseMetastore(hCConf));
Optional<String> serializerClassName =
Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer");
HBaseTable hBaseTable =
new HBaseTable(
"hbase",
"oldtable",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("oldtable"));
Mockito.doNothing().when(hDFSHBaseMetastore).putJsonToHdfs(Mockito.anyString(), Mockito.any(JSONObject.class));
hDFSHBaseMetastore.addHBaseTable(hBaseTable);
HBaseTable hBaseNewTable =
new HBaseTable(
"hbase",
"newtable",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("newtable"));
hDFSHBaseMetastore.addHBaseTable(hBaseNewTable);
// add an existing hBaseTable
hDFSHBaseMetastore.addHBaseTable(hBaseNewTable);
Map<String, HBaseTable> hbaseTables = hDFSHBaseMetastore.getAllHBaseTables();
assertEquals(true, hbaseTables.containsKey("hbase.oldtable"));
assertEquals(true, hbaseTables.containsKey("hbase.newtable"));
HBaseTable hBaseOldTable = hDFSHBaseMetastore.getHBaseTable("hbase.oldtable");
assertEquals("hbase.oldtable", hBaseOldTable.getFullTableName());
hDFSHBaseMetastore.renameHBaseTable(hBaseNewTable, "hbase.oldtable");
hbaseTables = hDFSHBaseMetastore.getAllHBaseTables();
assertEquals(false, hbaseTables.containsKey("hbase.oldtable"));
assertEquals(true, hbaseTables.containsKey("hbase.newtable"));
hDFSHBaseMetastore.dropHBaseTable(hBaseNewTable);
hbaseTables = hDFSHBaseMetastore.getAllHBaseTables();
assertEquals(false, hbaseTables.containsKey("hbase.oldtable"));
assertEquals(false, hbaseTables.containsKey("hbase.newtable"));
}
/**
* testLocalHBaseMetastore
*/
@Test
public void testLocalHBaseMetastore()
{
HBaseConfig conf = new HBaseConfig();
conf.setMetastoreUrl("LOCAL");
conf.setMetastoreUrl("./localHBaseMetastore.ini");
LocalHBaseMetastore localHBaseMetastore = new LocalHBaseMetastore(conf);
Optional<String> serializerClassName =
Optional.of("io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer");
HBaseTable hBaseTable =
new HBaseTable(
"hbase",
"oldtable",
TestUtils.createColumnList(),
"rowkey",
false,
serializerClassName,
Optional.of(""),
Optional.of("oldtable"));
localHBaseMetastore.addHBaseTable(hBaseTable);
localHBaseMetastore.dropHBaseTable(hBaseTable);
TestJsonHBaseTableUtils.delFile(conf.getMetastoreUrl());
}
/**
* clear
*/
@AfterClass
public void clear()
{
TestJsonHBaseTableUtils.delFile(hCConf.getMetastoreUrl());
TestJsonHBaseTableUtils.delFile("./.hbasetablecatalogtmp.ini.crc");
TestJsonHBaseTableUtils.delFile("./hdfs-site.xml.test");
TestJsonHBaseTableUtils.delFile("./core-site.xml.test");
TestJsonHBaseTableUtils.delFile("./user.keytab.test");
TestJsonHBaseTableUtils.delFile("./krb5.conf.test");
}
}

184
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/TestJsonHBaseTableUtils.java
View File

@ -1,184 +0,0 @@
/*
* Copyright (C) 2018-2020. Huawei Technologies Co., Ltd. All rights reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test;
import io.hetu.core.plugin.hbase.conf.HBaseConfig;
import io.hetu.core.plugin.hbase.metadata.LocalHBaseMetastore;
import io.hetu.core.plugin.hbase.utils.JsonHBaseTableUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.testng.annotations.Test;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import static org.junit.Assert.assertEquals;
/**
* TestJsonHBaseTableUtils
*
* @since 2020-03-20
*/
public class TestJsonHBaseTableUtils
{
private static LocalHBaseMetastore lHBMetastore = new LocalHBaseMetastore(new HBaseConfig());
private static String jsonStr =
"{\n"
+ " \"hbase.test_table\": {\n"
+ " \"schema\": \"hbase\",\n"
+ " \"external\": false,\n"
+ " \"rowIdOrdinal\": 0,\n"
+ " \"serializerClassName\":"
+ " \"io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer\",\n"
+ " \"indexed\": false,\n"
+ " \"rowId\": \"rowkey\",\n"
+ " \"table\": \"test_table\",\n"
+ " \"indexColumns\": \"\",\n"
+ " \"hbaseTableName\": \"hbase:test_table\",\n"
+ " \"columns\": [\n"
+ " {\n"
+ " \"name\": \"rowkey\",\n"
+ " \"family\": \"\",\n"
+ " \"qualifer\": \"\",\n"
+ " \"type\": \"io.prestosql.spi.type.VarcharType\",\n"
+ " \"ordinal\": 0,\n"
+ " \"comment\": \"HBase row ID\",\n"
+ " \"indexed\": false\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"name\",\n"
+ " \"family\": \"name\",\n"
+ " \"qualifer\": \"nick_name\",\n"
+ " \"type\": \"io.prestosql.spi.type.VarcharType\",\n"
+ " \"ordinal\": 1,\n"
+ " \"comment\": \"HBase column name:nick_name. Indexed: false\",\n"
+ " \"indexed\": false\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"age\",\n"
+ " \"family\": \"age\",\n"
+ " \"qualifer\": \"lit_age\",\n"
+ " \"type\": \"io.prestosql.spi.type.BigintType\",\n"
+ " \"ordinal\": 2,\n"
+ " \"comment\": \"HBase column age:lit_age. Indexed: false\",\n"
+ " \"indexed\": false\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"gender\",\n"
+ " \"family\": \"gender\",\n"
+ " \"qualifer\": \"gender\",\n"
+ " \"type\": \"io.prestosql.spi.type.DateType\",\n"
+ " \"ordinal\": 3,\n"
+ " \"comment\": \"HBase column gender:gender. Indexed: false\",\n"
+ " \"indexed\": false\n"
+ " },\n"
+ " {\n"
+ " \"name\": \"t\",\n"
+ " \"family\": \"t\",\n"
+ " \"qualifer\": \"t\",\n"
+ " \"type\": \"io.prestosql.spi.type.BigintType\",\n"
+ " \"ordinal\": 4,\n"
+ " \"comment\": \"HBase column t:t. Indexed: false\",\n"
+ " \"indexed\": false\n"
+ " }\n"
+ " ]\n"
+ " }}";
/**
* testFileUtils
*
* @throws NullPointerException Exception
*/
@Test
public void testFileUtils()
throws Exception
{
String file = "./tmpcatalog.ini";
try {
JSONObject json = new JSONObject(jsonStr);
JsonHBaseTableUtils.loadHBaseTablesFromJson(new HashMap<>(), json);
JSONObject json2 = lHBMetastore.readJsonFromFile(file);
assertEquals(json.toString(), json2.toString());
File pfile = new File(file);
pfile.delete();
}
catch (NullPointerException e) {
assertEquals(e.toString(), "java.lang.NullPointerException");
}
}
/**
* testLoadHBaseTablesFromJsonNull
*/
@Test
public void testLoadHBaseTablesFromJsonNull()
{
JsonHBaseTableUtils.loadHBaseTablesFromJson(new HashMap<>(), null);
}
/**
* preFile
*
* @throws JSONException Exception
*/
public static void preFile(String file)
{
try {
JSONObject json = new JSONObject(jsonStr);
lHBMetastore.putJsonToFile(file, json);
}
catch (JSONException e) {
assertEquals(e.toString().substring(0, 36), "putJsonToFile : json to string wrong");
}
}
/**
* delFile
*/
public static void delFile(String file)
{
File pfile = new File(file);
pfile.delete();
}
/**
* createFile
*/
public static void createFile(String filename)
throws Exception
{
File file = new File(filename);
if (!file.exists()) {
try {
file.createNewFile();
}
catch (IOException e) {
throw new IOException("createFile " + filename + " : failed");
}
}
}
/**
* testReadException
*/
@Test
public void testReadException()
{
assertEquals(null, lHBMetastore.readJsonFromFile("xxx"));
}
}

3
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/AbstractTestHBaseRowSerializer.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/AbstractTestHBaseRowSerializer.java
View File

@ -12,11 +12,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test.utils.serializers;
package io.hetu.core.plugin.hbase.utils.serializers;
import io.airlift.slice.Slice;
import io.hetu.core.plugin.hbase.utils.TestSliceUtils;
import io.hetu.core.plugin.hbase.utils.serializers.HBaseRowSerializer;
import io.prestosql.spi.PrestoException;
import io.prestosql.spi.block.Block;
import io.prestosql.spi.block.ShortArrayBlock;

4
hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/test/utils/serializers/TestStringRowSerializer.java → hetu-hbase/src/test/java/io/hetu/core/plugin/hbase/utils/serializers/TestStringRowSerializer.java
View File

@ -12,9 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.hetu.core.plugin.hbase.test.utils.serializers;
import io.hetu.core.plugin.hbase.utils.serializers.StringRowSerializer;
package io.hetu.core.plugin.hbase.utils.serializers;
/**
* TestStringRowSerializer