expand secure path whitelist
add root directory to black list and add docs for white list remove multi path
This commit is contained in:
farhan3
tushengxia
parent
dc78f25f2e
commit
0c1777e7f3
|
|
@ -16,21 +16,40 @@ package io.hetu.core.common.util;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
public class SecurePathWhiteList
|
||||
{
|
||||
private static Set<String> securePathwhiteList = new HashSet<>(Arrays.asList(
|
||||
"/tmp",
|
||||
"/opt/hetu",
|
||||
"/opt/openlookeng",
|
||||
"/etc/hetu",
|
||||
"/etc/openlookeng"));
|
||||
|
||||
private SecurePathWhiteList()
|
||||
{
|
||||
}
|
||||
|
||||
public static List<String> getSecurePathWhiteList() throws IOException
|
||||
/**
|
||||
* Due to security concerns, all data must be read from one of the whitelisted paths
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
public static Set<String> getSecurePathWhiteList() throws IOException
|
||||
{
|
||||
return new ArrayList<>(Arrays.asList(
|
||||
new File("..").getCanonicalPath(),
|
||||
"/tmp"));
|
||||
// current workspace can't be root directory.
|
||||
if (new File(".").getCanonicalPath().equals("/")) {
|
||||
throw new IllegalArgumentException("Current workspace can't be root directory," +
|
||||
"please make sure you have config node.data-dir in node.properties or don't put your bin directory under root directory");
|
||||
}
|
||||
// if user config node.data-dir, then current workspace is node.data-dir; if not, the current workspace is the same as bin.
|
||||
// the working directory is automatically set to {INSTALL_DIR}/bin by the launcher
|
||||
// the white list includes the {INSTALL_DIR}
|
||||
securePathwhiteList.add(new File(".").getCanonicalPath());
|
||||
return securePathwhiteList;
|
||||
}
|
||||
|
||||
public static boolean isSecurePath(String absolutePath) throws IOException
|
||||
|
|
|
|||
|
|
@ -120,6 +120,11 @@ In `etc/config.properties`:
|
|||
|
||||
In `etc/node.properties`:
|
||||
|
||||
Path white list：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]
|
||||
|
||||
Notice：avoid to choose root directory; ../ can't include in path; if you config node.date_dir, then the current workspace is node.data_dir;
|
||||
otherwise, the current workspace is the parent directory of openlookeng server's bin directory.
|
||||
|
||||
| Property Name | Mandatory | Description | Default Value |
|
||||
|----------------------------|-----------|---------------------------------------------------------------------------|---------------|
|
||||
| `catalog.config-dir` | YES | Root directory for storing configuration files in local disk. | |
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@
|
|||
|
||||
The HBase Connector allows querying and creating tables on an external Apache HBase instance. Users can create a table in HBase connector, mapping it to an existing table in HBase Cluster, and support insert, select, or delete.
|
||||
|
||||
The HBase Connector maintains a Metastore to persist HBase metadata, currently only following storage format are supported as Metastore: `Local File System`, `Hadoop Distributed File System (HDFS)`, and `openLooKeng Metastore`.
|
||||
The HBase Connector maintains a Metastore to persist HBase metadata, currently support Metastore: `openLooKeng Metastore`.
|
||||
|
||||
**Note:** *Apache HBase 1.3.1 version is supported by HBase connector*
|
||||
**Note:** *We use Apache HBase 2.2.3 version in HBase connector*
|
||||
|
||||
|
||||
|
||||
|
|
@ -23,35 +23,13 @@ hbase.zookeeper.quorum=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx
|
|||
|
||||
hbase.zookeeper.property.clientPort=xxxx
|
||||
|
||||
hbase.metastore.type=local
|
||||
|
||||
hbase.metastore.uri=/xxx/hbasemetastore.ini
|
||||
hbase.metastore.type=hetuMetastore
|
||||
```
|
||||
|
||||
For the value of `hbase.zookeeper.quorum`, please use comma (`,`) as the delimiter if it has multiple ip addresses.
|
||||
|
||||
**Note**
|
||||
|
||||
If you are using `Local File System`, `HDFS` as Metastore, you need to create an empty file , such as `hbasemetastore.ini`.
|
||||
|
||||
|
||||
|
||||
**Use HDFS to store HBase metadata**
|
||||
|
||||
You need to add below properties:
|
||||
|
||||
```properties
|
||||
hbase.core.site.path=/xxx/core-site.xml
|
||||
|
||||
hbase.hdfs.site.path=/xxx/hdfs-site.xml
|
||||
|
||||
hbase.metastore.type=hdfs
|
||||
|
||||
hbase.metastore.uri=hdfs://xxx.xxx.xxx.xxx:21088/xxx/hbasemetastore.ini
|
||||
```
|
||||
|
||||
|
||||
|
||||
**Use openLooKeng Metastore to store HBase metadata**
|
||||
|
||||
You have to create `etc/hetu-metastore.properties` to connect database. For the details of configuration, please refer to [VDM Connector](./vdm.md). Adding below property to the configuration file:
|
||||
|
|
@ -60,26 +38,6 @@ You have to create `etc/hetu-metastore.properties` to connect database. For the
|
|||
hbase.metastore.type=hetuMetastore
|
||||
```
|
||||
|
||||
|
||||
|
||||
**Kerberos Configuration:**
|
||||
|
||||
```properties
|
||||
hbase.jaas.conf.path=/xxx/jaas.conf
|
||||
|
||||
hbase.hbase.site.path=/xxx/hbase-site.xml
|
||||
|
||||
hbase.krb5.conf.path=/xxx/krb5.conf
|
||||
|
||||
hbase.kerberos.keytab=/xxx/user.keytab
|
||||
|
||||
hbase.kerberos.principal=xxx
|
||||
|
||||
hbase.authentication.type=KERBEROS
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Configuration Properties
|
||||
|
||||
| Property Name | Default Value | Required | Description |
|
||||
|
|
@ -90,17 +48,7 @@ hbase.authentication.type=KERBEROS
|
|||
| hbase.client.pause.time | 100 | No | HBase client disconnect time |
|
||||
| hbase.rpc.protection.enable | false | No | Communication privacy protection. You can get this from `hbase-site.xml`. |
|
||||
| hbase.default.value | NULL | No | The default value of data in table |
|
||||
| hbase.metastore.type | local | No | The storage of hbase metadata, you can choose one of `local/hdfs/hetuMetastore` |
|
||||
| hbase.metastore.uri | (none) | Yes | File path for storing hbase metadata |
|
||||
| hbase.core.site.path | (none) | No | Configuration file for connecting hdfs |
|
||||
| hbase.hdfs.site.path | (none) | No | Configuration file for connecting hdfs |
|
||||
| hbase.authentication.type | (none) | No | Access security authentication mode of hdfs/hbase component |
|
||||
| hbase.kerberos.principal | (none) | No | User name for security authentication |
|
||||
| hbase.kerberos.keytab | (none) | No | Key for security authentication |
|
||||
| hbase.hbase.site.path | (none) | No | Configuration used to connect to a secure hbase cluster |
|
||||
| hbase.jaas.conf.path | (none) | No | Jaas for security authentication |
|
||||
| hbase.krb5.conf.path | (none) | No | Krb5 for security authentication |
|
||||
|
||||
| hbase.metastore.type | hetuMetastore | No | The storage of hbase metadata, you can choose `hetuMetastore` |
|
||||
|
||||
|
||||
## Table Properties
|
||||
|
|
|
|||
|
|
@ -23,6 +23,12 @@ hetu.metastore.db.user=root
|
|||
hetu.metastore.db.password=123456
|
||||
```
|
||||
* Here is an example of using HDFS as metastore，create `etc/hetu-metastore.properties`：
|
||||
|
||||
Path white list：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]
|
||||
|
||||
Notice：avoid to choose root directory; ../ can't include in path; if you config node.date_dir, then the current workspace is node.data_dir;
|
||||
otherwise, the current workspace is the parent directory of openlookeng server's bin directory.
|
||||
|
||||
```
|
||||
# the type of metastore storage
|
||||
hetu.metastore.type=hetufilesystem
|
||||
|
|
|
|||
|
|
@ -48,6 +48,11 @@ We will use this example throughout this tutorial.
|
|||
|
||||
In `etc/config.properties`, add these lines:
|
||||
|
||||
Path white list：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", current workspace]
|
||||
|
||||
Notice：avoid to choose root directory; ../ can't include in path; if you config node.date_dir, then the current workspace is node.data_dir;
|
||||
otherwise, the current workspace is the parent directory of openlookeng server's bin directory.
|
||||
|
||||
hetu.heuristicindex.filter.enabled=true
|
||||
hetu.heuristicindex.filter.cache.max-indices-number=2000000
|
||||
hetu.heuristicindex.indexstore.uri=/opt/hetu/indices
|
||||
|
|
|
|||
|
|
@ -119,6 +119,11 @@ UPDATE操作是DELETE和ADD操作的组合。首先管理员向协调节点发
|
|||
|
||||
在`etc/node.properties`中：
|
||||
|
||||
路径配置白名单：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", 工作目录]
|
||||
|
||||
注意：避免选择根目录；路径不能包含../；如果配置了node.data_dir,那么当前工作目录为node.data_dir；
|
||||
如果没有配置，那么当前工作目录为openlookeng server的bin目录的上级目录
|
||||
|
||||
| 属性名称| 是否必选| 描述| 默认值|
|
||||
|----------|----------|----------|----------|
|
||||
| `catalog.config-dir`| 是| 本地磁盘存放配置文件的根目录。|
|
||||
|
|
|
|||
|
|
@ -8,9 +8,9 @@
|
|||
|
||||
HBase连接支持在外部Apache HBase实例上查询和创建表。用户可以在HBase连接器中创建表，并映射到HBase Cluster中已有的表，支持insert、select和delete操作。
|
||||
|
||||
HBase连接器维护着一个元存储，用于持久化HBase元数据，目前元存储只支持以下存储格式：`Local File System`、`Hadoop Distributed File System (HDFS)`和`openLooKeng Metastore`。
|
||||
HBase连接器维护着一个元存储，用于持久化HBase元数据，目前元存储只支持以下存储格式：`openLooKeng Metastore`。
|
||||
|
||||
**注意：** *Hbase连接器支持Apache HBase 1.3.1版本。*
|
||||
**注意：** *Hbase连接器使用Apache HBase 2.2.3版本。*
|
||||
|
||||
## 连接器配置
|
||||
|
||||
|
|
@ -23,31 +23,13 @@ hbase.zookeeper.quorum=xxx.xxx.xxx.xxx,xxx.xxx.xxx.xxx
|
|||
|
||||
hbase.zookeeper.property.clientPort=xxxx
|
||||
|
||||
hbase.metastore.type=local
|
||||
|
||||
hbase.metastore.uri=/xxx/hbasemetastore.ini
|
||||
hbase.metastore.type=hetuMetastore
|
||||
```
|
||||
|
||||
对于`hbase.zookeeper.quorum`的值，如果有多个IP地址，请使用逗号（`,`）作为分隔符。
|
||||
|
||||
**注意**
|
||||
|
||||
如果使用`Local File System`和`HDFS`作为元存储，则需要创建一个空文件，如`hbasemetastore.ini`。
|
||||
|
||||
**使用HDFS存储HBase元数据**
|
||||
|
||||
需要添加以下属性：
|
||||
|
||||
```properties
|
||||
hbase.core.site.path=/xxx/core-site.xml
|
||||
|
||||
hbase.hdfs.site.path=/xxx/hdfs-site.xml
|
||||
|
||||
hbase.metastore.type=hdfs
|
||||
|
||||
hbase.metastore.uri=hdfs://xxx.xxx.xxx.xxx:21088/xxx/hbasemetastore.ini
|
||||
```
|
||||
|
||||
**使用openLooKeng元存储来存储HBase元数据**
|
||||
|
||||
必须创建`etc/hetu-metastore.properties`来连接数据库。具体配置请参见[VDM](vdm.html)连接器。将以下属性添加到配置文件：
|
||||
|
|
@ -56,22 +38,6 @@ hbase.metastore.uri=hdfs://xxx.xxx.xxx.xxx:21088/xxx/hbasemetastore.ini
|
|||
hbase.metastore.type=hetuMetastore
|
||||
```
|
||||
|
||||
**Kerberos配置：**
|
||||
|
||||
```properties
|
||||
hbase.jaas.conf.path=/xxx/jaas.conf
|
||||
|
||||
hbase.hbase.site.path=/xxx/hbase-site.xml
|
||||
|
||||
hbase.krb5.conf.path=/xxx/krb5.conf
|
||||
|
||||
hbase.kerberos.keytab=/xxx/user.keytab
|
||||
|
||||
hbase.kerberos.principal=xxx
|
||||
|
||||
hbase.authentication.type=KERBEROS
|
||||
```
|
||||
|
||||
## 配置属性
|
||||
|
||||
| 属性名称| 默认值| 是否必填| 说明|
|
||||
|
|
@ -82,16 +48,8 @@ hbase.authentication.type=KERBEROS
|
|||
| hbase.client.pause.time| 100| 否| HBase客户端断连时间|
|
||||
| hbase.rpc.protection.enable| false| 否| 通信隐私保护。可以从`hbase-site.xml`获取该属性的值。|
|
||||
| hbase.default.value| NULL| 否| 表中数据的默认值|
|
||||
| hbase.metastore.type| local| 否| HBase元数据的存储，可以从`local/hdfs/hetuMetastore`中选择一种|
|
||||
| hbase.metastore.uri| （无）| 是| 存储HBase元数据的文件路径|
|
||||
| hbase.core.site.path| （无）| 否| 连接HDFS的配置文件|
|
||||
| hbase.hdfs.site.path| （无）| 否| 连接HDFS的配置文件|
|
||||
| hbase.authentication.type| （无）| 否| HDFS/HBase组件访问安全身份验证方式|
|
||||
| hbase.kerberos.principal| （无）| 否| 安全身份验证的用户名|
|
||||
| hbase.kerberos.keytab| （无）| 否| 安全身份验证的密钥|
|
||||
| hbase.hbase.site.path| （无）| 否| 连接安全HBase集群的配置|
|
||||
| hbase.jaas.conf.path| （无）| 否| 安全身份验证的JAAS|
|
||||
| hbase.krb5.conf.path| （无）| 否| 安全身份验证的krb5|
|
||||
| hbase.metastore.type| hetuMetastore| 否| HBase元数据的存储，`hetuMetastore`|
|
||||
|
||||
|
||||
## 表属性
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,12 @@ hetu.metastore.db.user=root
|
|||
hetu.metastore.db.password=my-mysql-pwd
|
||||
```
|
||||
* 下面是使用HDFS作为元数据存储的实例，创建`etc/hetu-metastore.properties`：
|
||||
|
||||
路径配置白名单：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", 工作目录]
|
||||
|
||||
注意：避免选择根目录；路径不能包含../；如果配置了node.data_dir,那么当前工作目录为node.data_dir；
|
||||
如果没有配置，那么当前工作目录为openlookeng server的bin目录的上级目录
|
||||
|
||||
```
|
||||
# the type of metastore storage
|
||||
hetu.metastore.type=hetufilesystem
|
||||
|
|
|
|||
|
|
@ -48,6 +48,11 @@
|
|||
|
||||
在 `etc/config.properties` 中加入这些行：
|
||||
|
||||
路径配置白名单：["/tmp", "/opt/hetu", "/opt/openlookeng", "/etc/hetu", "/etc/openlookeng", 工作目录]
|
||||
|
||||
注意：避免选择根目录；路径不能包含../；如果配置了node.data_dir,那么当前工作目录为node.data_dir；
|
||||
如果没有配置，那么当前工作目录为openlookeng server的bin目录的上级目录
|
||||
|
||||
hetu.heuristicindex.filter.enabled=true
|
||||
hetu.heuristicindex.filter.cache.max-indices-number=2000000
|
||||
hetu.heuristicindex.indexstore.uri=/opt/hetu/indices
|
||||
|
|
|
|||
Loading…
Reference in New Issue