jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update or filter component for cves

This commit is contained in:
bryanwongsz 2020-09-29 16:07:00 +08:00
parent 33de6e3dff
commit 6e4a4d3712
5 changed files with 166 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
hetu-carbondata/pom.xml
View File

@ -717,12 +717,40 @@
<artifactId>hadoop-yarn-api</artifactId>
<version>${hadoop.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
<version>${hadoop.version}</version>
<scope>runtime</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@ -734,6 +762,78 @@
<artifactId>jetty-util</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>zookeeper</artifactId>
<groupId>org.apache.zookeeper</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
<artifactId>json-smart</artifactId>
<groupId>net.minidev</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-guice</artifactId>
<groupId>com.sun.jersey.contribs</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>com.fasterxml.jackson.module</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
</exclusion>
<exclusion>
<artifactId>*</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
</exclusion>
<exclusion>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>nimbus-jose-jwt</artifactId>
<groupId>com.nimbusds</groupId>
</exclusion>
<exclusion>
<artifactId>okhttp</artifactId>
<groupId>com.squareup.okhttp</groupId>
</exclusion>
<exclusion>
<artifactId>protobuf-java</artifactId>
<groupId>com.google.protobuf</groupId>
</exclusion>
<exclusion>
<artifactId>guice-servlet</artifactId>
<groupId>com.google.inject.extensions</groupId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>

2
hetu-hazelcast/pom.xml
View File

@ -17,7 +17,7 @@
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
<hazelcast.version>4.0.3</hazelcast.version>
<aspectjweaver.version>1.9.4</aspectjweaver.version>
<spring-context.version>5.2.0.RELEASE</spring-context.version>
<spring-context.version>5.2.5.RELEASE</spring-context.version>
<cache-api.version>1.1.1</cache-api.version>
<jmockit.version>1.38</jmockit.version>
</properties>

52
hetu-heuristic-index/pom.xml
View File

@ -39,7 +39,7 @@
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId>
<version>0.16.0-incubating</version>
<version>0.19.0</version>
<exclusions>
<exclusion>
<artifactId>netty-resolver</artifactId>
@ -137,12 +137,36 @@
<artifactId>async-http-client-netty-utils</artifactId>
<groupId>org.asynchttpclient</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-annotations</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-datatype-guava</artifactId>
<groupId>com.fasterxml.jackson.datatype</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-datatype-joda</artifactId>
<groupId>com.fasterxml.jackson.datatype</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-dataformat-smile</artifactId>
<groupId>com.fasterxml.jackson.dataformat</groupId>
</exclusion>
<exclusion>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-validator</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
<version>0.16.0-incubating</version>
<version>0.19.0</version>
<exclusions>
<exclusion>
<groupId>joda-time</groupId>
@ -156,6 +180,30 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-annotations</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-datatype-guava</artifactId>
<groupId>com.fasterxml.jackson.datatype</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-datatype-joda</artifactId>
<groupId>com.fasterxml.jackson.datatype</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-dataformat-smile</artifactId>
<groupId>com.fasterxml.jackson.dataformat</groupId>
</exclusion>
<exclusion>
<artifactId>netty-common</artifactId>
<groupId>io.netty</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>

11
hetu-heuristic-index/src/main/java/io/hetu/core/plugin/heuristicindex/index/bitmap/BitMapIndex.java
View File

@ -27,6 +27,7 @@ import io.prestosql.spi.predicate.SortedRangeSet;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedInputRow;
import org.apache.druid.data.input.impl.DimensionsSpec;
import org.apache.druid.data.input.impl.StringDimensionSchema;
@ -75,6 +76,7 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@ -101,6 +103,11 @@ import static java.util.stream.Collectors.toMap;
public class BitMapIndex<T>
implements Index<T>
{
static
{
NullHandling.initializeForTests();
}
static final int DEFAULT_EXPECTED_NUM_OF_SIZE = 200000;
private static final Logger LOG = LoggerFactory.getLogger(BitMapIndex.class);
private static final String COLUMN_NAME = "column";
@ -449,10 +456,10 @@ public class BitMapIndex<T>
BitmapResultFactory<?> bitmapResultFactory = new DefaultBitmapResultFactory(bitmapIndexSelector.getBitmapFactory());
if (filters.size() == 0) {
filters.add(new TrueFilter());
filters.add(TrueFilter.instance());
}
ImmutableBitmap bm = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, filters);
ImmutableBitmap bm = AndFilter.getBitmapIndex(bitmapIndexSelector, bitmapResultFactory, new HashSet(filters));
if (lastBm == null) {
lastBm = bm;

6
hetu-hive-functions/pom.xml
View File

@ -118,6 +118,12 @@
<dependency>
<groupId>io.prestosql.hive</groupId>
<artifactId>hive-apache</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.prestosql.hadoop</groupId>