mirror of https://github.com/apache/cassandra
Integrate SASI index into Cassandra
patch by xedin; reviewed by beobal for CASSANDRA-10661
This commit is contained in:
Pavel Yaskevich
parent
11c8ca6b56
commit
72790dc8e3
|
|
@ -1,4 +1,5 @@
|
|||
3.4
|
||||
* Integrate SASI index into Cassandra (CASSANDRA-10661)
|
||||
* Add --skip-flush option to nodetool snapshot
|
||||
* Skip values for non-queried columns (CASSANDRA-10657)
|
||||
* Add support for secondary indexes on static columns (CASSANDRA-8103)
|
||||
|
|
|
|||
22
build.xml
22
build.xml
|
|
@ -243,6 +243,15 @@
|
|||
</wikitext-to-html>
|
||||
</target>
|
||||
|
||||
<!--
|
||||
Generates Java sources for tokenization support from jflex
|
||||
grammar files
|
||||
-->
|
||||
<target name="generate-jflex-java" description="Generate Java from jflex grammar">
|
||||
<taskdef classname="jflex.anttask.JFlexTask" classpath="${build.lib}/jflex-1.6.0.jar" name="jflex" />
|
||||
<jflex file="${build.src.java}/org/apache/cassandra/index/sasi/analyzer/StandardTokenizerImpl.jflex" destdir="${build.src.gen-java}/" />
|
||||
</target>
|
||||
|
||||
<!--
|
||||
Fetch Maven Ant Tasks and Cassandra's dependencies
|
||||
These targets are intentionally free of dependencies so that they
|
||||
|
|
@ -405,6 +414,11 @@
|
|||
<exclusion groupId="log4j" artifactId="log4j"/>
|
||||
</dependency>
|
||||
<dependency groupId="joda-time" artifactId="joda-time" version="2.4" />
|
||||
<dependency groupId="com.carrotsearch" artifactId="hppc" version="0.5.4" />
|
||||
<dependency groupId="de.jflex" artifactId="jflex" version="1.6.0" />
|
||||
<dependency groupId="net.mintern" artifactId="primitive" version="1.0" />
|
||||
<dependency groupId="com.github.rholder" artifactId="snowball-stemmer" version="1.3.0.581.1" />
|
||||
<dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" version="2.4.0" />
|
||||
|
||||
</dependencyManagement>
|
||||
<developer id="alakshman" name="Avinash Lakshman"/>
|
||||
|
|
@ -568,6 +582,12 @@
|
|||
<dependency groupId="org.slf4j" artifactId="log4j-over-slf4j"/>
|
||||
<dependency groupId="org.slf4j" artifactId="jcl-over-slf4j"/>
|
||||
<dependency groupId="org.apache.thrift" artifactId="libthrift"/>
|
||||
<dependency groupId="com.carrotsearch" artifactId="hppc" version="0.5.4" />
|
||||
<dependency groupId="de.jflex" artifactId="jflex" version="1.6.0" />
|
||||
<dependency groupId="net.mintern" artifactId="primitive" version="1.0" />
|
||||
<dependency groupId="com.github.rholder" artifactId="snowball-stemmer" version="1.3.0.581.1" />
|
||||
<dependency groupId="com.googlecode.concurrent-trees" artifactId="concurrent-trees" version="2.4.0" />
|
||||
|
||||
</artifact:pom>
|
||||
<artifact:pom id="clientutil-pom"
|
||||
artifactId="cassandra-clientutil"
|
||||
|
|
@ -734,7 +754,7 @@
|
|||
depends="maven-ant-tasks-retrieve-build,build-project" description="Compile Cassandra classes"/>
|
||||
<target name="codecoverage" depends="jacoco-run,jacoco-report" description="Create code coverage report"/>
|
||||
|
||||
<target depends="init,gen-cql3-grammar,generate-cql-html"
|
||||
<target depends="init,gen-cql3-grammar,generate-cql-html,generate-jflex-java"
|
||||
name="build-project">
|
||||
<echo message="${ant.project.name}: ${ant.file}"/>
|
||||
<!-- Order matters! -->
|
||||
|
|
|
|||
|
|
@ -0,0 +1,768 @@
|
|||
# SASIIndex
|
||||
|
||||
[`SASIIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/SASIIndex.java),
|
||||
or "SASI" for short, is an implementation of Cassandra's
|
||||
`Index` interface that can be used as an alternative to the
|
||||
existing implementations. SASI's indexing and querying improves on
|
||||
existing implementations by tailoring it specifically to Cassandra's
|
||||
needs. SASI has superior performance in cases where queries would
|
||||
previously require filtering. In achieving this performance, SASI aims
|
||||
to be significantly less resource intensive than existing
|
||||
implementations, in memory, disk, and CPU usage. In addition, SASI
|
||||
supports prefix and contains queries on strings (similar to SQL's
|
||||
`LIKE = "foo*"` or `LIKE = "*foo*"'`).
|
||||
|
||||
The following goes on describe how to get up and running with SASI,
|
||||
demonstrates usage with examples, and provides some details on its
|
||||
implementation.
|
||||
|
||||
## Using SASI
|
||||
|
||||
The examples below walk through creating a table and indexes on its
|
||||
columns, and performing queries on some inserted data. The patchset in
|
||||
this repository includes support for the Thrift and CQL3 interfaces.
|
||||
|
||||
The examples below assume the `demo` keyspace has been created and is
|
||||
in use.
|
||||
|
||||
```
|
||||
cqlsh> CREATE KEYSPACE demo WITH replication = {
|
||||
... 'class': 'SimpleStrategy',
|
||||
... 'replication_factor': '1'
|
||||
... };
|
||||
cqlsh> USE demo;
|
||||
```
|
||||
|
||||
All examples are performed on the `sasi` table:
|
||||
|
||||
```
|
||||
cqlsh:demo> CREATE TABLE sasi (id uuid, first_name text, last_name text,
|
||||
... age int, height int, created_at bigint, primary key (id));
|
||||
```
|
||||
|
||||
#### Creating Indexes
|
||||
|
||||
To create SASI indexes use CQLs `CREATE CUSTOM INDEX` statement:
|
||||
|
||||
```
|
||||
cqlsh:demo> CREATE CUSTOM INDEX ON sasi (first_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
|
||||
... WITH OPTIONS = {
|
||||
... 'analyzer_class':
|
||||
... 'org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer',
|
||||
... 'case_sensitive': 'false'
|
||||
... };
|
||||
|
||||
cqlsh:demo> CREATE CUSTOM INDEX ON sasi (last_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
|
||||
... WITH OPTIONS = {'mode': 'CONTAINS'};
|
||||
|
||||
cqlsh:demo> CREATE CUSTOM INDEX ON sasi (age) USING 'org.apache.cassandra.index.sasi.SASIIndex';
|
||||
|
||||
cqlsh:demo> CREATE CUSTOM INDEX ON sasi (created_at) USING 'org.apache.cassandra.index.sasi.SASIIndex'
|
||||
... WITH OPTIONS = {'mode': 'SPARSE'};
|
||||
```
|
||||
|
||||
The indexes created have some options specified that customize their
|
||||
behaviour and potentially performance. The index on `first_name` is
|
||||
case-insensitive. The analyzers are discussed more in a subsequent
|
||||
example. The `NonTokenizingAnalyzer` performs no analysis on the
|
||||
text. Each index has a mode: `PREFIX`, `CONTAINS`, or `SPARSE`, the
|
||||
first being the default. The `last_name` index is created with the
|
||||
mode `CONTAINS` which matches terms on suffixes instead of prefix
|
||||
only. Examples of this are available below and more detail can be
|
||||
found in the section on
|
||||
[OnDiskIndex](https://github.com/xedin/sasi#ondiskindexbuilder).The
|
||||
`created_at` column is created with its mode set to `SPARSE`, which is
|
||||
meant to improve performance of querying large, dense number ranges
|
||||
like timestamps for data inserted every millisecond. Details of the
|
||||
`SPARSE` implementation can also be found in the section on the
|
||||
[OnDiskIndex](https://github.com/xedin/sasi#ondiskindexbuilder). The `age`
|
||||
index is created with the default `PREFIX` mode and no
|
||||
case-sensitivity or text analysis options are specified since the
|
||||
field is numeric.
|
||||
|
||||
After inserting the following data and performing a `nodetool flush`,
|
||||
SASI performing index flushes to disk can be seen in Cassandra's logs
|
||||
-- although the direct call to flush is not required (see
|
||||
[IndexMemtable](#indexmemtable) for more details).
|
||||
|
||||
```
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (556ebd54-cbe5-4b75-9aae-bf2a31a24500, 'Pavel', 'Yaskevich', 27, 181, 1442959315018);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (5770382a-c56f-4f3f-b755-450e24d55217, 'Jordan', 'West', 26, 173, 1442959315019);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (96053844-45c3-4f15-b1b7-b02c441d3ee1, 'Mikhail', 'Stepura', 36, 173, 1442959315020);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (f5dfcabe-de96-4148-9b80-a1c41ed276b4, 'Michael', 'Kjellman', 26, 180, 1442959315021);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (2970da43-e070-41a8-8bcb-35df7a0e608a, 'Johnny', 'Zhang', 32, 175, 1442959315022);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (6b757016-631d-4fdb-ac62-40b127ccfbc7, 'Jason', 'Brown', 40, 182, 1442959315023);
|
||||
|
||||
cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
|
||||
... VALUES (8f909e8a-008e-49dd-8d43-1b0df348ed44, 'Vijay', 'Parthasarathy', 34, 183, 1442959315024);
|
||||
|
||||
cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi;
|
||||
|
||||
first_name | last_name | age | height | created_at
|
||||
------------+---------------+-----+--------+---------------
|
||||
Michael | Kjellman | 26 | 180 | 1442959315021
|
||||
Mikhail | Stepura | 36 | 173 | 1442959315020
|
||||
Jason | Brown | 40 | 182 | 1442959315023
|
||||
Pavel | Yaskevich | 27 | 181 | 1442959315018
|
||||
Vijay | Parthasarathy | 34 | 183 | 1442959315024
|
||||
Jordan | West | 26 | 173 | 1442959315019
|
||||
Johnny | Zhang | 32 | 175 | 1442959315022
|
||||
|
||||
(7 rows)
|
||||
```
|
||||
|
||||
#### Equality & Prefix Queries
|
||||
|
||||
SASI supports simple queries already supported by CQL, however, for
|
||||
text fields like `first_name` equals queries perform prefix searches
|
||||
-- this is similar to `first_name LIKE 'M*'` in SQL (excluding case
|
||||
sensitivity, which is dependent on the index configuration). The
|
||||
semantics of CQL's `=` were modified instead of making further
|
||||
modifications of the grammar with the introduction of a `LIKE`
|
||||
operator. Ideally, CQL would be modified to include such an operator,
|
||||
supporting both prefix and suffix searches.
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
|
||||
... WHERE first_name = 'M';
|
||||
|
||||
first_name | last_name | age | height | created_at
|
||||
------------+-----------+-----+--------+---------------
|
||||
Michael | Kjellman | 26 | 180 | 1442959315021
|
||||
Mikhail | Stepura | 36 | 173 | 1442959315020
|
||||
|
||||
(2 rows)
|
||||
```
|
||||
|
||||
Of course, the case of the query does not matter for the `first_name`
|
||||
column because of the options provided at index creation time.
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
|
||||
... WHERE first_name = 'm';
|
||||
|
||||
first_name | last_name | age | height | created_at
|
||||
------------+-----------+-----+--------+---------------
|
||||
Michael | Kjellman | 26 | 180 | 1442959315021
|
||||
Mikhail | Stepura | 36 | 173 | 1442959315020
|
||||
|
||||
(2 rows)
|
||||
```
|
||||
|
||||
#### Compound Queries
|
||||
|
||||
SASI supports queries with multiple predicates, however, due to the
|
||||
nature of the default indexing implementation, CQL requires the user
|
||||
to specify `ALLOW FILTERING` to opt-in to the potential performance
|
||||
pitfalls of such a query. With SASI, while the requirement to include
|
||||
`ALLOW FILTERING` remains, to reduce modifications to the grammar, the
|
||||
performance pitfalls do not exist because filtering is not
|
||||
performed. Details on how SASI joins data from multiple predicates is
|
||||
available below in the
|
||||
[Implementation Details](https://github.com/xedin/sasi#implementation-details)
|
||||
section.
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
|
||||
... WHERE first_name = 'M' and age < 30 ALLOW FILTERING;
|
||||
|
||||
first_name | last_name | age | height | created_at
|
||||
------------+-----------+-----+--------+---------------
|
||||
Michael | Kjellman | 26 | 180 | 1442959315021
|
||||
|
||||
(1 rows)
|
||||
```
|
||||
|
||||
#### Suffix Queries
|
||||
|
||||
The next example demonstrates `CONTAINS` mode on the `last_name`
|
||||
column. By using this mode predicates can search for any strings
|
||||
containing the search string as a sub-string. In this case the strings
|
||||
containing "a" or "an".
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE last_name = 'a';
|
||||
|
||||
id | age | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+---------------+------------+--------+---------------
|
||||
f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
|
||||
96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | 1442959315020 | Mikhail | 173 | Stepura
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
|
||||
2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
|
||||
|
||||
(5 rows)
|
||||
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE last_name = 'an';
|
||||
|
||||
id | age | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+---------------+------------+--------+-----------
|
||||
f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
|
||||
2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
|
||||
|
||||
(2 rows)
|
||||
```
|
||||
|
||||
#### Expressions on Non-Indexed Columns
|
||||
|
||||
SASI also supports filtering on non-indexed columns like `height`. The
|
||||
expression can only narrow down an existing query using `AND`.
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE last_name = 'a' AND height >= 175 ALLOW FILTERING;
|
||||
|
||||
id | age | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+---------------+------------+--------+---------------
|
||||
f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
|
||||
2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
|
||||
|
||||
(4 rows)
|
||||
```
|
||||
|
||||
#### Text Analysis (Tokenization and Stemming)
|
||||
|
||||
Lastly, to demonstrate text analysis an additional column is needed on
|
||||
the table. Its definition, index, and statements to update rows are shown below.
|
||||
|
||||
```
|
||||
cqlsh:demo> ALTER TABLE sasi ADD bio text;
|
||||
cqlsh:demo> CREATE CUSTOM INDEX ON sasi (bio) USING 'org.apache.cassandra.index.sasi.SASIIndex'
|
||||
... WITH OPTIONS = {
|
||||
... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',
|
||||
... 'tokenization_enable_stemming': 'true',
|
||||
... 'analyzed': 'true',
|
||||
... 'tokenization_normalize_lowercase': 'true',
|
||||
... 'tokenization_locale': 'en'
|
||||
... };
|
||||
cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, who likes distributed systems, doesnt like to argue.' WHERE id = 5770382a-c56f-4f3f-b755-450e24d55217;
|
||||
cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, works on the freight distribution at nights and likes arguing' WHERE id = 556ebd54-cbe5-4b75-9aae-bf2a31a24500;
|
||||
cqlsh:demo> SELECT * FROM sasi;
|
||||
|
||||
id | age | bio | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+---------------
|
||||
f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | null | 1442959315021 | Michael | 180 | Kjellman
|
||||
96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | null | 1442959315020 | Mikhail | 173 | Stepura
|
||||
6b757016-631d-4fdb-ac62-40b127ccfbc7 | 40 | null | 1442959315023 | Jason | 182 | Brown
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | null | 1442959315024 | Vijay | 183 | Parthasarathy
|
||||
5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
|
||||
2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | null | 1442959315022 | Johnny | 175 | Zhang
|
||||
|
||||
(7 rows)
|
||||
```
|
||||
|
||||
Index terms and query search strings are stemmed for the `bio` column
|
||||
because it was configured to use the
|
||||
[`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java)
|
||||
and `analyzed` is set to `true`. The
|
||||
`tokenization_normalize_lowercase` is similar to the `case_sensitive`
|
||||
property but for the
|
||||
[`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java). These
|
||||
query demonstrates the stemming applied by [`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java).
|
||||
|
||||
```
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE bio = 'distributing';
|
||||
|
||||
id | age | bio | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
|
||||
|
||||
(2 rows)
|
||||
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE bio = 'they argued';
|
||||
|
||||
id | age | bio | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
|
||||
|
||||
(2 rows)
|
||||
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE bio = 'working at the company';
|
||||
|
||||
id | age | bio | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
|
||||
(1 rows)
|
||||
|
||||
cqlsh:demo> SELECT * FROM sasi WHERE bio = 'soft eng';
|
||||
|
||||
id | age | bio | created_at | first_name | height | last_name
|
||||
--------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
|
||||
556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
|
||||
5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
|
||||
|
||||
(2 rows)
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
While SASI, at the surface, is simply an implementation of the
|
||||
`Index` interface, at its core there are several data
|
||||
structures and algorithms used to satisfy it. These are described
|
||||
here. Additionally, the changes internal to Cassandra to support SASIs
|
||||
integration are described.
|
||||
|
||||
The `Index` interface divides responsibility of the
|
||||
implementer into two parts: Indexing and Querying. Further, Cassandra
|
||||
makes it possible to divide those responsibilities into the memory and
|
||||
disk components. SASI takes advantage of Cassandra's write-once,
|
||||
immutable, ordered data model to build indexes along with the flushing
|
||||
of the memtable to disk -- this is the origin of the name "SSTable
|
||||
Attached Secondary Index".
|
||||
|
||||
The SASI index data structures are built in memory as the SSTable is
|
||||
being written and they are flushed to disk before the writing of the
|
||||
SSTable completes. The writing of each index file only requires
|
||||
sequential writes to disk. In some cases, partial flushes are
|
||||
performed, and later stitched back together, to reduce memory
|
||||
usage. These data structures are optimized for this use case.
|
||||
|
||||
Taking advantage of Cassandra's ordered data model, at query time,
|
||||
candidate indexes are narrowed down for searching minimize the amount
|
||||
of work done. Searching is then performed using an efficient method
|
||||
that streams data off disk as needed.
|
||||
|
||||
### Indexing
|
||||
|
||||
Per SSTable, SASI writes an index file for each indexed column. The
|
||||
data for these files is built in memory using the
|
||||
[`OnDiskIndexBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java). Once
|
||||
flushed to disk, the data is read using the
|
||||
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
|
||||
class. These are composed of bytes representing indexed terms,
|
||||
organized for efficient writing or searching respectively. The keys
|
||||
and values they hold represent tokens and positions in an SSTable and
|
||||
these are stored per-indexed term in
|
||||
[`TokenTreeBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java)s
|
||||
for writing, and
|
||||
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s
|
||||
for querying. These index files are memory mapped after being written
|
||||
to disk, for quicker access. For indexing data in the memtable SASI
|
||||
uses its
|
||||
[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
|
||||
class.
|
||||
|
||||
#### OnDiskIndex(Builder)
|
||||
|
||||
Each
|
||||
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
|
||||
is an instance of a modified
|
||||
[Suffix Array](https://en.wikipedia.org/wiki/Suffix_array) data
|
||||
structure. The
|
||||
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
|
||||
is comprised of page-size blocks of sorted terms and pointers to the
|
||||
terms' associated data, as well as the data itself, stored also in one
|
||||
or more page-sized blocks. The
|
||||
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
|
||||
is structured as a tree of arrays, where each level describes the
|
||||
terms in the level below, the final level being the terms
|
||||
themselves. The `PointerLevel`s and their `PointerBlock`s contain
|
||||
terms and pointers to other blocks that *end* with those terms. The
|
||||
`DataLevel`, the final level, and its `DataBlock`s contain terms and
|
||||
point to the data itself, contained in [`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s.
|
||||
|
||||
The terms written to the
|
||||
[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
|
||||
vary depending on its "mode": either `PREFIX`, `CONTAINS`, or
|
||||
`SPARSE`. In the `PREFIX` and `SPARSE` cases terms exact values are
|
||||
written exactly once per `OnDiskIndex`. For example, a `PREFIX` index
|
||||
with terms `Jason`, `Jordan`, `Pavel`, all three will be included in
|
||||
the index. A `CONTAINS` index writes additional terms for each suffix of
|
||||
each term recursively. Continuing with the example, a `CONTAINS` index
|
||||
storing the previous terms would also store `ason`, `ordan`, `avel`,
|
||||
`son`, `rdan`, `vel`, etc. This allows for queries on the suffix of
|
||||
strings. The `SPARSE` mode differs from `PREFIX` in that for every 64
|
||||
blocks of terms a
|
||||
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)
|
||||
is built merging all the
|
||||
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s
|
||||
for each term into a single one. This copy of the data is used for
|
||||
efficient iteration of large ranges of e.g. timestamps. The index
|
||||
"mode" is configurable per column at index creation time.
|
||||
|
||||
#### TokenTree(Builder)
|
||||
|
||||
The
|
||||
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)
|
||||
is an implementation of the well-known
|
||||
[B+-tree](https://en.wikipedia.org/wiki/B%2B_tree) that has been
|
||||
modified to optimize for its use-case. In particular, it has been
|
||||
optimized to associate tokens, longs, with a set of positions in an
|
||||
SSTable, also longs. Allowing the set of long values accommodates
|
||||
the possibility of a hash collision in the token, but the data
|
||||
structure is optimized for the unlikely possibility of such a
|
||||
collision.
|
||||
|
||||
To optimize for its write-once environment the
|
||||
[`TokenTreeBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java)
|
||||
completely loads its interior nodes as the tree is built and it uses
|
||||
the well-known algorithm optimized for bulk-loading the data
|
||||
structure.
|
||||
|
||||
[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s provide the means to iterate a tokens, and file
|
||||
positions, that match a given term, and to skip forward in that
|
||||
iteration, an operation used heavily at query time.
|
||||
|
||||
#### IndexMemtable
|
||||
|
||||
The
|
||||
[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
|
||||
handles indexing the in-memory data held in the memtable. The
|
||||
[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
|
||||
in turn manages either a
|
||||
[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
|
||||
or a
|
||||
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
|
||||
per-column. The choice of which index type is used is data
|
||||
dependent. The
|
||||
[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
|
||||
is used for literal types. `AsciiType` and `UTF8Type` are literal
|
||||
types by defualt but any column can be configured as a literal type
|
||||
using the `is_literal` option at index creation time. For non-literal
|
||||
types the
|
||||
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
|
||||
is used. The
|
||||
[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
|
||||
is an implementation that can efficiently support prefix queries on
|
||||
character-like data. The
|
||||
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java),
|
||||
conversely, is better suited for Cassandra other data types like
|
||||
numbers.
|
||||
|
||||
The
|
||||
[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
|
||||
is built using either the `ConcurrentRadixTree` or
|
||||
`ConcurrentSuffixTree` from the `com.goooglecode.concurrenttrees`
|
||||
package. The choice between the two is made based on the indexing
|
||||
mode, `PREFIX` or other modes, and `CONTAINS` mode, respectively.
|
||||
|
||||
The
|
||||
[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
|
||||
is built on top of `java.util.concurrent.ConcurrentSkipListSet`.
|
||||
|
||||
### Querying
|
||||
|
||||
Responsible for converting the internal `IndexExpression`
|
||||
representation into SASI's
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
|
||||
and
|
||||
[`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)
|
||||
tree, optimizing the tree to reduce the amount of work done, and
|
||||
driving the query itself the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
is the work horse of SASI's querying implementation. To efficiently
|
||||
perform union and intersection operations SASI provides several
|
||||
iterators similar to Cassandra's `MergeIterator` but tailored
|
||||
specifically for SASIs use, and with more features. The
|
||||
[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java),
|
||||
like its name suggests, performs set union over sets of tokens/keys
|
||||
matching the query, only reading as much data as it needs from each
|
||||
set to satisfy the query. The
|
||||
[`RangeIntersectionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java),
|
||||
similar to its counterpart, performs set intersection over its data.
|
||||
|
||||
#### QueryPlan
|
||||
|
||||
The
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
instantiated per search query is at the core of SASIs querying
|
||||
implementation. Its work can be divided in two stages: analysis and
|
||||
execution.
|
||||
|
||||
During the analysis phase,
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
converts from Cassandra's internal representation of
|
||||
`IndexExpression`s, which has also been modified to support encoding
|
||||
queries that contain ORs and groupings of expressions using
|
||||
parentheses (see the
|
||||
[Cassandra Internal Changes](https://github.com/xedin/sasi#cassandra-internal-changes)
|
||||
section below for more details). This process produces a tree of
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s, which in turn may contain [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s, all of which
|
||||
provide an alternative, more efficient, representation of the query.
|
||||
|
||||
During execution the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
uses the `DecoratedKey`-generating iterator created from the
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) tree. These keys are read from disk and a final check to
|
||||
ensure they satisfy the query is made, once again using the
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) tree. At the point the desired amount of matching data has
|
||||
been found, or there is no more matching data, the result set is
|
||||
returned to the coordinator through the existing internal components.
|
||||
|
||||
The number of queries (total/failed/timed-out), and their latencies,
|
||||
are maintined per-table/column family.
|
||||
|
||||
SASI also supports concurrently iterating terms for the same index
|
||||
accross SSTables. The concurrency factor is controlled by the
|
||||
`cassandra.search_concurrency_factor` system property. The default is
|
||||
`1`.
|
||||
|
||||
##### QueryController
|
||||
|
||||
Each
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
references a
|
||||
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
|
||||
used throughout the execution phase. The
|
||||
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
|
||||
has two responsibilities: to manage and ensure the proper cleanup of
|
||||
resources (indexes), and to strictly enforce the time bound for query,
|
||||
specified by the user via the range slice timeout. All indexes are
|
||||
accessed via the
|
||||
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
|
||||
so that they can be safely released by it later. The
|
||||
[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)'s
|
||||
`checkpoint` function is called in specific places in the execution
|
||||
path to ensure the time-bound is enforced.
|
||||
|
||||
##### QueryPlan Optimizations
|
||||
|
||||
While in the analysis phase, the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
performs several potential optimizations to the query. The goal of
|
||||
these optimizations is to reduce the amount of work performed during
|
||||
the execution phase.
|
||||
|
||||
The simplest optimization performed is compacting multiple expressions
|
||||
joined by logical intersection (`AND`) into a single [`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) with
|
||||
three or more [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s. For example, the query `WHERE age < 100 AND
|
||||
fname = 'p*' AND first_name != 'pa*' AND age > 21` would,
|
||||
without modification, have the following tree:
|
||||
|
||||
┌───────┐
|
||||
┌────────│ AND │──────┐
|
||||
│ └───────┘ │
|
||||
▼ ▼
|
||||
┌───────┐ ┌──────────┐
|
||||
┌─────│ AND │─────┐ │age < 100 │
|
||||
│ └───────┘ │ └──────────┘
|
||||
▼ ▼
|
||||
┌──────────┐ ┌───────┐
|
||||
│ fname=p* │ ┌─│ AND │───┐
|
||||
└──────────┘ │ └───────┘ │
|
||||
▼ ▼
|
||||
┌──────────┐ ┌──────────┐
|
||||
│fname!=pa*│ │ age > 21 │
|
||||
└──────────┘ └──────────┘
|
||||
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
will remove the redundant right branch whose root is the final `AND`
|
||||
and has leaves `fname != pa*` and `age > 21`. These [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s will
|
||||
be compacted into the parent `AND`, a safe operation due to `AND`
|
||||
being associative and commutative. The resulting tree looks like the
|
||||
following:
|
||||
|
||||
┌───────┐
|
||||
┌────────│ AND │──────┐
|
||||
│ └───────┘ │
|
||||
▼ ▼
|
||||
┌───────┐ ┌──────────┐
|
||||
┌───────────│ AND │────────┐ │age < 100 │
|
||||
│ └───────┘ │ └──────────┘
|
||||
▼ │ ▼
|
||||
┌──────────┐ │ ┌──────────┐
|
||||
│ fname=p* │ ▼ │ age > 21 │
|
||||
└──────────┘ ┌──────────┐ └──────────┘
|
||||
│fname!=pa*│
|
||||
└──────────┘
|
||||
|
||||
When excluding results from the result set, using `!=`, the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
determines the best method for handling it. For range queries, for
|
||||
example, it may be optimal to divide the range into multiple parts
|
||||
with a hole for the exclusion. For string queries, such as this one,
|
||||
it is more optimal, however, to simply note which data to skip, or
|
||||
exclude, while scanning the index. Following this optimization the
|
||||
tree looks like this:
|
||||
|
||||
┌───────┐
|
||||
┌────────│ AND │──────┐
|
||||
│ └───────┘ │
|
||||
▼ ▼
|
||||
┌───────┐ ┌──────────┐
|
||||
┌───────│ AND │────────┐ │age < 100 │
|
||||
│ └───────┘ │ └──────────┘
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌──────────┐
|
||||
│ fname=p* │ │ age > 21 │
|
||||
│ exclusions=[pa*] │ └──────────┘
|
||||
└──────────────────┘
|
||||
|
||||
The last type of optimization applied, for this query, is to merge
|
||||
range expressions across branches of the tree -- without modifying the
|
||||
meaning of the query, of course. In this case, because the query
|
||||
contains all `AND`s the `age` expressions can be collapsed. Along with
|
||||
this optimization, the initial collapsing of unneeded `AND`s can also
|
||||
be applied once more to result in this final tree using to execute the
|
||||
query:
|
||||
|
||||
┌───────┐
|
||||
┌──────│ AND │───────┐
|
||||
│ └───────┘ │
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌────────────────┐
|
||||
│ fname=p* │ │ 21 < age < 100 │
|
||||
│ exclusions=[pa*] │ └────────────────┘
|
||||
└──────────────────┘
|
||||
|
||||
#### Operations and Expressions
|
||||
|
||||
As discussed, the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
optimizes a tree represented by
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
|
||||
as interior nodes, and
|
||||
[`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s
|
||||
as leaves. The
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
|
||||
class, more specifically, can have zero, one, or two
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
|
||||
as children and an unlimited number of expressions. The iterators used
|
||||
to perform the queries, discussed below in the
|
||||
"Range(Union|Intersection)Iterator" section, implement the necessary
|
||||
logic to merge results transparently regardless of the
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
|
||||
children.
|
||||
|
||||
Besides participating in the optimizations performed by the
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java),
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
|
||||
is also responsible for taking a row that has been returned by the
|
||||
query and making a final validation that it in fact does match. This
|
||||
`satisfiesBy` operation is performed recursively from the root of the
|
||||
[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
|
||||
tree for a given query. These checks are performed directly on the
|
||||
data in a given row. For more details on how `satisfiesBy` works see
|
||||
the documentation
|
||||
[in the code](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java#L87-L123).
|
||||
|
||||
#### Range(Union|Intersection)Iterator
|
||||
|
||||
The abstract `RangeIterator` class provides a unified interface over
|
||||
the two main operations performed by SASI at various layers in the
|
||||
execution path: set intersection and union. These operations are
|
||||
performed in a iterated, or "streaming", fashion to prevent unneeded
|
||||
reads of elements from either set. In both the intersection and union
|
||||
cases the algorithms take advantage of the data being pre-sorted using
|
||||
the same sort order, e.g. term or token order.
|
||||
|
||||
The
|
||||
[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java)
|
||||
performs the "Merge-Join" portion of the
|
||||
[Sort-Merge-Join](https://en.wikipedia.org/wiki/Sort-merge_join)
|
||||
algorithm, with the properties of an outer-join, or union. It is
|
||||
implemented with several optimizations to improve its performance over
|
||||
a large number of iterators -- sets to union. Specifically, the
|
||||
iterator exploits the likely case of the data having many sub-groups
|
||||
of overlapping ranges and the unlikely case that all ranges will
|
||||
overlap each other. For more details see the
|
||||
[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java#L9-L21).
|
||||
|
||||
The
|
||||
[`RangeIntersectionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java)
|
||||
itself is not a subclass of `RangeIterator`. It is a container for
|
||||
several classes, one of which, `AbstractIntersectionIterator`,
|
||||
sub-classes `RangeIterator`. SASI supports two methods of performing
|
||||
the intersection operation, and the ability to be adaptive in choosing
|
||||
between them based on some properties of the data.
|
||||
|
||||
`BounceIntersectionIterator`, and the `BOUNCE` strategy, works like
|
||||
the
|
||||
[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java)
|
||||
in that it performs a "Merge-Join", however, its nature is similar to
|
||||
a inner-join, where like values are merged by a data-specific merge
|
||||
function (e.g. merging two tokens in a list to lookup in a SSTable
|
||||
later). See the
|
||||
[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L88-L101)
|
||||
for more details on its implementation.
|
||||
|
||||
`LookupIntersectionIterator`, and the `LOOKUP` strategy, performs a
|
||||
different operation, more similar to a lookup in an associative data
|
||||
structure, or "hash lookup" in database terminology. Once again,
|
||||
details on the implementation can be found in the
|
||||
[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L199-L208).
|
||||
|
||||
The choice between the two iterators, or the `ADAPTIVE` strategy, is
|
||||
based upon the ratio of data set sizes of the minimum and maximum
|
||||
range of the sets being intersected. If the number of the elements in
|
||||
minimum range divided by the number of elements is the maximum range
|
||||
is less than or equal to `0.01`, then the `ADAPTIVE` strategy chooses
|
||||
the `LookupIntersectionIterator`, otherwise the
|
||||
`BounceIntersectionIterator` is chosen.
|
||||
|
||||
### The SASIIndex Class
|
||||
|
||||
The above components are glued together by the
|
||||
[`SASIIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasiIndex.java)
|
||||
class which implements `Index`, and is instantiated
|
||||
per-table containing SASI indexes. It manages all indexes for a table
|
||||
via the
|
||||
[`sasi.conf.DataTracker`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java)
|
||||
and
|
||||
[`sasi.conf.view.View`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/view/View.java)
|
||||
components, controls writing of all indexes for an SSTable via its
|
||||
[`PerSSTableIndexWriter`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java), and initiates searches with
|
||||
`Indexer`. These classes glue the previously
|
||||
mentioned indexing components together with Cassandra's SSTable
|
||||
life-cycle ensuring indexes are not only written when Memtable's flush
|
||||
but also as SSTable's are compacted. For querying, the
|
||||
`Indexer` does little but defer to
|
||||
[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
|
||||
and update e.g. latency metrics exposed by SASI.
|
||||
|
||||
### Cassandra Internal Changes
|
||||
|
||||
To support the above changes and integrate them into Cassandra a few
|
||||
minor internal changes were made to Cassandra itself. These are
|
||||
described here.
|
||||
|
||||
#### SSTable Write Life-cycle Notifications
|
||||
|
||||
The `SSTableFlushObserver` is an observer pattern-like interface,
|
||||
whose sub-classes can register to be notified about events in the
|
||||
life-cycle of writing out a SSTable. Sub-classes can be notified when a
|
||||
flush begins and ends, as well as when each next row is about to be
|
||||
written, and each next column. SASI's `PerSSTableIndexWriter`,
|
||||
discussed above, is the only current subclass.
|
||||
|
||||
### Limitations and Caveats
|
||||
|
||||
The following are items that can be addressed in future updates but are not
|
||||
available in this repository or are not currently implemented.
|
||||
|
||||
* The cluster must be configured to use a partitioner that produces
|
||||
`LongToken`s, e.g. `Murmur3Partitioner`. Other existing partitioners which
|
||||
don't produce LongToken e.g. `ByteOrderedPartitioner` and `RandomPartitioner`
|
||||
will not work with SASI.
|
||||
* `ALLOW FILTERING`, the requirement of at least one indexes `=`
|
||||
expression, and lack of `LIKE` limit SASIs
|
||||
feature-set. Modifications to the grammar to allow `Index`
|
||||
implementations to enumerate its supported features would allow SASI
|
||||
to expose more features without need to support them in other
|
||||
implementations.
|
||||
* Not Equals and OR support have been removed in this release while
|
||||
changes are made to Cassandra itself to support them.
|
||||
|
||||
### Contributors
|
||||
|
||||
* [Pavel Yaskevich](https://github.com/xedin)
|
||||
* [Jordan West](https://github.com/jrwest)
|
||||
* [Michael Kjellman](https://github.com/mkjellman)
|
||||
* [Jason Brown](https://github.com/jasobrown)
|
||||
* [Mikhail Stepura](https://github.com/mishail)
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1964,5 +1964,10 @@ public class DatabaseDescriptor
|
|||
public static void setEncryptionContext(EncryptionContext ec)
|
||||
{
|
||||
encryptionContext = ec;
|
||||
}
|
||||
}
|
||||
|
||||
public static int searchConcurrencyFactor()
|
||||
{
|
||||
return Integer.valueOf(System.getProperty("cassandra.search_concurrency_factor", "1"));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -152,9 +152,9 @@ public class ColumnIndex
|
|||
|
||||
UnfilteredSerializer.serializer.serialize(unfiltered, header, writer, pos - previousRowStart, version);
|
||||
|
||||
// notify observers about each new cell added to the row
|
||||
if (!observers.isEmpty() && unfiltered.isRow())
|
||||
((Row) unfiltered).stream().forEach(cell -> observers.forEach((o) -> o.nextCell(cell)));
|
||||
// notify observers about each new row
|
||||
if (!observers.isEmpty())
|
||||
observers.forEach((o) -> o.nextUnfilteredCluster(unfiltered));
|
||||
|
||||
lastClustering = unfiltered.clustering();
|
||||
previousRowStart = pos;
|
||||
|
|
|
|||
|
|
@ -171,6 +171,11 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
return withNewExpressions(newExpressions);
|
||||
}
|
||||
|
||||
public RowFilter withoutExpressions()
|
||||
{
|
||||
return withNewExpressions(Collections.emptyList());
|
||||
}
|
||||
|
||||
protected abstract RowFilter withNewExpressions(List<Expression> expressions);
|
||||
|
||||
public boolean isEmpty()
|
||||
|
|
@ -312,7 +317,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
// Note: the order of this enum matter, it's used for serialization
|
||||
protected enum Kind { SIMPLE, MAP_EQUALITY, THRIFT_DYN_EXPR, CUSTOM }
|
||||
|
||||
abstract Kind kind();
|
||||
protected abstract Kind kind();
|
||||
protected final ColumnDefinition column;
|
||||
protected final Operator operator;
|
||||
protected final ByteBuffer value;
|
||||
|
|
@ -689,7 +694,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
}
|
||||
|
||||
@Override
|
||||
Kind kind()
|
||||
protected Kind kind()
|
||||
{
|
||||
return Kind.SIMPLE;
|
||||
}
|
||||
|
|
@ -782,7 +787,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
}
|
||||
|
||||
@Override
|
||||
Kind kind()
|
||||
protected Kind kind()
|
||||
{
|
||||
return Kind.MAP_EQUALITY;
|
||||
}
|
||||
|
|
@ -833,7 +838,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
}
|
||||
|
||||
@Override
|
||||
Kind kind()
|
||||
protected Kind kind()
|
||||
{
|
||||
return Kind.THRIFT_DYN_EXPR;
|
||||
}
|
||||
|
|
@ -883,7 +888,7 @@ public abstract class RowFilter implements Iterable<RowFilter.Expression>
|
|||
.customExpressionValueType());
|
||||
}
|
||||
|
||||
Kind kind()
|
||||
protected Kind kind()
|
||||
{
|
||||
return Kind.CUSTOM;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -665,6 +665,11 @@ public class SecondaryIndexManager implements IndexRegistry
|
|||
return selected;
|
||||
}
|
||||
|
||||
public Optional<Index> getBestIndexFor(RowFilter.Expression expression)
|
||||
{
|
||||
return indexes.values().stream().filter((i) -> i.supportsExpression(expression.column(), expression.operator())).findFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Called at write time to ensure that values present in the update
|
||||
* are valid according to the rules of all registered indexes which
|
||||
|
|
@ -1040,6 +1045,12 @@ public class SecondaryIndexManager implements IndexRegistry
|
|||
|
||||
private static void executeAllBlocking(Stream<Index> indexers, Function<Index, Callable<?>> function)
|
||||
{
|
||||
if (function == null)
|
||||
{
|
||||
logger.error("failed to flush indexes: {} because flush task is missing.", indexers);
|
||||
return;
|
||||
}
|
||||
|
||||
List<Future<?>> waitFor = new ArrayList<>();
|
||||
indexers.forEach(indexer -> {
|
||||
Callable<?> task = function.apply(indexer);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,288 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.function.BiFunction;
|
||||
|
||||
import com.googlecode.concurrenttrees.common.Iterables;
|
||||
import org.apache.cassandra.config.CFMetaData;
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.config.Schema;
|
||||
import org.apache.cassandra.cql3.Operator;
|
||||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.compaction.OperationType;
|
||||
import org.apache.cassandra.db.filter.RowFilter;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.partitions.PartitionIterator;
|
||||
import org.apache.cassandra.db.partitions.PartitionUpdate;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.exceptions.InvalidRequestException;
|
||||
import org.apache.cassandra.index.Index;
|
||||
import org.apache.cassandra.index.IndexRegistry;
|
||||
import org.apache.cassandra.index.SecondaryIndexBuilder;
|
||||
import org.apache.cassandra.index.internal.CassandraIndex;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter;
|
||||
import org.apache.cassandra.index.sasi.plan.QueryPlan;
|
||||
import org.apache.cassandra.index.transactions.IndexTransaction;
|
||||
import org.apache.cassandra.io.sstable.Descriptor;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.notifications.*;
|
||||
import org.apache.cassandra.schema.IndexMetadata;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.concurrent.OpOrder;
|
||||
|
||||
public class SASIIndex implements Index, INotificationConsumer
|
||||
{
|
||||
private static class SASIIndexBuildingSupport implements IndexBuildingSupport
|
||||
{
|
||||
public SecondaryIndexBuilder getIndexBuildTask(ColumnFamilyStore cfs,
|
||||
Set<Index> indexes,
|
||||
Collection<SSTableReader> sstablesToRebuild)
|
||||
{
|
||||
NavigableMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables = new TreeMap<>((a, b) -> {
|
||||
return Integer.compare(a.descriptor.generation, b.descriptor.generation);
|
||||
});
|
||||
|
||||
indexes.stream()
|
||||
.filter((i) -> i instanceof SASIIndex)
|
||||
.forEach((i) -> {
|
||||
SASIIndex sasi = (SASIIndex) i;
|
||||
sstablesToRebuild.stream()
|
||||
.filter((sstable) -> !sasi.index.hasSSTable(sstable))
|
||||
.forEach((sstable) -> {
|
||||
Map<ColumnDefinition, ColumnIndex> toBuild = sstables.get(sstable);
|
||||
if (toBuild == null)
|
||||
sstables.put(sstable, (toBuild = new HashMap<>()));
|
||||
|
||||
toBuild.put(sasi.index.getDefinition(), sasi.index);
|
||||
});
|
||||
});
|
||||
|
||||
return new SASIIndexBuilder(cfs, sstables);
|
||||
}
|
||||
}
|
||||
|
||||
private static final SASIIndexBuildingSupport INDEX_BUILDER_SUPPORT = new SASIIndexBuildingSupport();
|
||||
|
||||
private final ColumnFamilyStore baseCfs;
|
||||
private final IndexMetadata config;
|
||||
private final ColumnIndex index;
|
||||
|
||||
public SASIIndex(ColumnFamilyStore baseCfs, IndexMetadata config)
|
||||
{
|
||||
this.baseCfs = baseCfs;
|
||||
this.config = config;
|
||||
|
||||
ColumnDefinition column = CassandraIndex.parseTarget(baseCfs.metadata, config).left;
|
||||
this.index = new ColumnIndex(baseCfs.metadata.getKeyValidator(), column, config);
|
||||
|
||||
baseCfs.getTracker().subscribe(this);
|
||||
}
|
||||
|
||||
public static Map<String, String> validateOptions(Map<String, String> options)
|
||||
{
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
|
||||
public void register(IndexRegistry registry)
|
||||
{
|
||||
registry.registerIndex(this);
|
||||
}
|
||||
|
||||
public IndexMetadata getIndexMetadata()
|
||||
{
|
||||
return config;
|
||||
}
|
||||
|
||||
public Callable<?> getInitializationTask()
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
public Callable<?> getMetadataReloadTask(IndexMetadata indexMetadata)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
public Callable<?> getBlockingFlushTask()
|
||||
{
|
||||
return null; // SASI indexes are flushed along side memtable
|
||||
}
|
||||
|
||||
public Callable<?> getInvalidateTask()
|
||||
{
|
||||
return getTruncateTask(FBUtilities.timestampMicros());
|
||||
}
|
||||
|
||||
public Callable<?> getTruncateTask(long truncatedAt)
|
||||
{
|
||||
return () -> {
|
||||
index.dropData(truncatedAt);
|
||||
return null;
|
||||
};
|
||||
}
|
||||
|
||||
public boolean shouldBuildBlocking()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public Optional<ColumnFamilyStore> getBackingTable()
|
||||
{
|
||||
return Optional.empty();
|
||||
}
|
||||
|
||||
public boolean indexes(PartitionColumns columns)
|
||||
{
|
||||
return columns.contains(index.getDefinition());
|
||||
}
|
||||
|
||||
public boolean dependsOn(ColumnDefinition column)
|
||||
{
|
||||
return index.getDefinition().compareTo(column) == 0;
|
||||
}
|
||||
|
||||
public boolean supportsExpression(ColumnDefinition column, Operator operator)
|
||||
{
|
||||
return dependsOn(column);
|
||||
}
|
||||
|
||||
public AbstractType<?> customExpressionValueType()
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
public RowFilter getPostIndexQueryFilter(RowFilter filter)
|
||||
{
|
||||
return filter.withoutExpressions();
|
||||
}
|
||||
|
||||
public long getEstimatedResultRows()
|
||||
{
|
||||
// this is temporary (until proper QueryPlan is integrated into Cassandra)
|
||||
// and allows us to priority SASI indexes if any in the query since they
|
||||
// are going to be more efficient, to query and intersect, than built-in indexes.
|
||||
return Long.MIN_VALUE;
|
||||
}
|
||||
|
||||
public void validate(PartitionUpdate update) throws InvalidRequestException
|
||||
{}
|
||||
|
||||
public Indexer indexerFor(DecoratedKey key, PartitionColumns columns, int nowInSec, OpOrder.Group opGroup, IndexTransaction.Type transactionType)
|
||||
{
|
||||
return new Indexer()
|
||||
{
|
||||
public void begin()
|
||||
{}
|
||||
|
||||
public void partitionDelete(DeletionTime deletionTime)
|
||||
{}
|
||||
|
||||
public void rangeTombstone(RangeTombstone tombstone)
|
||||
{}
|
||||
|
||||
public void insertRow(Row row)
|
||||
{
|
||||
if (isNewData())
|
||||
adjustMemtableSize(index.index(key, row), opGroup);
|
||||
}
|
||||
|
||||
public void updateRow(Row oldRow, Row newRow)
|
||||
{
|
||||
insertRow(newRow);
|
||||
}
|
||||
|
||||
public void removeRow(Row row)
|
||||
{}
|
||||
|
||||
public void finish()
|
||||
{}
|
||||
|
||||
// we are only interested in the data from Memtable
|
||||
// everything else is going to be handled by SSTableWriter observers
|
||||
private boolean isNewData()
|
||||
{
|
||||
return transactionType == IndexTransaction.Type.UPDATE;
|
||||
}
|
||||
|
||||
public void adjustMemtableSize(long additionalSpace, OpOrder.Group opGroup)
|
||||
{
|
||||
baseCfs.getTracker().getView().getCurrentMemtable().getAllocator().onHeap().allocate(additionalSpace, opGroup);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public Searcher searcherFor(ReadCommand command) throws InvalidRequestException
|
||||
{
|
||||
CFMetaData config = command.metadata();
|
||||
ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(config.cfId);
|
||||
return controller -> new QueryPlan(cfs, command, DatabaseDescriptor.getRangeRpcTimeout()).execute(controller);
|
||||
}
|
||||
|
||||
public SSTableFlushObserver getFlushObserver(Descriptor descriptor, OperationType opType)
|
||||
{
|
||||
return newWriter(baseCfs.metadata.getKeyValidator(), descriptor, Collections.singletonMap(index.getDefinition(), index), opType);
|
||||
}
|
||||
|
||||
public BiFunction<PartitionIterator, ReadCommand, PartitionIterator> postProcessorFor(ReadCommand command)
|
||||
{
|
||||
return (partitionIterator, readCommand) -> partitionIterator;
|
||||
}
|
||||
|
||||
public IndexBuildingSupport getBuildTaskSupport()
|
||||
{
|
||||
return INDEX_BUILDER_SUPPORT;
|
||||
}
|
||||
|
||||
public void handleNotification(INotification notification, Object sender)
|
||||
{
|
||||
// unfortunately, we can only check the type of notification via instanceof :(
|
||||
if (notification instanceof SSTableAddedNotification)
|
||||
{
|
||||
SSTableAddedNotification notice = (SSTableAddedNotification) notification;
|
||||
index.update(Collections.<SSTableReader>emptyList(), Iterables.toList(notice.added));
|
||||
}
|
||||
else if (notification instanceof SSTableListChangedNotification)
|
||||
{
|
||||
SSTableListChangedNotification notice = (SSTableListChangedNotification) notification;
|
||||
index.update(notice.removed, notice.added);
|
||||
}
|
||||
else if (notification instanceof MemtableRenewedNotification)
|
||||
{
|
||||
index.switchMemtable();
|
||||
}
|
||||
}
|
||||
|
||||
public ColumnIndex getIndex()
|
||||
{
|
||||
return index;
|
||||
}
|
||||
|
||||
protected static PerSSTableIndexWriter newWriter(AbstractType<?> keyValidator,
|
||||
Descriptor descriptor,
|
||||
Map<ColumnDefinition, ColumnIndex> indexes,
|
||||
OperationType opType)
|
||||
{
|
||||
return new PerSSTableIndexWriter(keyValidator, descriptor, opType, indexes);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
package org.apache.cassandra.index.sasi;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.RowIndexEntry;
|
||||
import org.apache.cassandra.db.compaction.CompactionInfo;
|
||||
import org.apache.cassandra.db.compaction.CompactionInterruptedException;
|
||||
import org.apache.cassandra.db.compaction.OperationType;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.index.SecondaryIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.PerSSTableIndexWriter;
|
||||
import org.apache.cassandra.io.FSReadError;
|
||||
import org.apache.cassandra.io.sstable.KeyIterator;
|
||||
import org.apache.cassandra.io.sstable.SSTable;
|
||||
import org.apache.cassandra.io.sstable.SSTableIdentityIterator;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.io.util.RandomAccessReader;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.cassandra.utils.UUIDGen;
|
||||
|
||||
class SASIIndexBuilder extends SecondaryIndexBuilder
|
||||
{
|
||||
private final ColumnFamilyStore cfs;
|
||||
private final UUID compactionId = UUIDGen.getTimeUUID();
|
||||
|
||||
private final SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables;
|
||||
|
||||
private long bytesProcessed = 0;
|
||||
private final long totalSizeInBytes;
|
||||
|
||||
public SASIIndexBuilder(ColumnFamilyStore cfs, SortedMap<SSTableReader, Map<ColumnDefinition, ColumnIndex>> sstables)
|
||||
{
|
||||
long totalIndexBytes = 0;
|
||||
for (SSTableReader sstable : sstables.keySet())
|
||||
totalIndexBytes += getPrimaryIndexLength(sstable);
|
||||
|
||||
this.cfs = cfs;
|
||||
this.sstables = sstables;
|
||||
this.totalSizeInBytes = totalIndexBytes;
|
||||
}
|
||||
|
||||
public void build()
|
||||
{
|
||||
AbstractType<?> keyValidator = cfs.metadata.getKeyValidator();
|
||||
for (Map.Entry<SSTableReader, Map<ColumnDefinition, ColumnIndex>> e : sstables.entrySet())
|
||||
{
|
||||
SSTableReader sstable = e.getKey();
|
||||
Map<ColumnDefinition, ColumnIndex> indexes = e.getValue();
|
||||
|
||||
try (RandomAccessReader dataFile = sstable.openDataReader())
|
||||
{
|
||||
PerSSTableIndexWriter indexWriter = SASIIndex.newWriter(keyValidator, sstable.descriptor, indexes, OperationType.COMPACTION);
|
||||
|
||||
long previousKeyPosition = 0;
|
||||
try (KeyIterator keys = new KeyIterator(sstable.descriptor, cfs.metadata))
|
||||
{
|
||||
while (keys.hasNext())
|
||||
{
|
||||
if (isStopRequested())
|
||||
throw new CompactionInterruptedException(getCompactionInfo());
|
||||
|
||||
final DecoratedKey key = keys.next();
|
||||
final long keyPosition = keys.getKeyPosition();
|
||||
|
||||
indexWriter.startPartition(key, keyPosition);
|
||||
|
||||
try
|
||||
{
|
||||
RowIndexEntry indexEntry = sstable.getPosition(key, SSTableReader.Operator.EQ);
|
||||
dataFile.seek(indexEntry.position + indexEntry.headerOffset());
|
||||
ByteBufferUtil.readWithShortLength(dataFile); // key
|
||||
|
||||
try (SSTableIdentityIterator partition = new SSTableIdentityIterator(sstable, dataFile, key))
|
||||
{
|
||||
while (partition.hasNext())
|
||||
indexWriter.nextUnfilteredCluster(partition.next());
|
||||
}
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
throw new FSReadError(ex, sstable.getFilename());
|
||||
}
|
||||
|
||||
bytesProcessed += keyPosition - previousKeyPosition;
|
||||
previousKeyPosition = keyPosition;
|
||||
}
|
||||
|
||||
completeSSTable(indexWriter, sstable, indexes.values());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public CompactionInfo getCompactionInfo()
|
||||
{
|
||||
return new CompactionInfo(cfs.metadata,
|
||||
OperationType.INDEX_BUILD,
|
||||
bytesProcessed,
|
||||
totalSizeInBytes,
|
||||
compactionId);
|
||||
}
|
||||
|
||||
private long getPrimaryIndexLength(SSTable sstable)
|
||||
{
|
||||
File primaryIndex = new File(sstable.getIndexFilename());
|
||||
return primaryIndex.exists() ? primaryIndex.length() : 0;
|
||||
}
|
||||
|
||||
private void completeSSTable(PerSSTableIndexWriter indexWriter, SSTableReader sstable, Collection<ColumnIndex> indexes)
|
||||
{
|
||||
indexWriter.complete();
|
||||
|
||||
for (ColumnIndex index : indexes)
|
||||
{
|
||||
File tmpIndex = new File(sstable.descriptor.filenameFor(index.getComponent()));
|
||||
if (!tmpIndex.exists()) // no data was inserted into the index for given sstable
|
||||
continue;
|
||||
|
||||
index.update(Collections.<SSTableReader>emptyList(), Collections.singletonList(sstable));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,187 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.io.FSReadError;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.utils.concurrent.Ref;
|
||||
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
|
||||
public class SSTableIndex
|
||||
{
|
||||
private final ColumnIndex columnIndex;
|
||||
private final Ref<SSTableReader> sstableRef;
|
||||
private final SSTableReader sstable;
|
||||
private final OnDiskIndex index;
|
||||
private final AtomicInteger references = new AtomicInteger(1);
|
||||
private final AtomicBoolean obsolete = new AtomicBoolean(false);
|
||||
|
||||
public SSTableIndex(ColumnIndex index, File indexFile, SSTableReader referent)
|
||||
{
|
||||
this.columnIndex = index;
|
||||
this.sstableRef = referent.tryRef();
|
||||
this.sstable = sstableRef.get();
|
||||
|
||||
if (sstable == null)
|
||||
throw new IllegalStateException("Couldn't acquire reference to the sstable: " + referent);
|
||||
|
||||
AbstractType<?> validator = columnIndex.getValidator();
|
||||
|
||||
assert validator != null;
|
||||
assert indexFile.exists() : String.format("SSTable %s should have index %s.",
|
||||
sstable.getFilename(),
|
||||
columnIndex.getIndexName());
|
||||
|
||||
this.index = new OnDiskIndex(indexFile, validator, new DecoratedKeyFetcher(sstable));
|
||||
}
|
||||
|
||||
public ByteBuffer minTerm()
|
||||
{
|
||||
return index.minTerm();
|
||||
}
|
||||
|
||||
public ByteBuffer maxTerm()
|
||||
{
|
||||
return index.maxTerm();
|
||||
}
|
||||
|
||||
public ByteBuffer minKey()
|
||||
{
|
||||
return index.minKey();
|
||||
}
|
||||
|
||||
public ByteBuffer maxKey()
|
||||
{
|
||||
return index.maxKey();
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> search(Expression expression)
|
||||
{
|
||||
return index.search(expression);
|
||||
}
|
||||
|
||||
public SSTableReader getSSTable()
|
||||
{
|
||||
return sstable;
|
||||
}
|
||||
|
||||
public String getPath()
|
||||
{
|
||||
return index.getIndexPath();
|
||||
}
|
||||
|
||||
public boolean reference()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
int n = references.get();
|
||||
if (n <= 0)
|
||||
return false;
|
||||
if (references.compareAndSet(n, n + 1))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public void release()
|
||||
{
|
||||
int n = references.decrementAndGet();
|
||||
if (n == 0)
|
||||
{
|
||||
FileUtils.closeQuietly(index);
|
||||
sstableRef.release();
|
||||
if (obsolete.get() || sstableRef.globalCount() == 0)
|
||||
FileUtils.delete(index.getIndexPath());
|
||||
}
|
||||
}
|
||||
|
||||
public void markObsolete()
|
||||
{
|
||||
obsolete.getAndSet(true);
|
||||
release();
|
||||
}
|
||||
|
||||
public boolean isObsolete()
|
||||
{
|
||||
return obsolete.get();
|
||||
}
|
||||
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
return o instanceof SSTableIndex && index.getIndexPath().equals(((SSTableIndex) o).index.getIndexPath());
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return new HashCodeBuilder().append(index.getIndexPath()).build();
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return String.format("SSTableIndex(column: %s, SSTable: %s)", columnIndex.getColumnName(), sstable.descriptor);
|
||||
}
|
||||
|
||||
private static class DecoratedKeyFetcher implements Function<Long, DecoratedKey>
|
||||
{
|
||||
private final SSTableReader sstable;
|
||||
|
||||
DecoratedKeyFetcher(SSTableReader reader)
|
||||
{
|
||||
sstable = reader;
|
||||
}
|
||||
|
||||
public DecoratedKey apply(Long offset)
|
||||
{
|
||||
try
|
||||
{
|
||||
return sstable.keyAt(offset);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new FSReadError(new IOException("Failed to read key from " + sstable.descriptor, e), sstable.getFilename());
|
||||
}
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return sstable.descriptor.hashCode();
|
||||
}
|
||||
|
||||
public boolean equals(Object other)
|
||||
{
|
||||
return other instanceof DecoratedKeyFetcher
|
||||
&& sstable.descriptor.equals(((DecoratedKeyFetcher) other).sstable.descriptor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder.TermSize;
|
||||
import org.apache.cassandra.index.sasi.utils.MappedBuffer;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public class Term
|
||||
{
|
||||
protected final MappedBuffer content;
|
||||
protected final TermSize termSize;
|
||||
|
||||
|
||||
public Term(MappedBuffer content, TermSize size)
|
||||
{
|
||||
this.content = content;
|
||||
this.termSize = size;
|
||||
}
|
||||
|
||||
public ByteBuffer getTerm()
|
||||
{
|
||||
long offset = termSize.isConstant() ? content.position() : content.position() + 2;
|
||||
int length = termSize.isConstant() ? termSize.size : content.getShort(content.position());
|
||||
|
||||
return content.getPageRegion(offset, length);
|
||||
}
|
||||
|
||||
public long getDataOffset()
|
||||
{
|
||||
long position = content.position();
|
||||
return position + (termSize.isConstant() ? termSize.size : 2 + content.getShort(position));
|
||||
}
|
||||
|
||||
public int compareTo(AbstractType<?> comparator, ByteBuffer query)
|
||||
{
|
||||
return compareTo(comparator, query, true);
|
||||
}
|
||||
|
||||
public int compareTo(AbstractType<?> comparator, ByteBuffer query, boolean checkFully)
|
||||
{
|
||||
long position = content.position();
|
||||
int padding = termSize.isConstant() ? 0 : 2;
|
||||
int len = termSize.isConstant() ? termSize.size : content.getShort(position);
|
||||
|
||||
return content.comparePageTo(position + padding, checkFully ? len : Math.min(len, query.remaining()), comparator, query);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import com.google.common.util.concurrent.Uninterruptibles;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class TermIterator extends RangeIterator<Long, Token>
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(TermIterator.class);
|
||||
|
||||
private static final ThreadLocal<ExecutorService> SEARCH_EXECUTOR = new ThreadLocal<ExecutorService>()
|
||||
{
|
||||
public ExecutorService initialValue()
|
||||
{
|
||||
final String currentThread = Thread.currentThread().getName();
|
||||
final int concurrencyFactor = DatabaseDescriptor.searchConcurrencyFactor();
|
||||
|
||||
logger.info("Search Concurrency Factor is set to {} for {}", concurrencyFactor, currentThread);
|
||||
|
||||
return (concurrencyFactor <= 1)
|
||||
? MoreExecutors.newDirectExecutorService()
|
||||
: Executors.newFixedThreadPool(concurrencyFactor, new ThreadFactory()
|
||||
{
|
||||
public final AtomicInteger count = new AtomicInteger();
|
||||
|
||||
public Thread newThread(Runnable task)
|
||||
{
|
||||
return new Thread(task, currentThread + "-SEARCH-" + count.incrementAndGet()) {{ setDaemon(true); }};
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
private final Expression expression;
|
||||
|
||||
private final RangeIterator<Long, Token> union;
|
||||
private final Set<SSTableIndex> referencedIndexes;
|
||||
|
||||
private TermIterator(Expression e,
|
||||
RangeIterator<Long, Token> union,
|
||||
Set<SSTableIndex> referencedIndexes)
|
||||
{
|
||||
super(union.getMinimum(), union.getMaximum(), union.getCount());
|
||||
|
||||
this.expression = e;
|
||||
this.union = union;
|
||||
this.referencedIndexes = referencedIndexes;
|
||||
}
|
||||
|
||||
public static TermIterator build(final Expression e, Set<SSTableIndex> perSSTableIndexes)
|
||||
{
|
||||
final List<RangeIterator<Long, Token>> tokens = new CopyOnWriteArrayList<>();
|
||||
final AtomicLong tokenCount = new AtomicLong(0);
|
||||
|
||||
RangeIterator<Long, Token> memtableIterator = e.index.searchMemtable(e);
|
||||
if (memtableIterator != null)
|
||||
{
|
||||
tokens.add(memtableIterator);
|
||||
tokenCount.addAndGet(memtableIterator.getCount());
|
||||
}
|
||||
|
||||
final Set<SSTableIndex> referencedIndexes = new CopyOnWriteArraySet<>();
|
||||
|
||||
try
|
||||
{
|
||||
final CountDownLatch latch = new CountDownLatch(perSSTableIndexes.size());
|
||||
final ExecutorService searchExecutor = SEARCH_EXECUTOR.get();
|
||||
|
||||
for (final SSTableIndex index : perSSTableIndexes)
|
||||
{
|
||||
if (!index.reference())
|
||||
{
|
||||
latch.countDown();
|
||||
continue;
|
||||
}
|
||||
|
||||
// add to referenced right after the reference was acquired,
|
||||
// that helps to release index if something goes bad inside of the search
|
||||
referencedIndexes.add(index);
|
||||
|
||||
searchExecutor.submit((Runnable) () -> {
|
||||
try
|
||||
{
|
||||
e.checkpoint();
|
||||
|
||||
RangeIterator<Long, Token> keyIterator = index.search(e);
|
||||
if (keyIterator == null)
|
||||
{
|
||||
releaseIndex(referencedIndexes, index);
|
||||
return;
|
||||
}
|
||||
|
||||
tokens.add(keyIterator);
|
||||
tokenCount.getAndAdd(keyIterator.getCount());
|
||||
}
|
||||
catch (Throwable e1)
|
||||
{
|
||||
releaseIndex(referencedIndexes, index);
|
||||
|
||||
if (logger.isDebugEnabled())
|
||||
logger.debug(String.format("Failed search an index %s, skipping.", index.getPath()), e1);
|
||||
}
|
||||
finally
|
||||
{
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Uninterruptibles.awaitUninterruptibly(latch);
|
||||
|
||||
// checkpoint right away after all indexes complete search because we might have crossed the quota
|
||||
e.checkpoint();
|
||||
|
||||
RangeIterator<Long, Token> ranges = RangeUnionIterator.build(tokens);
|
||||
return ranges == null ? null : new TermIterator(e, ranges, referencedIndexes);
|
||||
}
|
||||
catch (Throwable ex)
|
||||
{
|
||||
// if execution quota was exceeded while opening indexes or something else happened
|
||||
// local (yet to be tracked) indexes should be released first before re-throwing exception
|
||||
referencedIndexes.forEach(TermIterator::releaseQuietly);
|
||||
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
|
||||
protected Token computeNext()
|
||||
{
|
||||
try
|
||||
{
|
||||
return union.hasNext() ? union.next() : endOfData();
|
||||
}
|
||||
finally
|
||||
{
|
||||
expression.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
protected void performSkipTo(Long nextToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
union.skipTo(nextToken);
|
||||
}
|
||||
finally
|
||||
{
|
||||
expression.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
public void close()
|
||||
{
|
||||
FileUtils.closeQuietly(union);
|
||||
referencedIndexes.forEach(TermIterator::releaseQuietly);
|
||||
referencedIndexes.clear();
|
||||
}
|
||||
|
||||
private static void releaseIndex(Set<SSTableIndex> indexes, SSTableIndex index)
|
||||
{
|
||||
indexes.remove(index);
|
||||
releaseQuietly(index);
|
||||
}
|
||||
|
||||
private static void releaseQuietly(SSTableIndex index)
|
||||
{
|
||||
try
|
||||
{
|
||||
index.release();
|
||||
}
|
||||
catch (Throwable e)
|
||||
{
|
||||
logger.error(String.format("Failed to release index %s", index.getPath()), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.text.Normalizer;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public abstract class AbstractAnalyzer implements Iterator<ByteBuffer>
|
||||
{
|
||||
protected ByteBuffer next = null;
|
||||
|
||||
public ByteBuffer next()
|
||||
{
|
||||
return next;
|
||||
}
|
||||
|
||||
public void remove()
|
||||
{
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public abstract void init(Map<String, String> options, AbstractType validator);
|
||||
|
||||
public abstract void reset(ByteBuffer input);
|
||||
|
||||
public static String normalize(String original)
|
||||
{
|
||||
return Normalizer.isNormalized(original, Normalizer.Form.NFC)
|
||||
? original
|
||||
: Normalizer.normalize(original, Normalizer.Form.NFC);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
/**
|
||||
* Default noOp tokenizer. The iterator will iterate only once
|
||||
* returning the unmodified input
|
||||
*/
|
||||
public class NoOpAnalyzer extends AbstractAnalyzer
|
||||
{
|
||||
private ByteBuffer input;
|
||||
private boolean hasNext = false;
|
||||
|
||||
public void init(Map<String, String> options, AbstractType validator)
|
||||
{}
|
||||
|
||||
public boolean hasNext()
|
||||
{
|
||||
if (hasNext)
|
||||
{
|
||||
this.next = input;
|
||||
this.hasNext = false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void reset(ByteBuffer input)
|
||||
{
|
||||
this.next = null;
|
||||
this.input = input;
|
||||
this.hasNext = true;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.index.sasi.analyzer.filter.BasicResultFilters;
|
||||
import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineBuilder;
|
||||
import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineExecutor;
|
||||
import org.apache.cassandra.index.sasi.analyzer.filter.FilterPipelineTask;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.AsciiType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.serializers.MarshalException;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Analyzer that does *not* tokenize the input. Optionally will
|
||||
* apply filters for the input output as defined in analyzers options
|
||||
*/
|
||||
public class NonTokenizingAnalyzer extends AbstractAnalyzer
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(NonTokenizingAnalyzer.class);
|
||||
|
||||
private static final Set<AbstractType<?>> VALID_ANALYZABLE_TYPES = new HashSet<AbstractType<?>>()
|
||||
{{
|
||||
add(UTF8Type.instance);
|
||||
add(AsciiType.instance);
|
||||
}};
|
||||
|
||||
private AbstractType validator;
|
||||
private NonTokenizingOptions options;
|
||||
private FilterPipelineTask filterPipeline;
|
||||
|
||||
private ByteBuffer input;
|
||||
private boolean hasNext = false;
|
||||
|
||||
public void init(Map<String, String> options, AbstractType validator)
|
||||
{
|
||||
init(NonTokenizingOptions.buildFromMap(options), validator);
|
||||
}
|
||||
|
||||
public void init(NonTokenizingOptions tokenizerOptions, AbstractType validator)
|
||||
{
|
||||
this.validator = validator;
|
||||
this.options = tokenizerOptions;
|
||||
this.filterPipeline = getFilterPipeline();
|
||||
}
|
||||
|
||||
public boolean hasNext()
|
||||
{
|
||||
// check that we know how to handle the input, otherwise bail
|
||||
if (!VALID_ANALYZABLE_TYPES.contains(validator))
|
||||
return false;
|
||||
|
||||
if (hasNext)
|
||||
{
|
||||
String inputStr;
|
||||
|
||||
try
|
||||
{
|
||||
inputStr = validator.getString(input);
|
||||
if (inputStr == null)
|
||||
throw new MarshalException(String.format("'null' deserialized value for %s with %s", ByteBufferUtil.bytesToHex(input), validator));
|
||||
|
||||
Object pipelineRes = FilterPipelineExecutor.execute(filterPipeline, inputStr);
|
||||
if (pipelineRes == null)
|
||||
return false;
|
||||
|
||||
next = validator.fromString(normalize((String) pipelineRes));
|
||||
return true;
|
||||
}
|
||||
catch (MarshalException e)
|
||||
{
|
||||
logger.error("Failed to deserialize value with " + validator, e);
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
hasNext = false;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void reset(ByteBuffer input)
|
||||
{
|
||||
this.next = null;
|
||||
this.input = input;
|
||||
this.hasNext = true;
|
||||
}
|
||||
|
||||
private FilterPipelineTask getFilterPipeline()
|
||||
{
|
||||
FilterPipelineBuilder builder = new FilterPipelineBuilder(new BasicResultFilters.NoOperation());
|
||||
if (options.isCaseSensitive() && options.shouldLowerCaseOutput())
|
||||
builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
|
||||
if (options.isCaseSensitive() && options.shouldUpperCaseOutput())
|
||||
builder = builder.add("to_upper", new BasicResultFilters.UpperCase());
|
||||
if (!options.isCaseSensitive())
|
||||
builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,147 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
public class NonTokenizingOptions
|
||||
{
|
||||
public static final String NORMALIZE_LOWERCASE = "normalize_lowercase";
|
||||
public static final String NORMALIZE_UPPERCASE = "normalize_uppercase";
|
||||
public static final String CASE_SENSITIVE = "case_sensitive";
|
||||
|
||||
private boolean caseSensitive;
|
||||
private boolean upperCaseOutput;
|
||||
private boolean lowerCaseOutput;
|
||||
|
||||
public boolean isCaseSensitive()
|
||||
{
|
||||
return caseSensitive;
|
||||
}
|
||||
|
||||
public void setCaseSensitive(boolean caseSensitive)
|
||||
{
|
||||
this.caseSensitive = caseSensitive;
|
||||
}
|
||||
|
||||
public boolean shouldUpperCaseOutput()
|
||||
{
|
||||
return upperCaseOutput;
|
||||
}
|
||||
|
||||
public void setUpperCaseOutput(boolean upperCaseOutput)
|
||||
{
|
||||
this.upperCaseOutput = upperCaseOutput;
|
||||
}
|
||||
|
||||
public boolean shouldLowerCaseOutput()
|
||||
{
|
||||
return lowerCaseOutput;
|
||||
}
|
||||
|
||||
public void setLowerCaseOutput(boolean lowerCaseOutput)
|
||||
{
|
||||
this.lowerCaseOutput = lowerCaseOutput;
|
||||
}
|
||||
|
||||
public static class OptionsBuilder
|
||||
{
|
||||
private boolean caseSensitive = true;
|
||||
private boolean upperCaseOutput = false;
|
||||
private boolean lowerCaseOutput = false;
|
||||
|
||||
public OptionsBuilder()
|
||||
{
|
||||
}
|
||||
|
||||
public OptionsBuilder caseSensitive(boolean caseSensitive)
|
||||
{
|
||||
this.caseSensitive = caseSensitive;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder upperCaseOutput(boolean upperCaseOutput)
|
||||
{
|
||||
this.upperCaseOutput = upperCaseOutput;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder lowerCaseOutput(boolean lowerCaseOutput)
|
||||
{
|
||||
this.lowerCaseOutput = lowerCaseOutput;
|
||||
return this;
|
||||
}
|
||||
|
||||
public NonTokenizingOptions build()
|
||||
{
|
||||
if (lowerCaseOutput && upperCaseOutput)
|
||||
throw new IllegalArgumentException("Options to normalize terms cannot be " +
|
||||
"both uppercase and lowercase at the same time");
|
||||
|
||||
NonTokenizingOptions options = new NonTokenizingOptions();
|
||||
options.setCaseSensitive(caseSensitive);
|
||||
options.setUpperCaseOutput(upperCaseOutput);
|
||||
options.setLowerCaseOutput(lowerCaseOutput);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
public static NonTokenizingOptions buildFromMap(Map<String, String> optionsMap)
|
||||
{
|
||||
OptionsBuilder optionsBuilder = new OptionsBuilder();
|
||||
|
||||
if (optionsMap.containsKey(CASE_SENSITIVE) && (optionsMap.containsKey(NORMALIZE_LOWERCASE)
|
||||
|| optionsMap.containsKey(NORMALIZE_UPPERCASE)))
|
||||
throw new IllegalArgumentException("case_sensitive option cannot be specified together " +
|
||||
"with either normalize_lowercase or normalize_uppercase");
|
||||
|
||||
for (Map.Entry<String, String> entry : optionsMap.entrySet())
|
||||
{
|
||||
switch (entry.getKey())
|
||||
{
|
||||
case NORMALIZE_LOWERCASE:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.lowerCaseOutput(bool);
|
||||
break;
|
||||
}
|
||||
case NORMALIZE_UPPERCASE:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.upperCaseOutput(bool);
|
||||
break;
|
||||
}
|
||||
case CASE_SENSITIVE:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.caseSensitive(bool);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return optionsBuilder.build();
|
||||
}
|
||||
|
||||
public static NonTokenizingOptions getDefaultOptions()
|
||||
{
|
||||
return new OptionsBuilder()
|
||||
.caseSensitive(true).lowerCaseOutput(false)
|
||||
.upperCaseOutput(false)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
// Generated using ICU4J 52.1.0.0
|
||||
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
|
||||
|
||||
|
||||
ALetterSupp = (
|
||||
([\ud83b][\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB])
|
||||
| ([\ud81a][\uDC00-\uDE38])
|
||||
| ([\ud81b][\uDF00-\uDF44\uDF50\uDF93-\uDF9F])
|
||||
| ([\ud835][\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB])
|
||||
| ([\ud80d][\uDC00-\uDC2E])
|
||||
| ([\ud80c][\uDC00-\uDFFF])
|
||||
| ([\ud809][\uDC00-\uDC62])
|
||||
| ([\ud808][\uDC00-\uDF6E])
|
||||
| ([\ud805][\uDE80-\uDEAA])
|
||||
| ([\ud804][\uDC03-\uDC37\uDC83-\uDCAF\uDCD0-\uDCE8\uDD03-\uDD26\uDD83-\uDDB2\uDDC1-\uDDC4])
|
||||
| ([\ud801][\uDC00-\uDC9D])
|
||||
| ([\ud800][\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD40-\uDD74\uDE80-\uDE9C\uDEA0-\uDED0\uDF00-\uDF1E\uDF30-\uDF4A\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5])
|
||||
| ([\ud803][\uDC00-\uDC48])
|
||||
| ([\ud802][\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDD00-\uDD15\uDD20-\uDD39\uDD80-\uDDB7\uDDBE\uDDBF\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE33\uDE60-\uDE7C\uDF00-\uDF35\uDF40-\uDF55\uDF60-\uDF72])
|
||||
)
|
||||
FormatSupp = (
|
||||
([\ud804][\uDCBD])
|
||||
| ([\ud834][\uDD73-\uDD7A])
|
||||
| ([\udb40][\uDC01\uDC20-\uDC7F])
|
||||
)
|
||||
NumericSupp = (
|
||||
([\ud805][\uDEC0-\uDEC9])
|
||||
| ([\ud804][\uDC66-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9])
|
||||
| ([\ud835][\uDFCE-\uDFFF])
|
||||
| ([\ud801][\uDCA0-\uDCA9])
|
||||
)
|
||||
ExtendSupp = (
|
||||
([\ud81b][\uDF51-\uDF7E\uDF8F-\uDF92])
|
||||
| ([\ud805][\uDEAB-\uDEB7])
|
||||
| ([\ud804][\uDC00-\uDC02\uDC38-\uDC46\uDC80-\uDC82\uDCB0-\uDCBA\uDD00-\uDD02\uDD27-\uDD34\uDD80-\uDD82\uDDB3-\uDDC0])
|
||||
| ([\ud834][\uDD65-\uDD69\uDD6D-\uDD72\uDD7B-\uDD82\uDD85-\uDD8B\uDDAA-\uDDAD\uDE42-\uDE44])
|
||||
| ([\ud800][\uDDFD])
|
||||
| ([\udb40][\uDD00-\uDDEF])
|
||||
| ([\ud802][\uDE01-\uDE03\uDE05\uDE06\uDE0C-\uDE0F\uDE38-\uDE3A\uDE3F])
|
||||
)
|
||||
KatakanaSupp = (
|
||||
([\ud82c][\uDC00])
|
||||
)
|
||||
MidLetterSupp = (
|
||||
[]
|
||||
)
|
||||
MidNumSupp = (
|
||||
[]
|
||||
)
|
||||
MidNumLetSupp = (
|
||||
[]
|
||||
)
|
||||
ExtendNumLetSupp = (
|
||||
[]
|
||||
)
|
||||
ExtendNumLetSupp = (
|
||||
[]
|
||||
)
|
||||
ComplexContextSupp = (
|
||||
[]
|
||||
)
|
||||
HanSupp = (
|
||||
([\ud87e][\uDC00-\uDE1D])
|
||||
| ([\ud86b][\uDC00-\uDFFF])
|
||||
| ([\ud86a][\uDC00-\uDFFF])
|
||||
| ([\ud869][\uDC00-\uDED6\uDF00-\uDFFF])
|
||||
| ([\ud868][\uDC00-\uDFFF])
|
||||
| ([\ud86e][\uDC00-\uDC1D])
|
||||
| ([\ud86d][\uDC00-\uDF34\uDF40-\uDFFF])
|
||||
| ([\ud86c][\uDC00-\uDFFF])
|
||||
| ([\ud863][\uDC00-\uDFFF])
|
||||
| ([\ud862][\uDC00-\uDFFF])
|
||||
| ([\ud861][\uDC00-\uDFFF])
|
||||
| ([\ud860][\uDC00-\uDFFF])
|
||||
| ([\ud867][\uDC00-\uDFFF])
|
||||
| ([\ud866][\uDC00-\uDFFF])
|
||||
| ([\ud865][\uDC00-\uDFFF])
|
||||
| ([\ud864][\uDC00-\uDFFF])
|
||||
| ([\ud858][\uDC00-\uDFFF])
|
||||
| ([\ud859][\uDC00-\uDFFF])
|
||||
| ([\ud85a][\uDC00-\uDFFF])
|
||||
| ([\ud85b][\uDC00-\uDFFF])
|
||||
| ([\ud85c][\uDC00-\uDFFF])
|
||||
| ([\ud85d][\uDC00-\uDFFF])
|
||||
| ([\ud85e][\uDC00-\uDFFF])
|
||||
| ([\ud85f][\uDC00-\uDFFF])
|
||||
| ([\ud850][\uDC00-\uDFFF])
|
||||
| ([\ud851][\uDC00-\uDFFF])
|
||||
| ([\ud852][\uDC00-\uDFFF])
|
||||
| ([\ud853][\uDC00-\uDFFF])
|
||||
| ([\ud854][\uDC00-\uDFFF])
|
||||
| ([\ud855][\uDC00-\uDFFF])
|
||||
| ([\ud856][\uDC00-\uDFFF])
|
||||
| ([\ud857][\uDC00-\uDFFF])
|
||||
| ([\ud849][\uDC00-\uDFFF])
|
||||
| ([\ud848][\uDC00-\uDFFF])
|
||||
| ([\ud84b][\uDC00-\uDFFF])
|
||||
| ([\ud84a][\uDC00-\uDFFF])
|
||||
| ([\ud84d][\uDC00-\uDFFF])
|
||||
| ([\ud84c][\uDC00-\uDFFF])
|
||||
| ([\ud84f][\uDC00-\uDFFF])
|
||||
| ([\ud84e][\uDC00-\uDFFF])
|
||||
| ([\ud841][\uDC00-\uDFFF])
|
||||
| ([\ud840][\uDC00-\uDFFF])
|
||||
| ([\ud843][\uDC00-\uDFFF])
|
||||
| ([\ud842][\uDC00-\uDFFF])
|
||||
| ([\ud845][\uDC00-\uDFFF])
|
||||
| ([\ud844][\uDC00-\uDFFF])
|
||||
| ([\ud847][\uDC00-\uDFFF])
|
||||
| ([\ud846][\uDC00-\uDFFF])
|
||||
)
|
||||
HiraganaSupp = (
|
||||
([\ud83c][\uDE00])
|
||||
| ([\ud82c][\uDC01])
|
||||
)
|
||||
SingleQuoteSupp = (
|
||||
[]
|
||||
)
|
||||
DoubleQuoteSupp = (
|
||||
[]
|
||||
)
|
||||
HebrewLetterSupp = (
|
||||
[]
|
||||
)
|
||||
RegionalIndicatorSupp = (
|
||||
([\ud83c][\uDDE6-\uDDFF])
|
||||
)
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.Reader;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.cassandra.index.sasi.analyzer.filter.*;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.io.util.DataInputBuffer;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import com.carrotsearch.hppc.IntObjectMap;
|
||||
import com.carrotsearch.hppc.IntObjectOpenHashMap;
|
||||
|
||||
public class StandardAnalyzer extends AbstractAnalyzer
|
||||
{
|
||||
public enum TokenType
|
||||
{
|
||||
EOF(-1),
|
||||
ALPHANUM(0),
|
||||
NUM(6),
|
||||
SOUTHEAST_ASIAN(9),
|
||||
IDEOGRAPHIC(10),
|
||||
HIRAGANA(11),
|
||||
KATAKANA(12),
|
||||
HANGUL(13);
|
||||
|
||||
private static final IntObjectMap<TokenType> TOKENS = new IntObjectOpenHashMap<>();
|
||||
|
||||
static
|
||||
{
|
||||
for (TokenType type : TokenType.values())
|
||||
TOKENS.put(type.value, type);
|
||||
}
|
||||
|
||||
public final int value;
|
||||
|
||||
TokenType(int value)
|
||||
{
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public int getValue()
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
public static TokenType fromValue(int val)
|
||||
{
|
||||
return TOKENS.get(val);
|
||||
}
|
||||
}
|
||||
|
||||
private AbstractType validator;
|
||||
|
||||
private StandardTokenizerInterface scanner;
|
||||
private StandardTokenizerOptions options;
|
||||
private FilterPipelineTask filterPipeline;
|
||||
|
||||
protected Reader inputReader = null;
|
||||
|
||||
public String getToken()
|
||||
{
|
||||
return scanner.getText();
|
||||
}
|
||||
|
||||
public final boolean incrementToken() throws IOException
|
||||
{
|
||||
while(true)
|
||||
{
|
||||
TokenType currentTokenType = TokenType.fromValue(scanner.getNextToken());
|
||||
if (currentTokenType == TokenType.EOF)
|
||||
return false;
|
||||
if (scanner.yylength() <= options.getMaxTokenLength()
|
||||
&& scanner.yylength() >= options.getMinTokenLength())
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
protected String getFilteredCurrentToken() throws IOException
|
||||
{
|
||||
String token = getToken();
|
||||
Object pipelineRes;
|
||||
|
||||
while (true)
|
||||
{
|
||||
pipelineRes = FilterPipelineExecutor.execute(filterPipeline, token);
|
||||
if (pipelineRes != null)
|
||||
break;
|
||||
|
||||
boolean reachedEOF = incrementToken();
|
||||
if (!reachedEOF)
|
||||
break;
|
||||
|
||||
token = getToken();
|
||||
}
|
||||
|
||||
return (String) pipelineRes;
|
||||
}
|
||||
|
||||
private FilterPipelineTask getFilterPipeline()
|
||||
{
|
||||
FilterPipelineBuilder builder = new FilterPipelineBuilder(new BasicResultFilters.NoOperation());
|
||||
if (!options.isCaseSensitive() && options.shouldLowerCaseTerms())
|
||||
builder = builder.add("to_lower", new BasicResultFilters.LowerCase());
|
||||
if (!options.isCaseSensitive() && options.shouldUpperCaseTerms())
|
||||
builder = builder.add("to_upper", new BasicResultFilters.UpperCase());
|
||||
if (options.shouldStemTerms())
|
||||
builder = builder.add("term_stemming", new StemmingFilters.DefaultStemmingFilter(options.getLocale()));
|
||||
if (options.shouldIgnoreStopTerms())
|
||||
builder = builder.add("skip_stop_words", new StopWordFilters.DefaultStopWordFilter(options.getLocale()));
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
public void init(Map<String, String> options, AbstractType validator)
|
||||
{
|
||||
init(StandardTokenizerOptions.buildFromMap(options), validator);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected void init(StandardTokenizerOptions options)
|
||||
{
|
||||
init(options, UTF8Type.instance);
|
||||
}
|
||||
|
||||
public void init(StandardTokenizerOptions tokenizerOptions, AbstractType validator)
|
||||
{
|
||||
this.validator = validator;
|
||||
this.options = tokenizerOptions;
|
||||
this.filterPipeline = getFilterPipeline();
|
||||
|
||||
Reader reader = new InputStreamReader(new DataInputBuffer(ByteBufferUtil.EMPTY_BYTE_BUFFER, false));
|
||||
this.scanner = new StandardTokenizerImpl(reader);
|
||||
this.inputReader = reader;
|
||||
}
|
||||
|
||||
public boolean hasNext()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (incrementToken())
|
||||
{
|
||||
if (getFilteredCurrentToken() != null)
|
||||
{
|
||||
this.next = validator.fromString(normalize(getFilteredCurrentToken()));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
{}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void reset(ByteBuffer input)
|
||||
{
|
||||
this.next = null;
|
||||
Reader reader = new InputStreamReader(new DataInputBuffer(input, false));
|
||||
scanner.yyreset(reader);
|
||||
this.inputReader = reader;
|
||||
}
|
||||
|
||||
public void reset(InputStream input)
|
||||
{
|
||||
this.next = null;
|
||||
Reader reader = new InputStreamReader(input);
|
||||
scanner.yyreset(reader);
|
||||
this.inputReader = reader;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,220 @@
|
|||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.util.Arrays;
|
||||
|
||||
/**
|
||||
* This class implements Word Break rules from the Unicode Text Segmentation
|
||||
* algorithm, as specified in
|
||||
* <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>. ∂
|
||||
* <p/>
|
||||
* Tokens produced are of the following types:
|
||||
* <ul>
|
||||
* <li><ALPHANUM>: A sequence of alphabetic and numeric characters</li>
|
||||
* <li><NUM>: A number</li>
|
||||
* <li><SOUTHEAST_ASIAN>: A sequence of characters from South and Southeast
|
||||
* Asian languages, including Thai, Lao, Myanmar, and Khmer</li>
|
||||
* <li><IDEOGRAPHIC>: A single CJKV ideographic character</li>
|
||||
* <li><HIRAGANA>: A single hiragana character</li>
|
||||
* <li><KATAKANA>: A sequence of katakana characters</li>
|
||||
* <li><HANGUL>: A sequence of Hangul characters</li>
|
||||
* </ul>
|
||||
*/
|
||||
%%
|
||||
|
||||
%unicode 6.3
|
||||
%integer
|
||||
%final
|
||||
%public
|
||||
%class StandardTokenizerImpl
|
||||
%implements StandardTokenizerInterface
|
||||
%function getNextToken
|
||||
%char
|
||||
%buffer 4096
|
||||
|
||||
%include SUPPLEMENTARY.jflex-macro
|
||||
ALetter = (\p{WB:ALetter} | {ALetterSupp})
|
||||
Format = (\p{WB:Format} | {FormatSupp})
|
||||
Numeric = ([\p{WB:Numeric}[\p{Blk:HalfAndFullForms}&&\p{Nd}]] | {NumericSupp})
|
||||
Extend = (\p{WB:Extend} | {ExtendSupp})
|
||||
Katakana = (\p{WB:Katakana} | {KatakanaSupp})
|
||||
MidLetter = (\p{WB:MidLetter} | {MidLetterSupp})
|
||||
MidNum = (\p{WB:MidNum} | {MidNumSupp})
|
||||
MidNumLet = (\p{WB:MidNumLet} | {MidNumLetSupp})
|
||||
ExtendNumLet = (\p{WB:ExtendNumLet} | {ExtendNumLetSupp})
|
||||
ComplexContext = (\p{LB:Complex_Context} | {ComplexContextSupp})
|
||||
Han = (\p{Script:Han} | {HanSupp})
|
||||
Hiragana = (\p{Script:Hiragana} | {HiraganaSupp})
|
||||
SingleQuote = (\p{WB:Single_Quote} | {SingleQuoteSupp})
|
||||
DoubleQuote = (\p{WB:Double_Quote} | {DoubleQuoteSupp})
|
||||
HebrewLetter = (\p{WB:Hebrew_Letter} | {HebrewLetterSupp})
|
||||
RegionalIndicator = (\p{WB:Regional_Indicator} | {RegionalIndicatorSupp})
|
||||
HebrewOrALetter = ({HebrewLetter} | {ALetter})
|
||||
|
||||
// UAX#29 WB4. X (Extend | Format)* --> X
|
||||
//
|
||||
HangulEx = [\p{Script:Hangul}&&[\p{WB:ALetter}\p{WB:Hebrew_Letter}]] ({Format} | {Extend})*
|
||||
HebrewOrALetterEx = {HebrewOrALetter} ({Format} | {Extend})*
|
||||
NumericEx = {Numeric} ({Format} | {Extend})*
|
||||
KatakanaEx = {Katakana} ({Format} | {Extend})*
|
||||
MidLetterEx = ({MidLetter} | {MidNumLet} | {SingleQuote}) ({Format} | {Extend})*
|
||||
MidNumericEx = ({MidNum} | {MidNumLet} | {SingleQuote}) ({Format} | {Extend})*
|
||||
ExtendNumLetEx = {ExtendNumLet} ({Format} | {Extend})*
|
||||
HanEx = {Han} ({Format} | {Extend})*
|
||||
HiraganaEx = {Hiragana} ({Format} | {Extend})*
|
||||
SingleQuoteEx = {SingleQuote} ({Format} | {Extend})*
|
||||
DoubleQuoteEx = {DoubleQuote} ({Format} | {Extend})*
|
||||
HebrewLetterEx = {HebrewLetter} ({Format} | {Extend})*
|
||||
RegionalIndicatorEx = {RegionalIndicator} ({Format} | {Extend})*
|
||||
|
||||
|
||||
%{
|
||||
/** Alphanumeric sequences */
|
||||
public static final int WORD_TYPE = StandardAnalyzer.TokenType.ALPHANUM.value;
|
||||
|
||||
/** Numbers */
|
||||
public static final int NUMERIC_TYPE = StandardAnalyzer.TokenType.NUM.value;
|
||||
|
||||
/**
|
||||
* Chars in class \p{Line_Break = Complex_Context} are from South East Asian
|
||||
* scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept
|
||||
* together as as a single token rather than broken up, because the logic
|
||||
* required to break them at word boundaries is too complex for UAX#29.
|
||||
* <p>
|
||||
* See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
|
||||
*/
|
||||
public static final int SOUTH_EAST_ASIAN_TYPE = StandardAnalyzer.TokenType.SOUTHEAST_ASIAN.value;
|
||||
|
||||
public static final int IDEOGRAPHIC_TYPE = StandardAnalyzer.TokenType.IDEOGRAPHIC.value;
|
||||
|
||||
public static final int HIRAGANA_TYPE = StandardAnalyzer.TokenType.HIRAGANA.value;
|
||||
|
||||
public static final int KATAKANA_TYPE = StandardAnalyzer.TokenType.KATAKANA.value;
|
||||
|
||||
public static final int HANGUL_TYPE = StandardAnalyzer.TokenType.HANGUL.value;
|
||||
|
||||
public final int yychar()
|
||||
{
|
||||
return yychar;
|
||||
}
|
||||
|
||||
public String getText()
|
||||
{
|
||||
return String.valueOf(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
|
||||
}
|
||||
|
||||
public char[] getArray()
|
||||
{
|
||||
return Arrays.copyOfRange(zzBuffer, zzStartRead, zzMarkedPos);
|
||||
}
|
||||
|
||||
public byte[] getBytes()
|
||||
{
|
||||
return getText().getBytes();
|
||||
}
|
||||
|
||||
%}
|
||||
|
||||
%%
|
||||
|
||||
// UAX#29 WB1. sot ÷
|
||||
// WB2. ÷ eot
|
||||
//
|
||||
<<EOF>> { return StandardAnalyzer.TokenType.EOF.value; }
|
||||
|
||||
// UAX#29 WB8. Numeric × Numeric
|
||||
// WB11. Numeric (MidNum | MidNumLet | Single_Quote) × Numeric
|
||||
// WB12. Numeric × (MidNum | MidNumLet | Single_Quote) Numeric
|
||||
// WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
|
||||
// WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana)
|
||||
//
|
||||
{ExtendNumLetEx}* {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )* {ExtendNumLetEx}*
|
||||
{ return NUMERIC_TYPE; }
|
||||
|
||||
// subset of the below for typing purposes only!
|
||||
{HangulEx}+
|
||||
{ return HANGUL_TYPE; }
|
||||
|
||||
{KatakanaEx}+
|
||||
{ return KATAKANA_TYPE; }
|
||||
|
||||
// UAX#29 WB5. (ALetter | Hebrew_Letter) × (ALetter | Hebrew_Letter)
|
||||
// WB6. (ALetter | Hebrew_Letter) × (MidLetter | MidNumLet | Single_Quote) (ALetter | Hebrew_Letter)
|
||||
// WB7. (ALetter | Hebrew_Letter) (MidLetter | MidNumLet | Single_Quote) × (ALetter | Hebrew_Letter)
|
||||
// WB7a. Hebrew_Letter × Single_Quote
|
||||
// WB7b. Hebrew_Letter × Double_Quote Hebrew_Letter
|
||||
// WB7c. Hebrew_Letter Double_Quote × Hebrew_Letter
|
||||
// WB9. (ALetter | Hebrew_Letter) × Numeric
|
||||
// WB10. Numeric × (ALetter | Hebrew_Letter)
|
||||
// WB13. Katakana × Katakana
|
||||
// WB13a. (ALetter | Hebrew_Letter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
|
||||
// WB13b. ExtendNumLet × (ALetter | Hebrew_Letter | Numeric | Katakana)
|
||||
//
|
||||
{ExtendNumLetEx}* ( {KatakanaEx} ( {ExtendNumLetEx}* {KatakanaEx} )*
|
||||
| ( {HebrewLetterEx} ( {SingleQuoteEx} | {DoubleQuoteEx} {HebrewLetterEx} )
|
||||
| {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )*
|
||||
| {HebrewOrALetterEx} ( ( {ExtendNumLetEx}* | {MidLetterEx} ) {HebrewOrALetterEx} )*
|
||||
)+
|
||||
)
|
||||
({ExtendNumLetEx}+ ( {KatakanaEx} ( {ExtendNumLetEx}* {KatakanaEx} )*
|
||||
| ( {HebrewLetterEx} ( {SingleQuoteEx} | {DoubleQuoteEx} {HebrewLetterEx} )
|
||||
| {NumericEx} ( ( {ExtendNumLetEx}* | {MidNumericEx} ) {NumericEx} )*
|
||||
| {HebrewOrALetterEx} ( ( {ExtendNumLetEx}* | {MidLetterEx} ) {HebrewOrALetterEx} )*
|
||||
)+
|
||||
)
|
||||
)*
|
||||
{ExtendNumLetEx}*
|
||||
{ return WORD_TYPE; }
|
||||
|
||||
|
||||
// From UAX #29:
|
||||
//
|
||||
// [C]haracters with the Line_Break property values of Contingent_Break (CB),
|
||||
// Complex_Context (SA/South East Asian), and XX (Unknown) are assigned word
|
||||
// boundary property values based on criteria outside of the scope of this
|
||||
// annex. That means that satisfactory treatment of languages like Chinese
|
||||
// or Thai requires special handling.
|
||||
//
|
||||
// In Unicode 6.3, only one character has the \p{Line_Break = Contingent_Break}
|
||||
// property: U+FFFC (  ) OBJECT REPLACEMENT CHARACTER.
|
||||
//
|
||||
// In the ICU implementation of UAX#29, \p{Line_Break = Complex_Context}
|
||||
// character sequences (from South East Asian scripts like Thai, Myanmar, Khmer,
|
||||
// Lao, etc.) are kept together. This grammar does the same below.
|
||||
//
|
||||
// See also the Unicode Line Breaking Algorithm:
|
||||
//
|
||||
// http://www.unicode.org/reports/tr14/#SA
|
||||
//
|
||||
{ComplexContext}+ { return SOUTH_EAST_ASIAN_TYPE; }
|
||||
|
||||
// UAX#29 WB14. Any ÷ Any
|
||||
//
|
||||
{HanEx} { return IDEOGRAPHIC_TYPE; }
|
||||
{HiraganaEx} { return HIRAGANA_TYPE; }
|
||||
|
||||
|
||||
// UAX#29 WB3. CR × LF
|
||||
// WB3a. (Newline | CR | LF) ÷
|
||||
// WB3b. ÷ (Newline | CR | LF)
|
||||
// WB13c. Regional_Indicator × Regional_Indicator
|
||||
// WB14. Any ÷ Any
|
||||
//
|
||||
{RegionalIndicatorEx} {RegionalIndicatorEx}+ | [^]
|
||||
{ /* Break so we don't hit fall-through warning: */ break; /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.Reader;
|
||||
|
||||
/**
|
||||
* Internal interface for supporting versioned grammars.
|
||||
*/
|
||||
public interface StandardTokenizerInterface
|
||||
{
|
||||
|
||||
String getText();
|
||||
|
||||
char[] getArray();
|
||||
|
||||
byte[] getBytes();
|
||||
|
||||
/**
|
||||
* Returns the current position.
|
||||
*/
|
||||
int yychar();
|
||||
|
||||
/**
|
||||
* Returns the length of the matched text region.
|
||||
*/
|
||||
int yylength();
|
||||
|
||||
/**
|
||||
* Resumes scanning until the next regular expression is matched,
|
||||
* the end of input is encountered or an I/O-Error occurs.
|
||||
*
|
||||
* @return the next token, {@link #YYEOF} on end of stream
|
||||
* @exception java.io.IOException if any I/O-Error occurs
|
||||
*/
|
||||
int getNextToken() throws IOException;
|
||||
|
||||
/**
|
||||
* Resets the scanner to read from a new input stream.
|
||||
* Does not close the old reader.
|
||||
*
|
||||
* All internal variables are reset, the old input stream
|
||||
* <b>cannot</b> be reused (internal buffer is discarded and lost).
|
||||
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
|
||||
*
|
||||
* @param reader the new input stream
|
||||
*/
|
||||
void yyreset(Reader reader);
|
||||
}
|
||||
|
|
@ -0,0 +1,272 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Various options for controlling tokenization and enabling
|
||||
* or disabling features
|
||||
*/
|
||||
public class StandardTokenizerOptions
|
||||
{
|
||||
public static final String TOKENIZATION_ENABLE_STEMMING = "tokenization_enable_stemming";
|
||||
public static final String TOKENIZATION_SKIP_STOP_WORDS = "tokenization_skip_stop_words";
|
||||
public static final String TOKENIZATION_LOCALE = "tokenization_locale";
|
||||
public static final String TOKENIZATION_NORMALIZE_LOWERCASE = "tokenization_normalize_lowercase";
|
||||
public static final String TOKENIZATION_NORMALIZE_UPPERCASE = "tokenization_normalize_uppercase";
|
||||
|
||||
public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
|
||||
public static final int DEFAULT_MIN_TOKEN_LENGTH = 0;
|
||||
|
||||
private boolean stemTerms;
|
||||
private boolean ignoreStopTerms;
|
||||
private Locale locale;
|
||||
private boolean caseSensitive;
|
||||
private boolean allTermsToUpperCase;
|
||||
private boolean allTermsToLowerCase;
|
||||
private int minTokenLength;
|
||||
private int maxTokenLength;
|
||||
|
||||
public boolean shouldStemTerms()
|
||||
{
|
||||
return stemTerms;
|
||||
}
|
||||
|
||||
public void setStemTerms(boolean stemTerms)
|
||||
{
|
||||
this.stemTerms = stemTerms;
|
||||
}
|
||||
|
||||
public boolean shouldIgnoreStopTerms()
|
||||
{
|
||||
return ignoreStopTerms;
|
||||
}
|
||||
|
||||
public void setIgnoreStopTerms(boolean ignoreStopTerms)
|
||||
{
|
||||
this.ignoreStopTerms = ignoreStopTerms;
|
||||
}
|
||||
|
||||
public Locale getLocale()
|
||||
{
|
||||
return locale;
|
||||
}
|
||||
|
||||
public void setLocale(Locale locale)
|
||||
{
|
||||
this.locale = locale;
|
||||
}
|
||||
|
||||
public boolean isCaseSensitive()
|
||||
{
|
||||
return caseSensitive;
|
||||
}
|
||||
|
||||
public void setCaseSensitive(boolean caseSensitive)
|
||||
{
|
||||
this.caseSensitive = caseSensitive;
|
||||
}
|
||||
|
||||
public boolean shouldUpperCaseTerms()
|
||||
{
|
||||
return allTermsToUpperCase;
|
||||
}
|
||||
|
||||
public void setAllTermsToUpperCase(boolean allTermsToUpperCase)
|
||||
{
|
||||
this.allTermsToUpperCase = allTermsToUpperCase;
|
||||
}
|
||||
|
||||
public boolean shouldLowerCaseTerms()
|
||||
{
|
||||
return allTermsToLowerCase;
|
||||
}
|
||||
|
||||
public void setAllTermsToLowerCase(boolean allTermsToLowerCase)
|
||||
{
|
||||
this.allTermsToLowerCase = allTermsToLowerCase;
|
||||
}
|
||||
|
||||
public int getMinTokenLength()
|
||||
{
|
||||
return minTokenLength;
|
||||
}
|
||||
|
||||
public void setMinTokenLength(int minTokenLength)
|
||||
{
|
||||
this.minTokenLength = minTokenLength;
|
||||
}
|
||||
|
||||
public int getMaxTokenLength()
|
||||
{
|
||||
return maxTokenLength;
|
||||
}
|
||||
|
||||
public void setMaxTokenLength(int maxTokenLength)
|
||||
{
|
||||
this.maxTokenLength = maxTokenLength;
|
||||
}
|
||||
|
||||
public static class OptionsBuilder {
|
||||
private boolean stemTerms;
|
||||
private boolean ignoreStopTerms;
|
||||
private Locale locale;
|
||||
private boolean caseSensitive;
|
||||
private boolean allTermsToUpperCase;
|
||||
private boolean allTermsToLowerCase;
|
||||
private int minTokenLength = DEFAULT_MIN_TOKEN_LENGTH;
|
||||
private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
|
||||
|
||||
public OptionsBuilder()
|
||||
{
|
||||
}
|
||||
|
||||
public OptionsBuilder stemTerms(boolean stemTerms)
|
||||
{
|
||||
this.stemTerms = stemTerms;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder ignoreStopTerms(boolean ignoreStopTerms)
|
||||
{
|
||||
this.ignoreStopTerms = ignoreStopTerms;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder useLocale(Locale locale)
|
||||
{
|
||||
this.locale = locale;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder caseSensitive(boolean caseSensitive)
|
||||
{
|
||||
this.caseSensitive = caseSensitive;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder alwaysUpperCaseTerms(boolean allTermsToUpperCase)
|
||||
{
|
||||
this.allTermsToUpperCase = allTermsToUpperCase;
|
||||
return this;
|
||||
}
|
||||
|
||||
public OptionsBuilder alwaysLowerCaseTerms(boolean allTermsToLowerCase)
|
||||
{
|
||||
this.allTermsToLowerCase = allTermsToLowerCase;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the min allowed token length. Any token shorter
|
||||
* than this is skipped.
|
||||
*/
|
||||
public OptionsBuilder minTokenLength(int minTokenLength)
|
||||
{
|
||||
if (minTokenLength < 1)
|
||||
throw new IllegalArgumentException("minTokenLength must be greater than zero");
|
||||
this.minTokenLength = minTokenLength;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the max allowed token length. Any token longer
|
||||
* than this is skipped.
|
||||
*/
|
||||
public OptionsBuilder maxTokenLength(int maxTokenLength)
|
||||
{
|
||||
if (maxTokenLength < 1)
|
||||
throw new IllegalArgumentException("maxTokenLength must be greater than zero");
|
||||
this.maxTokenLength = maxTokenLength;
|
||||
return this;
|
||||
}
|
||||
|
||||
public StandardTokenizerOptions build()
|
||||
{
|
||||
if(allTermsToLowerCase && allTermsToUpperCase)
|
||||
throw new IllegalArgumentException("Options to normalize terms cannot be " +
|
||||
"both uppercase and lowercase at the same time");
|
||||
|
||||
StandardTokenizerOptions options = new StandardTokenizerOptions();
|
||||
options.setIgnoreStopTerms(ignoreStopTerms);
|
||||
options.setStemTerms(stemTerms);
|
||||
options.setLocale(locale);
|
||||
options.setCaseSensitive(caseSensitive);
|
||||
options.setAllTermsToLowerCase(allTermsToLowerCase);
|
||||
options.setAllTermsToUpperCase(allTermsToUpperCase);
|
||||
options.setMinTokenLength(minTokenLength);
|
||||
options.setMaxTokenLength(maxTokenLength);
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
public static StandardTokenizerOptions buildFromMap(Map<String, String> optionsMap)
|
||||
{
|
||||
OptionsBuilder optionsBuilder = new OptionsBuilder();
|
||||
|
||||
for (Map.Entry<String, String> entry : optionsMap.entrySet())
|
||||
{
|
||||
switch(entry.getKey())
|
||||
{
|
||||
case TOKENIZATION_ENABLE_STEMMING:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.stemTerms(bool);
|
||||
break;
|
||||
}
|
||||
case TOKENIZATION_SKIP_STOP_WORDS:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.ignoreStopTerms(bool);
|
||||
break;
|
||||
}
|
||||
case TOKENIZATION_LOCALE:
|
||||
{
|
||||
Locale locale = new Locale(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.useLocale(locale);
|
||||
break;
|
||||
}
|
||||
case TOKENIZATION_NORMALIZE_UPPERCASE:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.alwaysUpperCaseTerms(bool);
|
||||
break;
|
||||
}
|
||||
case TOKENIZATION_NORMALIZE_LOWERCASE:
|
||||
{
|
||||
boolean bool = Boolean.parseBoolean(entry.getValue());
|
||||
optionsBuilder = optionsBuilder.alwaysLowerCaseTerms(bool);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
return optionsBuilder.build();
|
||||
}
|
||||
|
||||
public static StandardTokenizerOptions getDefaultOptions()
|
||||
{
|
||||
return new OptionsBuilder()
|
||||
.ignoreStopTerms(true).alwaysLowerCaseTerms(true)
|
||||
.stemTerms(false).useLocale(Locale.ENGLISH).build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
/**
|
||||
* Basic/General Token Filters
|
||||
*/
|
||||
public class BasicResultFilters
|
||||
{
|
||||
private static final Locale DEFAULT_LOCALE = Locale.getDefault();
|
||||
|
||||
public static class LowerCase extends FilterPipelineTask<String, String>
|
||||
{
|
||||
private Locale locale;
|
||||
|
||||
public LowerCase(Locale locale)
|
||||
{
|
||||
this.locale = locale;
|
||||
}
|
||||
|
||||
public LowerCase()
|
||||
{
|
||||
this.locale = DEFAULT_LOCALE;
|
||||
}
|
||||
|
||||
public String process(String input) throws Exception
|
||||
{
|
||||
return input.toLowerCase(locale);
|
||||
}
|
||||
}
|
||||
|
||||
public static class UpperCase extends FilterPipelineTask<String, String>
|
||||
{
|
||||
private Locale locale;
|
||||
|
||||
public UpperCase(Locale locale)
|
||||
{
|
||||
this.locale = locale;
|
||||
}
|
||||
|
||||
public UpperCase()
|
||||
{
|
||||
this.locale = DEFAULT_LOCALE;
|
||||
}
|
||||
|
||||
public String process(String input) throws Exception
|
||||
{
|
||||
return input.toUpperCase(locale);
|
||||
}
|
||||
}
|
||||
|
||||
public static class NoOperation extends FilterPipelineTask<Object, Object>
|
||||
{
|
||||
public Object process(Object input) throws Exception
|
||||
{
|
||||
return input;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
/**
|
||||
* Creates a Pipeline object for applying n pieces of logic
|
||||
* from the provided methods to the builder in a guaranteed order
|
||||
*/
|
||||
public class FilterPipelineBuilder
|
||||
{
|
||||
private final FilterPipelineTask<?,?> parent;
|
||||
private FilterPipelineTask<?,?> current;
|
||||
|
||||
public FilterPipelineBuilder(FilterPipelineTask<?, ?> first)
|
||||
{
|
||||
this(first, first);
|
||||
}
|
||||
|
||||
private FilterPipelineBuilder(FilterPipelineTask<?, ?> first, FilterPipelineTask<?, ?> current)
|
||||
{
|
||||
this.parent = first;
|
||||
this.current = current;
|
||||
}
|
||||
|
||||
public FilterPipelineBuilder add(String name, FilterPipelineTask<?,?> nextTask)
|
||||
{
|
||||
this.current.setLast(name, nextTask);
|
||||
this.current = nextTask;
|
||||
return this;
|
||||
}
|
||||
|
||||
public FilterPipelineTask<?,?> build()
|
||||
{
|
||||
return this.parent;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Executes all linked Pipeline Tasks serially and returns
|
||||
* output (if exists) from the executed logic
|
||||
*/
|
||||
public class FilterPipelineExecutor
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(FilterPipelineExecutor.class);
|
||||
|
||||
public static <F,T> T execute(FilterPipelineTask<F, T> task, T initialInput)
|
||||
{
|
||||
FilterPipelineTask<?, ?> taskPtr = task;
|
||||
T result = initialInput;
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
FilterPipelineTask<F,T> taskGeneric = (FilterPipelineTask<F,T>) taskPtr;
|
||||
result = taskGeneric.process((F) result);
|
||||
taskPtr = taskPtr.next;
|
||||
if(taskPtr == null)
|
||||
return result;
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.info("An unhandled exception to occurred while processing " +
|
||||
"pipeline [{}]", task.getName(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
/**
|
||||
* A single task or set of work to process an input
|
||||
* and return a single output. Maintains a link to the
|
||||
* next task to be executed after itself
|
||||
*/
|
||||
public abstract class FilterPipelineTask<F, T>
|
||||
{
|
||||
private String name;
|
||||
public FilterPipelineTask<?, ?> next;
|
||||
|
||||
protected <K, V> void setLast(String name, FilterPipelineTask<K, V> last)
|
||||
{
|
||||
if (last == this)
|
||||
throw new IllegalArgumentException("provided last task [" + last.name + "] cannot be set to itself");
|
||||
|
||||
if (this.next == null)
|
||||
{
|
||||
this.next = last;
|
||||
this.name = name;
|
||||
}
|
||||
else
|
||||
{
|
||||
this.next.setLast(name, last);
|
||||
}
|
||||
}
|
||||
|
||||
public abstract T process(F input) throws Exception;
|
||||
|
||||
public String getName()
|
||||
{
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.util.HashMap;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
import org.tartarus.snowball.SnowballStemmer;
|
||||
import org.tartarus.snowball.ext.*;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Returns a SnowballStemmer instance appropriate for
|
||||
* a given language
|
||||
*/
|
||||
public class StemmerFactory
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(StemmerFactory.class);
|
||||
private static final LoadingCache<Class, Constructor<?>> STEMMER_CONSTRUCTOR_CACHE = CacheBuilder.newBuilder()
|
||||
.build(new CacheLoader<Class, Constructor<?>>()
|
||||
{
|
||||
public Constructor<?> load(Class aClass) throws Exception
|
||||
{
|
||||
try
|
||||
{
|
||||
return aClass.getConstructor();
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.error("Failed to get stemmer constructor", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
private static final Map<String, Class> SUPPORTED_LANGUAGES;
|
||||
|
||||
static
|
||||
{
|
||||
SUPPORTED_LANGUAGES = new HashMap<>();
|
||||
SUPPORTED_LANGUAGES.put("de", germanStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("da", danishStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("es", spanishStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("en", englishStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("fl", finnishStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("fr", frenchStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("hu", hungarianStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("it", italianStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("nl", dutchStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("no", norwegianStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("pt", portugueseStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("ro", romanianStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("ru", russianStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("sv", swedishStemmer.class);
|
||||
SUPPORTED_LANGUAGES.put("tr", turkishStemmer.class);
|
||||
}
|
||||
|
||||
public static SnowballStemmer getStemmer(Locale locale)
|
||||
{
|
||||
if (locale == null)
|
||||
return null;
|
||||
|
||||
String rootLang = locale.getLanguage().substring(0, 2);
|
||||
try
|
||||
{
|
||||
Class clazz = SUPPORTED_LANGUAGES.get(rootLang);
|
||||
if(clazz == null)
|
||||
return null;
|
||||
Constructor<?> ctor = STEMMER_CONSTRUCTOR_CACHE.get(clazz);
|
||||
return (SnowballStemmer) ctor.newInstance();
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.debug("Failed to create new SnowballStemmer instance " +
|
||||
"for language [{}]", locale.getLanguage(), e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import java.util.Locale;
|
||||
|
||||
import org.tartarus.snowball.SnowballStemmer;
|
||||
|
||||
/**
|
||||
* Filters for performing Stemming on tokens
|
||||
*/
|
||||
public class StemmingFilters
|
||||
{
|
||||
public static class DefaultStemmingFilter extends FilterPipelineTask<String, String>
|
||||
{
|
||||
private SnowballStemmer stemmer;
|
||||
|
||||
public DefaultStemmingFilter(Locale locale)
|
||||
{
|
||||
stemmer = StemmerFactory.getStemmer(locale);
|
||||
}
|
||||
|
||||
public String process(String input) throws Exception
|
||||
{
|
||||
if (stemmer == null)
|
||||
return input;
|
||||
stemmer.setCurrent(input);
|
||||
return (stemmer.stem()) ? stemmer.getCurrent() : input;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import com.google.common.cache.CacheBuilder;
|
||||
import com.google.common.cache.CacheLoader;
|
||||
import com.google.common.cache.LoadingCache;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* Provides a list of Stop Words for a given language
|
||||
*/
|
||||
public class StopWordFactory
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(StopWordFactory.class);
|
||||
|
||||
private static final String DEFAULT_RESOURCE_EXT = "_ST.txt";
|
||||
private static final String DEFAULT_RESOURCE_PREFIX = StopWordFactory.class.getPackage()
|
||||
.getName().replace(".", File.separator);
|
||||
private static final Set<String> SUPPORTED_LANGUAGES = new HashSet<>(
|
||||
Arrays.asList("ar","bg","cs","de","en","es","fi","fr","hi","hu","it",
|
||||
"pl","pt","ro","ru","sv"));
|
||||
|
||||
private static final LoadingCache<String, Set<String>> STOP_WORDS_CACHE = CacheBuilder.newBuilder()
|
||||
.build(new CacheLoader<String, Set<String>>()
|
||||
{
|
||||
public Set<String> load(String s) throws Exception
|
||||
{
|
||||
return getStopWordsFromResource(s);
|
||||
}
|
||||
});
|
||||
|
||||
public static Set<String> getStopWordsForLanguage(Locale locale)
|
||||
{
|
||||
if (locale == null)
|
||||
return null;
|
||||
|
||||
String rootLang = locale.getLanguage().substring(0, 2);
|
||||
try
|
||||
{
|
||||
return (!SUPPORTED_LANGUAGES.contains(rootLang)) ? null : STOP_WORDS_CACHE.get(rootLang);
|
||||
}
|
||||
catch (ExecutionException e)
|
||||
{
|
||||
logger.error("Failed to populate Stop Words Cache for language [{}]", locale.getLanguage(), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static Set<String> getStopWordsFromResource(String language)
|
||||
{
|
||||
Set<String> stopWords = new HashSet<>();
|
||||
String resourceName = DEFAULT_RESOURCE_PREFIX + File.separator + language + DEFAULT_RESOURCE_EXT;
|
||||
try (InputStream is = StopWordFactory.class.getClassLoader().getResourceAsStream(resourceName);
|
||||
BufferedReader r = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)))
|
||||
{
|
||||
String line;
|
||||
while ((line = r.readLine()) != null)
|
||||
{
|
||||
//skip comments (lines starting with # char)
|
||||
if(line.charAt(0) == '#')
|
||||
continue;
|
||||
stopWords.add(line.trim());
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.error("Failed to retrieve Stop Terms resource for language [{}]", language, e);
|
||||
}
|
||||
return stopWords;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.analyzer.filter;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* Filter implementations for input matching Stop Words
|
||||
*/
|
||||
public class StopWordFilters
|
||||
{
|
||||
public static class DefaultStopWordFilter extends FilterPipelineTask<String, String>
|
||||
{
|
||||
private Set<String> stopWords = null;
|
||||
|
||||
public DefaultStopWordFilter(Locale locale)
|
||||
{
|
||||
this.stopWords = StopWordFactory.getStopWordsForLanguage(locale);
|
||||
}
|
||||
|
||||
public String process(String input) throws Exception
|
||||
{
|
||||
return (stopWords != null && stopWords.contains(input)) ? null : input;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,193 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.AsciiType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.db.rows.Cell;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.exceptions.ConfigurationException;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.conf.view.View;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.memory.IndexMemtable;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.io.sstable.Component;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.schema.IndexMetadata;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
|
||||
public class ColumnIndex
|
||||
{
|
||||
private static final String FILE_NAME_FORMAT = "SI_%s.db";
|
||||
|
||||
private final AbstractType<?> keyValidator;
|
||||
|
||||
private final ColumnDefinition column;
|
||||
private final Optional<IndexMetadata> config;
|
||||
|
||||
private final AtomicReference<IndexMemtable> memtable;
|
||||
private final IndexMode mode;
|
||||
|
||||
private final Component component;
|
||||
private final DataTracker tracker;
|
||||
|
||||
public ColumnIndex(AbstractType<?> keyValidator, ColumnDefinition column, IndexMetadata metadata)
|
||||
{
|
||||
this.keyValidator = keyValidator;
|
||||
this.column = column;
|
||||
this.config = metadata == null ? Optional.empty() : Optional.of(metadata);
|
||||
this.mode = IndexMode.getMode(column, config);
|
||||
this.memtable = new AtomicReference<>(new IndexMemtable(this));
|
||||
this.tracker = new DataTracker(keyValidator, this);
|
||||
this.component = new Component(Component.Type.SECONDARY_INDEX, String.format(FILE_NAME_FORMAT, getIndexName()));
|
||||
}
|
||||
|
||||
public void validate() throws ConfigurationException
|
||||
{
|
||||
mode.validate(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize this column index with specific set of SSTables.
|
||||
*
|
||||
* @param sstables The sstables to be used by index initially.
|
||||
*
|
||||
* @return A collection of sstables which don't have this specific index attached to them.
|
||||
*/
|
||||
public Iterable<SSTableReader> init(Set<SSTableReader> sstables)
|
||||
{
|
||||
return tracker.update(Collections.emptySet(), sstables);
|
||||
}
|
||||
|
||||
public AbstractType<?> keyValidator()
|
||||
{
|
||||
return keyValidator;
|
||||
}
|
||||
|
||||
public long index(DecoratedKey key, Row row)
|
||||
{
|
||||
return memtable.get().index(key, getValueOf(column, row, FBUtilities.nowInSeconds()));
|
||||
}
|
||||
|
||||
public void switchMemtable()
|
||||
{
|
||||
memtable.set(new IndexMemtable(this));
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> searchMemtable(Expression e)
|
||||
{
|
||||
return memtable.get().search(e);
|
||||
}
|
||||
|
||||
public void update(Collection<SSTableReader> oldSSTables, Collection<SSTableReader> newSSTables)
|
||||
{
|
||||
tracker.update(oldSSTables, newSSTables);
|
||||
}
|
||||
|
||||
public ColumnDefinition getDefinition()
|
||||
{
|
||||
return column;
|
||||
}
|
||||
|
||||
public AbstractType<?> getValidator()
|
||||
{
|
||||
return column.cellValueType();
|
||||
}
|
||||
|
||||
public Component getComponent()
|
||||
{
|
||||
return component;
|
||||
}
|
||||
|
||||
public IndexMode getMode()
|
||||
{
|
||||
return mode;
|
||||
}
|
||||
|
||||
public String getColumnName()
|
||||
{
|
||||
return column.name.toString();
|
||||
}
|
||||
|
||||
public String getIndexName()
|
||||
{
|
||||
return config.isPresent() ? config.get().name : "undefined";
|
||||
}
|
||||
|
||||
public AbstractAnalyzer getAnalyzer()
|
||||
{
|
||||
AbstractAnalyzer analyzer = mode.getAnalyzer(getValidator());
|
||||
analyzer.init(config.isPresent() ? config.get().options : Collections.emptyMap(), column.cellValueType());
|
||||
return analyzer;
|
||||
}
|
||||
|
||||
public View getView()
|
||||
{
|
||||
return tracker.getView();
|
||||
}
|
||||
|
||||
public boolean hasSSTable(SSTableReader sstable)
|
||||
{
|
||||
return tracker.hasSSTable(sstable);
|
||||
}
|
||||
|
||||
public void dropData(long truncateUntil)
|
||||
{
|
||||
switchMemtable();
|
||||
tracker.dropData(truncateUntil);
|
||||
}
|
||||
|
||||
public boolean isIndexed()
|
||||
{
|
||||
return mode != IndexMode.NOT_INDEXED;
|
||||
}
|
||||
|
||||
public boolean isLiteral()
|
||||
{
|
||||
AbstractType<?> validator = getValidator();
|
||||
return isIndexed() ? mode.isLiteral : (validator instanceof UTF8Type || validator instanceof AsciiType);
|
||||
}
|
||||
|
||||
public static ByteBuffer getValueOf(ColumnDefinition column, Row row, int nowInSecs)
|
||||
{
|
||||
switch (column.kind)
|
||||
{
|
||||
case CLUSTERING:
|
||||
return row.clustering().get(column.position());
|
||||
|
||||
case REGULAR:
|
||||
Cell cell = row.getCell(column);
|
||||
return cell == null || !cell.isLive(nowInSecs) ? null : cell.value();
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.conf.view.View;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/** a pared-down version of DataTracker and DT.View. need one for each index of each column family */
|
||||
public class DataTracker
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(DataTracker.class);
|
||||
|
||||
private final AbstractType<?> keyValidator;
|
||||
private final ColumnIndex columnIndex;
|
||||
private final AtomicReference<View> view = new AtomicReference<>();
|
||||
|
||||
public DataTracker(AbstractType<?> keyValidator, ColumnIndex index)
|
||||
{
|
||||
this.keyValidator = keyValidator;
|
||||
this.columnIndex = index;
|
||||
this.view.set(new View(index, Collections.<SSTableIndex>emptySet()));
|
||||
}
|
||||
|
||||
public View getView()
|
||||
{
|
||||
return view.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces old SSTables with new by creating new immutable tracker.
|
||||
*
|
||||
* @param oldSSTables A set of SSTables to remove.
|
||||
* @param newSSTables A set of SSTables to add to tracker.
|
||||
*
|
||||
* @return A collection of SSTables which don't have component attached for current index.
|
||||
*/
|
||||
public Iterable<SSTableReader> update(Collection<SSTableReader> oldSSTables, Collection<SSTableReader> newSSTables)
|
||||
{
|
||||
final Set<SSTableIndex> newIndexes = getIndexes(newSSTables);
|
||||
final Set<SSTableReader> indexedSSTables = getSSTables(newIndexes);
|
||||
|
||||
View currentView, newView;
|
||||
do
|
||||
{
|
||||
currentView = view.get();
|
||||
newView = new View(columnIndex, currentView.getIndexes(), oldSSTables, newIndexes);
|
||||
}
|
||||
while (!view.compareAndSet(currentView, newView));
|
||||
|
||||
return newSSTables.stream().filter(sstable -> !indexedSSTables.contains(sstable)).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public boolean hasSSTable(SSTableReader sstable)
|
||||
{
|
||||
View currentView = view.get();
|
||||
for (SSTableIndex index : currentView)
|
||||
{
|
||||
if (index.getSSTable().equals(sstable))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public void dropData(long truncateUntil)
|
||||
{
|
||||
View currentView = view.get();
|
||||
if (currentView == null)
|
||||
return;
|
||||
|
||||
Set<SSTableReader> toRemove = new HashSet<>();
|
||||
for (SSTableIndex index : currentView)
|
||||
{
|
||||
SSTableReader sstable = index.getSSTable();
|
||||
if (sstable.getMaxTimestamp() > truncateUntil)
|
||||
continue;
|
||||
|
||||
index.markObsolete();
|
||||
toRemove.add(sstable);
|
||||
}
|
||||
|
||||
update(toRemove, Collections.<SSTableReader>emptyList());
|
||||
}
|
||||
|
||||
private Set<SSTableIndex> getIndexes(Collection<SSTableReader> sstables)
|
||||
{
|
||||
Set<SSTableIndex> indexes = new HashSet<>(sstables.size());
|
||||
for (SSTableReader sstable : sstables)
|
||||
{
|
||||
if (sstable.isMarkedCompacted())
|
||||
continue;
|
||||
|
||||
File indexFile = new File(sstable.descriptor.filenameFor(columnIndex.getComponent()));
|
||||
if (!indexFile.exists())
|
||||
continue;
|
||||
|
||||
SSTableIndex index = null;
|
||||
|
||||
try
|
||||
{
|
||||
index = new SSTableIndex(columnIndex, indexFile, sstable);
|
||||
|
||||
logger.info("SSTableIndex.open(column: {}, minTerm: {}, maxTerm: {}, minKey: {}, maxKey: {}, sstable: {})",
|
||||
columnIndex.getColumnName(),
|
||||
columnIndex.getValidator().getString(index.minTerm()),
|
||||
columnIndex.getValidator().getString(index.maxTerm()),
|
||||
keyValidator.getString(index.minKey()),
|
||||
keyValidator.getString(index.maxKey()),
|
||||
index.getSSTable());
|
||||
|
||||
// Try to add new index to the set, if set already has such index, we'll simply release and move on.
|
||||
// This covers situation when sstable collection has the same sstable multiple
|
||||
// times because we don't know what kind of collection it actually is.
|
||||
if (!indexes.add(index))
|
||||
index.release();
|
||||
}
|
||||
catch (Throwable t)
|
||||
{
|
||||
logger.error("Can't open index file at " + indexFile.getAbsolutePath() + ", skipping.", t);
|
||||
if (index != null)
|
||||
index.release();
|
||||
}
|
||||
}
|
||||
|
||||
return indexes;
|
||||
}
|
||||
|
||||
private Set<SSTableReader> getSSTables(Set<SSTableIndex> indexes)
|
||||
{
|
||||
return Sets.newHashSet(indexes.stream().map(SSTableIndex::getSSTable).collect(Collectors.toList()));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,169 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.analyzer.NoOpAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder.Mode;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.AsciiType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.exceptions.ConfigurationException;
|
||||
import org.apache.cassandra.schema.IndexMetadata;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class IndexMode
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(IndexMode.class);
|
||||
|
||||
public static final IndexMode NOT_INDEXED = new IndexMode(Mode.PREFIX, true, false, NonTokenizingAnalyzer.class, 0);
|
||||
|
||||
private static final Set<AbstractType<?>> TOKENIZABLE_TYPES = new HashSet<AbstractType<?>>()
|
||||
{{
|
||||
add(UTF8Type.instance);
|
||||
add(AsciiType.instance);
|
||||
}};
|
||||
|
||||
private static final String INDEX_MODE_OPTION = "mode";
|
||||
private static final String INDEX_ANALYZED_OPTION = "analyzed";
|
||||
private static final String INDEX_ANALYZER_CLASS_OPTION = "analyzer_class";
|
||||
private static final String INDEX_IS_LITERAL_OPTION = "is_literal";
|
||||
private static final String INDEX_MAX_FLUSH_MEMORY_OPTION = "max_compaction_flush_memory_in_mb";
|
||||
private static final double INDEX_MAX_FLUSH_DEFAULT_MULTIPLIER = 0.15;
|
||||
|
||||
public final Mode mode;
|
||||
public final boolean isAnalyzed, isLiteral;
|
||||
public final Class analyzerClass;
|
||||
public final long maxCompactionFlushMemoryInMb;
|
||||
|
||||
private IndexMode(Mode mode, boolean isLiteral, boolean isAnalyzed, Class analyzerClass, long maxFlushMemMb)
|
||||
{
|
||||
this.mode = mode;
|
||||
this.isLiteral = isLiteral;
|
||||
this.isAnalyzed = isAnalyzed;
|
||||
this.analyzerClass = analyzerClass;
|
||||
this.maxCompactionFlushMemoryInMb = maxFlushMemMb;
|
||||
}
|
||||
|
||||
public void validate(Optional<IndexMetadata> config) throws ConfigurationException
|
||||
{
|
||||
if (!config.isPresent())
|
||||
return;
|
||||
|
||||
Map<String, String> indexOptions = config.get().options;
|
||||
// validate that a valid analyzer class was provided if specified
|
||||
if (indexOptions.containsKey(INDEX_ANALYZER_CLASS_OPTION))
|
||||
{
|
||||
try
|
||||
{
|
||||
Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
|
||||
}
|
||||
catch (ClassNotFoundException e)
|
||||
{
|
||||
throw new ConfigurationException(String.format("Invalid analyzer class option specified [%s]",
|
||||
indexOptions.get(INDEX_ANALYZER_CLASS_OPTION)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public AbstractAnalyzer getAnalyzer(AbstractType<?> validator)
|
||||
{
|
||||
AbstractAnalyzer analyzer = new NoOpAnalyzer();
|
||||
|
||||
try
|
||||
{
|
||||
if (isAnalyzed)
|
||||
{
|
||||
if (analyzerClass != null)
|
||||
analyzer = (AbstractAnalyzer) analyzerClass.newInstance();
|
||||
else if (TOKENIZABLE_TYPES.contains(validator))
|
||||
analyzer = new StandardAnalyzer();
|
||||
}
|
||||
}
|
||||
catch (InstantiationException | IllegalAccessException e)
|
||||
{
|
||||
logger.error("Failed to create new instance of analyzer with class [{}]", analyzerClass.getName(), e);
|
||||
}
|
||||
|
||||
return analyzer;
|
||||
}
|
||||
|
||||
public static IndexMode getMode(ColumnDefinition column, Optional<IndexMetadata> config)
|
||||
{
|
||||
Map<String, String> indexOptions = config.isPresent() ? config.get().options : null;
|
||||
if (indexOptions == null || indexOptions.isEmpty())
|
||||
return IndexMode.NOT_INDEXED;
|
||||
|
||||
Mode mode = indexOptions.get(INDEX_MODE_OPTION) == null
|
||||
? Mode.PREFIX
|
||||
: Mode.mode(indexOptions.get(INDEX_MODE_OPTION));
|
||||
|
||||
boolean isAnalyzed = false;
|
||||
Class analyzerClass = null;
|
||||
try
|
||||
{
|
||||
if (indexOptions.get(INDEX_ANALYZER_CLASS_OPTION) != null)
|
||||
{
|
||||
analyzerClass = Class.forName(indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
|
||||
isAnalyzed = indexOptions.get(INDEX_ANALYZED_OPTION) == null
|
||||
? true : Boolean.valueOf(indexOptions.get(INDEX_ANALYZED_OPTION));
|
||||
}
|
||||
else if (indexOptions.get(INDEX_ANALYZED_OPTION) != null)
|
||||
{
|
||||
isAnalyzed = Boolean.valueOf(indexOptions.get(INDEX_ANALYZED_OPTION));
|
||||
}
|
||||
}
|
||||
catch (ClassNotFoundException e)
|
||||
{
|
||||
// should not happen as we already validated we could instantiate an instance in validateOptions()
|
||||
logger.error("Failed to find specified analyzer class [{}]. Falling back to default analyzer",
|
||||
indexOptions.get(INDEX_ANALYZER_CLASS_OPTION));
|
||||
}
|
||||
|
||||
boolean isLiteral = false;
|
||||
try
|
||||
{
|
||||
String literalOption = indexOptions.get(INDEX_IS_LITERAL_OPTION);
|
||||
AbstractType<?> validator = column.cellValueType();
|
||||
|
||||
isLiteral = literalOption == null
|
||||
? (validator instanceof UTF8Type || validator instanceof AsciiType)
|
||||
: Boolean.valueOf(literalOption);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.error("failed to parse {} option, defaulting to 'false' for {} index.", INDEX_IS_LITERAL_OPTION, config.get().name);
|
||||
}
|
||||
|
||||
Long maxMemMb = indexOptions.get(INDEX_MAX_FLUSH_MEMORY_OPTION) == null
|
||||
? (long) (1073741824 * INDEX_MAX_FLUSH_DEFAULT_MULTIPLIER) // 1G default for memtable
|
||||
: Long.parseLong(indexOptions.get(INDEX_MAX_FLUSH_MEMORY_OPTION));
|
||||
|
||||
return new IndexMode(mode, isLiteral, isAnalyzed, analyzerClass, maxMemMb);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,194 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf.view;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.trie.KeyAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.utils.trie.PatriciaTrie;
|
||||
import org.apache.cassandra.index.sasi.utils.trie.Trie;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.utils.Interval;
|
||||
import org.apache.cassandra.utils.IntervalTree;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
/**
|
||||
* This class is an extension over RangeTermTree for string terms,
|
||||
* it is required because interval tree can't handle matching if search is on the
|
||||
* prefix of min/max of the range, so for ascii/utf8 fields we build an additional
|
||||
* prefix trie (including both min/max terms of the index) and do union of the results
|
||||
* of the prefix tree search and results from the interval tree lookup.
|
||||
*/
|
||||
public class PrefixTermTree extends RangeTermTree
|
||||
{
|
||||
private final OnDiskIndexBuilder.Mode mode;
|
||||
private final Trie<ByteBuffer, Set<SSTableIndex>> trie;
|
||||
|
||||
public PrefixTermTree(ByteBuffer min, ByteBuffer max,
|
||||
Trie<ByteBuffer, Set<SSTableIndex>> trie,
|
||||
IntervalTree<ByteBuffer, SSTableIndex, Interval<ByteBuffer, SSTableIndex>> ranges,
|
||||
OnDiskIndexBuilder.Mode mode)
|
||||
{
|
||||
super(min, max, ranges);
|
||||
|
||||
this.mode = mode;
|
||||
this.trie = trie;
|
||||
}
|
||||
|
||||
public Set<SSTableIndex> search(Expression e)
|
||||
{
|
||||
Map<ByteBuffer, Set<SSTableIndex>> indexes = (e == null || e.lower == null || mode == OnDiskIndexBuilder.Mode.CONTAINS)
|
||||
? trie : trie.prefixMap(e.lower.value);
|
||||
|
||||
Set<SSTableIndex> view = new HashSet<>(indexes.size());
|
||||
indexes.values().forEach(view::addAll);
|
||||
|
||||
return Sets.union(view, super.search(e));
|
||||
}
|
||||
|
||||
public static class Builder extends RangeTermTree.Builder
|
||||
{
|
||||
private final PatriciaTrie<ByteBuffer, Set<SSTableIndex>> trie;
|
||||
|
||||
protected Builder(OnDiskIndexBuilder.Mode mode, final AbstractType<?> comparator)
|
||||
{
|
||||
super(mode, comparator);
|
||||
trie = new PatriciaTrie<>(new ByteBufferKeyAnalyzer(comparator));
|
||||
}
|
||||
|
||||
public void addIndex(SSTableIndex index)
|
||||
{
|
||||
super.addIndex(index);
|
||||
addTerm(index.minTerm(), index);
|
||||
addTerm(index.maxTerm(), index);
|
||||
}
|
||||
|
||||
public TermTree build()
|
||||
{
|
||||
return new PrefixTermTree(min, max, trie, IntervalTree.build(intervals), mode);
|
||||
}
|
||||
|
||||
private void addTerm(ByteBuffer term, SSTableIndex index)
|
||||
{
|
||||
Set<SSTableIndex> indexes = trie.get(term);
|
||||
if (indexes == null)
|
||||
trie.put(term, (indexes = new HashSet<>()));
|
||||
|
||||
indexes.add(index);
|
||||
}
|
||||
}
|
||||
|
||||
private static class ByteBufferKeyAnalyzer implements KeyAnalyzer<ByteBuffer>
|
||||
{
|
||||
private final AbstractType<?> comparator;
|
||||
|
||||
public ByteBufferKeyAnalyzer(AbstractType<?> comparator)
|
||||
{
|
||||
this.comparator = comparator;
|
||||
}
|
||||
|
||||
/**
|
||||
* A bit mask where the first bit is 1 and the others are zero
|
||||
*/
|
||||
private static final int MSB = 1 << Byte.SIZE-1;
|
||||
|
||||
public int compare(ByteBuffer a, ByteBuffer b)
|
||||
{
|
||||
return comparator.compare(a, b);
|
||||
}
|
||||
|
||||
public int lengthInBits(ByteBuffer o)
|
||||
{
|
||||
return o.remaining() * Byte.SIZE;
|
||||
}
|
||||
|
||||
public boolean isBitSet(ByteBuffer key, int bitIndex)
|
||||
{
|
||||
if (bitIndex >= lengthInBits(key))
|
||||
return false;
|
||||
|
||||
int index = bitIndex / Byte.SIZE;
|
||||
int bit = bitIndex % Byte.SIZE;
|
||||
return (key.get(index) & mask(bit)) != 0;
|
||||
}
|
||||
|
||||
public int bitIndex(ByteBuffer key, ByteBuffer otherKey)
|
||||
{
|
||||
int length = Math.max(key.remaining(), otherKey.remaining());
|
||||
|
||||
boolean allNull = true;
|
||||
for (int i = 0; i < length; i++)
|
||||
{
|
||||
byte b1 = valueAt(key, i);
|
||||
byte b2 = valueAt(otherKey, i);
|
||||
|
||||
if (b1 != b2)
|
||||
{
|
||||
int xor = b1 ^ b2;
|
||||
for (int j = 0; j < Byte.SIZE; j++)
|
||||
{
|
||||
if ((xor & mask(j)) != 0)
|
||||
return (i * Byte.SIZE) + j;
|
||||
}
|
||||
}
|
||||
|
||||
if (b1 != 0)
|
||||
allNull = false;
|
||||
}
|
||||
|
||||
return allNull ? KeyAnalyzer.NULL_BIT_KEY : KeyAnalyzer.EQUAL_BIT_KEY;
|
||||
}
|
||||
|
||||
public boolean isPrefix(ByteBuffer key, ByteBuffer prefix)
|
||||
{
|
||||
if (key.remaining() < prefix.remaining())
|
||||
return false;
|
||||
|
||||
for (int i = 0; i < prefix.remaining(); i++)
|
||||
{
|
||||
if (key.get(i) != prefix.get(i))
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the {@code byte} value at the given index.
|
||||
*/
|
||||
private byte valueAt(ByteBuffer value, int index)
|
||||
{
|
||||
return index >= 0 && index < value.remaining() ? value.get(index) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a bit mask where the given bit is set
|
||||
*/
|
||||
private int mask(int bit)
|
||||
{
|
||||
return MSB >>> bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf.view;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.utils.Interval;
|
||||
import org.apache.cassandra.utils.IntervalTree;
|
||||
|
||||
public class RangeTermTree implements TermTree
|
||||
{
|
||||
protected final ByteBuffer min, max;
|
||||
protected final IntervalTree<ByteBuffer, SSTableIndex, Interval<ByteBuffer, SSTableIndex>> rangeTree;
|
||||
|
||||
public RangeTermTree(ByteBuffer min, ByteBuffer max, IntervalTree<ByteBuffer, SSTableIndex, Interval<ByteBuffer, SSTableIndex>> rangeTree)
|
||||
{
|
||||
this.min = min;
|
||||
this.max = max;
|
||||
this.rangeTree = rangeTree;
|
||||
}
|
||||
|
||||
public Set<SSTableIndex> search(Expression e)
|
||||
{
|
||||
ByteBuffer minTerm = e.lower == null ? min : e.lower.value;
|
||||
ByteBuffer maxTerm = e.upper == null ? max : e.upper.value;
|
||||
|
||||
return new HashSet<>(rangeTree.search(Interval.create(minTerm, maxTerm, (SSTableIndex) null)));
|
||||
}
|
||||
|
||||
public int intervalCount()
|
||||
{
|
||||
return rangeTree.intervalCount();
|
||||
}
|
||||
|
||||
static class Builder extends TermTree.Builder
|
||||
{
|
||||
protected final List<Interval<ByteBuffer, SSTableIndex>> intervals = new ArrayList<>();
|
||||
|
||||
protected Builder(OnDiskIndexBuilder.Mode mode, AbstractType<?> comparator)
|
||||
{
|
||||
super(mode, comparator);
|
||||
}
|
||||
|
||||
public void addIndex(SSTableIndex index)
|
||||
{
|
||||
intervals.add(Interval.create(index.minTerm(), index.maxTerm(), index));
|
||||
}
|
||||
|
||||
public TermTree build()
|
||||
{
|
||||
return new RangeTermTree(min, max, IntervalTree.build(intervals));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf.view;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public interface TermTree
|
||||
{
|
||||
Set<SSTableIndex> search(Expression e);
|
||||
|
||||
int intervalCount();
|
||||
|
||||
abstract class Builder
|
||||
{
|
||||
protected final OnDiskIndexBuilder.Mode mode;
|
||||
protected final AbstractType<?> comparator;
|
||||
protected ByteBuffer min, max;
|
||||
|
||||
protected Builder(OnDiskIndexBuilder.Mode mode, AbstractType<?> comparator)
|
||||
{
|
||||
this.mode = mode;
|
||||
this.comparator = comparator;
|
||||
}
|
||||
|
||||
public final void add(SSTableIndex index)
|
||||
{
|
||||
addIndex(index);
|
||||
|
||||
min = min == null || comparator.compare(min, index.minTerm()) > 0 ? index.minTerm() : min;
|
||||
max = max == null || comparator.compare(max, index.maxTerm()) < 0 ? index.maxTerm() : max;
|
||||
}
|
||||
|
||||
protected abstract void addIndex(SSTableIndex index);
|
||||
|
||||
public abstract TermTree build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.conf.view;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.AsciiType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.io.sstable.Descriptor;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.utils.Interval;
|
||||
import org.apache.cassandra.utils.IntervalTree;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Sets;
|
||||
|
||||
public class View implements Iterable<SSTableIndex>
|
||||
{
|
||||
private final Map<Descriptor, SSTableIndex> view;
|
||||
|
||||
private final TermTree termTree;
|
||||
private final IntervalTree<ByteBuffer, SSTableIndex, Interval<ByteBuffer, SSTableIndex>> keyIntervalTree;
|
||||
|
||||
public View(ColumnIndex index, Set<SSTableIndex> indexes)
|
||||
{
|
||||
this(index, Collections.<SSTableIndex>emptyList(), Collections.<SSTableReader>emptyList(), indexes);
|
||||
}
|
||||
|
||||
public View(ColumnIndex index,
|
||||
Collection<SSTableIndex> currentView,
|
||||
Collection<SSTableReader> oldSSTables,
|
||||
Set<SSTableIndex> newIndexes)
|
||||
{
|
||||
Map<Descriptor, SSTableIndex> newView = new HashMap<>();
|
||||
|
||||
AbstractType<?> validator = index.getValidator();
|
||||
TermTree.Builder termTreeBuilder = (validator instanceof AsciiType || validator instanceof UTF8Type)
|
||||
? new PrefixTermTree.Builder(index.getMode().mode, validator)
|
||||
: new RangeTermTree.Builder(index.getMode().mode, validator);
|
||||
|
||||
List<Interval<ByteBuffer, SSTableIndex>> keyIntervals = new ArrayList<>();
|
||||
for (SSTableIndex sstableIndex : Iterables.concat(currentView, newIndexes))
|
||||
{
|
||||
SSTableReader sstable = sstableIndex.getSSTable();
|
||||
if (oldSSTables.contains(sstable) || sstable.isMarkedCompacted() || newView.containsKey(sstable.descriptor))
|
||||
{
|
||||
sstableIndex.release();
|
||||
continue;
|
||||
}
|
||||
|
||||
newView.put(sstable.descriptor, sstableIndex);
|
||||
|
||||
termTreeBuilder.add(sstableIndex);
|
||||
keyIntervals.add(Interval.create(sstableIndex.minKey(), sstableIndex.maxKey(), sstableIndex));
|
||||
}
|
||||
|
||||
this.view = newView;
|
||||
this.termTree = termTreeBuilder.build();
|
||||
this.keyIntervalTree = IntervalTree.build(keyIntervals);
|
||||
|
||||
if (keyIntervalTree.intervalCount() != termTree.intervalCount())
|
||||
throw new IllegalStateException(String.format("mismatched sizes for intervals tree for keys vs terms: %d != %d", keyIntervalTree.intervalCount(), termTree.intervalCount()));
|
||||
}
|
||||
|
||||
public Set<SSTableIndex> match(final Set<SSTableReader> scope, Expression expression)
|
||||
{
|
||||
return Sets.filter(termTree.search(expression), index -> scope.contains(index.getSSTable()));
|
||||
}
|
||||
|
||||
public List<SSTableIndex> match(ByteBuffer minKey, ByteBuffer maxKey)
|
||||
{
|
||||
return keyIntervalTree.search(Interval.create(minKey, maxKey, (SSTableIndex) null));
|
||||
}
|
||||
|
||||
public Iterator<SSTableIndex> iterator()
|
||||
{
|
||||
return view.values().iterator();
|
||||
}
|
||||
|
||||
public Collection<SSTableIndex> getIndexes()
|
||||
{
|
||||
return view.values();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
/**
|
||||
* Object descriptor for SSTableAttachedSecondaryIndex files. Similar to, and based upon, the sstable descriptor.
|
||||
*/
|
||||
public class Descriptor
|
||||
{
|
||||
public static final String VERSION_AA = "aa";
|
||||
public static final String VERSION_AB = "ab";
|
||||
public static final String CURRENT_VERSION = VERSION_AB;
|
||||
public static final Descriptor CURRENT = new Descriptor(CURRENT_VERSION);
|
||||
|
||||
public static class Version
|
||||
{
|
||||
public final String version;
|
||||
|
||||
public Version(String version)
|
||||
{
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return version;
|
||||
}
|
||||
}
|
||||
|
||||
public final Version version;
|
||||
|
||||
public Descriptor(String v)
|
||||
{
|
||||
this.version = new Version(v);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.Term;
|
||||
import org.apache.cassandra.index.sasi.utils.MappedBuffer;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public abstract class OnDiskBlock<T extends Term>
|
||||
{
|
||||
public enum BlockType
|
||||
{
|
||||
POINTER, DATA
|
||||
}
|
||||
|
||||
// this contains offsets of the terms and term data
|
||||
protected final MappedBuffer blockIndex;
|
||||
protected final int blockIndexSize;
|
||||
|
||||
protected final boolean hasCombinedIndex;
|
||||
protected final TokenTree combinedIndex;
|
||||
|
||||
public OnDiskBlock(Descriptor descriptor, MappedBuffer block, BlockType blockType)
|
||||
{
|
||||
blockIndex = block;
|
||||
|
||||
if (blockType == BlockType.POINTER)
|
||||
{
|
||||
hasCombinedIndex = false;
|
||||
combinedIndex = null;
|
||||
blockIndexSize = block.getInt() << 1; // num terms * sizeof(short)
|
||||
return;
|
||||
}
|
||||
|
||||
long blockOffset = block.position();
|
||||
int combinedIndexOffset = block.getInt(blockOffset + OnDiskIndexBuilder.BLOCK_SIZE);
|
||||
|
||||
hasCombinedIndex = (combinedIndexOffset >= 0);
|
||||
long blockIndexOffset = blockOffset + OnDiskIndexBuilder.BLOCK_SIZE + 4 + combinedIndexOffset;
|
||||
|
||||
combinedIndex = hasCombinedIndex ? new TokenTree(descriptor, blockIndex.duplicate().position(blockIndexOffset)) : null;
|
||||
blockIndexSize = block.getInt() * 2;
|
||||
}
|
||||
|
||||
public SearchResult<T> search(AbstractType<?> comparator, ByteBuffer query)
|
||||
{
|
||||
int cmp = -1, start = 0, end = termCount() - 1, middle = 0;
|
||||
|
||||
T element = null;
|
||||
while (start <= end)
|
||||
{
|
||||
middle = start + ((end - start) >> 1);
|
||||
element = getTerm(middle);
|
||||
|
||||
cmp = element.compareTo(comparator, query);
|
||||
if (cmp == 0)
|
||||
return new SearchResult<>(element, cmp, middle);
|
||||
else if (cmp < 0)
|
||||
start = middle + 1;
|
||||
else
|
||||
end = middle - 1;
|
||||
}
|
||||
|
||||
return new SearchResult<>(element, cmp, middle);
|
||||
}
|
||||
|
||||
protected T getTerm(int index)
|
||||
{
|
||||
MappedBuffer dup = blockIndex.duplicate();
|
||||
long startsAt = getTermPosition(index);
|
||||
if (termCount() - 1 == index) // last element
|
||||
dup.position(startsAt);
|
||||
else
|
||||
dup.position(startsAt).limit(getTermPosition(index + 1));
|
||||
|
||||
return cast(dup);
|
||||
}
|
||||
|
||||
protected long getTermPosition(int idx)
|
||||
{
|
||||
return getTermPosition(blockIndex, idx, blockIndexSize);
|
||||
}
|
||||
|
||||
protected int termCount()
|
||||
{
|
||||
return blockIndexSize >> 1;
|
||||
}
|
||||
|
||||
protected abstract T cast(MappedBuffer data);
|
||||
|
||||
static long getTermPosition(MappedBuffer data, int idx, int indexSize)
|
||||
{
|
||||
idx <<= 1;
|
||||
assert idx < indexSize;
|
||||
return data.position() + indexSize + data.getShort(data.position() + idx);
|
||||
}
|
||||
|
||||
public TokenTree getBlockIndex()
|
||||
{
|
||||
return combinedIndex;
|
||||
}
|
||||
|
||||
public int minOffset(OnDiskIndex.IteratorOrder order)
|
||||
{
|
||||
return order == OnDiskIndex.IteratorOrder.DESC ? 0 : termCount() - 1;
|
||||
}
|
||||
|
||||
public int maxOffset(OnDiskIndex.IteratorOrder order)
|
||||
{
|
||||
return minOffset(order) == 0 ? termCount() - 1 : 0;
|
||||
}
|
||||
|
||||
public static class SearchResult<T>
|
||||
{
|
||||
public final T result;
|
||||
public final int index, cmp;
|
||||
|
||||
public SearchResult(T result, int cmp, int index)
|
||||
{
|
||||
this.result = result;
|
||||
this.index = index;
|
||||
this.cmp = cmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,773 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.Term;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression.Op;
|
||||
import org.apache.cassandra.index.sasi.utils.MappedBuffer;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.AbstractIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.io.FSReadError;
|
||||
import org.apache.cassandra.io.util.ChannelProxy;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
|
||||
import static org.apache.cassandra.index.sasi.disk.OnDiskBlock.SearchResult;
|
||||
|
||||
public class OnDiskIndex implements Iterable<OnDiskIndex.DataTerm>, Closeable
|
||||
{
|
||||
public enum IteratorOrder
|
||||
{
|
||||
DESC(1), ASC(-1);
|
||||
|
||||
public final int step;
|
||||
|
||||
IteratorOrder(int step)
|
||||
{
|
||||
this.step = step;
|
||||
}
|
||||
|
||||
public int startAt(OnDiskBlock<DataTerm> block, Expression e)
|
||||
{
|
||||
switch (this)
|
||||
{
|
||||
case DESC:
|
||||
return e.lower == null
|
||||
? 0
|
||||
: startAt(block.search(e.validator, e.lower.value), e.lower.inclusive);
|
||||
|
||||
case ASC:
|
||||
return e.upper == null
|
||||
? block.termCount() - 1
|
||||
: startAt(block.search(e.validator, e.upper.value), e.upper.inclusive);
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown order: " + this);
|
||||
}
|
||||
}
|
||||
|
||||
public int startAt(SearchResult<DataTerm> found, boolean inclusive)
|
||||
{
|
||||
switch (this)
|
||||
{
|
||||
case DESC:
|
||||
if (found.cmp < 0)
|
||||
return found.index + 1;
|
||||
|
||||
return inclusive || found.cmp != 0 ? found.index : found.index + 1;
|
||||
|
||||
case ASC:
|
||||
if (found.cmp < 0) // search term was bigger then whole data set
|
||||
return found.index;
|
||||
return inclusive && (found.cmp == 0 || found.cmp < 0) ? found.index : found.index - 1;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown order: " + this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public final Descriptor descriptor;
|
||||
protected final OnDiskIndexBuilder.Mode mode;
|
||||
protected final OnDiskIndexBuilder.TermSize termSize;
|
||||
|
||||
protected final AbstractType<?> comparator;
|
||||
protected final MappedBuffer indexFile;
|
||||
protected final long indexSize;
|
||||
|
||||
protected final Function<Long, DecoratedKey> keyFetcher;
|
||||
|
||||
protected final String indexPath;
|
||||
|
||||
protected final PointerLevel[] levels;
|
||||
protected final DataLevel dataLevel;
|
||||
|
||||
protected final ByteBuffer minTerm, maxTerm, minKey, maxKey;
|
||||
|
||||
public OnDiskIndex(File index, AbstractType<?> cmp, Function<Long, DecoratedKey> keyReader)
|
||||
{
|
||||
keyFetcher = keyReader;
|
||||
|
||||
comparator = cmp;
|
||||
indexPath = index.getAbsolutePath();
|
||||
|
||||
RandomAccessFile backingFile = null;
|
||||
try
|
||||
{
|
||||
backingFile = new RandomAccessFile(index, "r");
|
||||
|
||||
descriptor = new Descriptor(backingFile.readUTF());
|
||||
|
||||
termSize = OnDiskIndexBuilder.TermSize.of(backingFile.readShort());
|
||||
|
||||
minTerm = ByteBufferUtil.readWithShortLength(backingFile);
|
||||
maxTerm = ByteBufferUtil.readWithShortLength(backingFile);
|
||||
|
||||
minKey = ByteBufferUtil.readWithShortLength(backingFile);
|
||||
maxKey = ByteBufferUtil.readWithShortLength(backingFile);
|
||||
|
||||
mode = OnDiskIndexBuilder.Mode.mode(backingFile.readUTF());
|
||||
|
||||
indexSize = backingFile.length();
|
||||
indexFile = new MappedBuffer(new ChannelProxy(indexPath, backingFile.getChannel()));
|
||||
|
||||
// start of the levels
|
||||
indexFile.position(indexFile.getLong(indexSize - 8));
|
||||
|
||||
int numLevels = indexFile.getInt();
|
||||
levels = new PointerLevel[numLevels];
|
||||
for (int i = 0; i < levels.length; i++)
|
||||
{
|
||||
int blockCount = indexFile.getInt();
|
||||
levels[i] = new PointerLevel(indexFile.position(), blockCount);
|
||||
indexFile.position(indexFile.position() + blockCount * 8);
|
||||
}
|
||||
|
||||
int blockCount = indexFile.getInt();
|
||||
dataLevel = new DataLevel(indexFile.position(), blockCount);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new FSReadError(e, index);
|
||||
}
|
||||
finally
|
||||
{
|
||||
FileUtils.closeQuietly(backingFile);
|
||||
}
|
||||
}
|
||||
|
||||
public ByteBuffer minTerm()
|
||||
{
|
||||
return minTerm;
|
||||
}
|
||||
|
||||
public ByteBuffer maxTerm()
|
||||
{
|
||||
return maxTerm;
|
||||
}
|
||||
|
||||
public ByteBuffer minKey()
|
||||
{
|
||||
return minKey;
|
||||
}
|
||||
|
||||
public ByteBuffer maxKey()
|
||||
{
|
||||
return maxKey;
|
||||
}
|
||||
|
||||
public DataTerm min()
|
||||
{
|
||||
return dataLevel.getBlock(0).getTerm(0);
|
||||
}
|
||||
|
||||
public DataTerm max()
|
||||
{
|
||||
DataBlock block = dataLevel.getBlock(dataLevel.blockCount - 1);
|
||||
return block.getTerm(block.termCount() - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for rows which match all of the terms inside the given expression in the index file.
|
||||
*
|
||||
* @param exp The expression to use for the query.
|
||||
*
|
||||
* @return Iterator which contains rows for all of the terms from the given range.
|
||||
*/
|
||||
public RangeIterator<Long, Token> search(Expression exp)
|
||||
{
|
||||
// convert single NOT_EQ to range with exclusion
|
||||
final Expression expression = (exp.getOp() != Op.NOT_EQ)
|
||||
? exp
|
||||
: new Expression(exp).setOp(Op.RANGE)
|
||||
.setLower(new Expression.Bound(minTerm, true))
|
||||
.setUpper(new Expression.Bound(maxTerm, true))
|
||||
.addExclusion(exp.lower.value);
|
||||
|
||||
List<ByteBuffer> exclusions = new ArrayList<>(expression.exclusions.size());
|
||||
|
||||
Iterables.addAll(exclusions, expression.exclusions.stream().filter(exclusion -> {
|
||||
// accept only exclusions which are in the bounds of lower/upper
|
||||
return !(expression.lower != null && comparator.compare(exclusion, expression.lower.value) < 0)
|
||||
&& !(expression.upper != null && comparator.compare(exclusion, expression.upper.value) > 0);
|
||||
}).collect(Collectors.toList()));
|
||||
|
||||
Collections.sort(exclusions, comparator);
|
||||
|
||||
if (exclusions.size() == 0)
|
||||
return searchRange(expression);
|
||||
|
||||
List<Expression> ranges = new ArrayList<>(exclusions.size());
|
||||
|
||||
// calculate range splits based on the sorted exclusions
|
||||
Iterator<ByteBuffer> exclusionsIterator = exclusions.iterator();
|
||||
|
||||
Expression.Bound min = expression.lower, max = null;
|
||||
while (exclusionsIterator.hasNext())
|
||||
{
|
||||
max = new Expression.Bound(exclusionsIterator.next(), false);
|
||||
ranges.add(new Expression(expression).setOp(Op.RANGE).setLower(min).setUpper(max));
|
||||
min = max;
|
||||
}
|
||||
|
||||
assert max != null;
|
||||
ranges.add(new Expression(expression).setOp(Op.RANGE).setLower(max).setUpper(expression.upper));
|
||||
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
for (Expression e : ranges)
|
||||
{
|
||||
RangeIterator<Long, Token> range = searchRange(e);
|
||||
if (range != null)
|
||||
builder.add(range);
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private RangeIterator<Long, Token> searchRange(Expression range)
|
||||
{
|
||||
Expression.Bound lower = range.lower;
|
||||
Expression.Bound upper = range.upper;
|
||||
|
||||
int lowerBlock = lower == null ? 0 : getDataBlock(lower.value);
|
||||
int upperBlock = upper == null
|
||||
? dataLevel.blockCount - 1
|
||||
// optimization so we don't have to fetch upperBlock when query has lower == upper
|
||||
: (lower != null && comparator.compare(lower.value, upper.value) == 0) ? lowerBlock : getDataBlock(upper.value);
|
||||
|
||||
return (mode != OnDiskIndexBuilder.Mode.SPARSE || lowerBlock == upperBlock || upperBlock - lowerBlock <= 1)
|
||||
? searchPoint(lowerBlock, range)
|
||||
: searchRange(lowerBlock, lower, upperBlock, upper);
|
||||
}
|
||||
|
||||
private RangeIterator<Long, Token> searchRange(int lowerBlock, Expression.Bound lower, int upperBlock, Expression.Bound upper)
|
||||
{
|
||||
// if lower is at the beginning of the block that means we can just do a single iterator per block
|
||||
SearchResult<DataTerm> lowerPosition = (lower == null) ? null : searchIndex(lower.value, lowerBlock);
|
||||
SearchResult<DataTerm> upperPosition = (upper == null) ? null : searchIndex(upper.value, upperBlock);
|
||||
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
|
||||
// optimistically assume that first and last blocks are full block reads, saves at least 3 'else' conditions
|
||||
int firstFullBlockIdx = lowerBlock, lastFullBlockIdx = upperBlock;
|
||||
|
||||
// 'lower' doesn't cover the whole block so we need to do a partial iteration
|
||||
// Two reasons why that can happen:
|
||||
// - 'lower' is not the first element of the block
|
||||
// - 'lower' is first element but it's not inclusive in the query
|
||||
if (lowerPosition != null && (lowerPosition.index > 0 || !lower.inclusive))
|
||||
{
|
||||
DataBlock block = dataLevel.getBlock(lowerBlock);
|
||||
int start = (lower.inclusive || lowerPosition.cmp != 0) ? lowerPosition.index : lowerPosition.index + 1;
|
||||
|
||||
builder.add(block.getRange(start, block.termCount()));
|
||||
firstFullBlockIdx = lowerBlock + 1;
|
||||
}
|
||||
|
||||
if (upperPosition != null)
|
||||
{
|
||||
DataBlock block = dataLevel.getBlock(upperBlock);
|
||||
int lastIndex = block.termCount() - 1;
|
||||
|
||||
// The save as with 'lower' but here we need to check if the upper is the last element of the block,
|
||||
// which means that we only have to get individual results if:
|
||||
// - if it *is not* the last element, or
|
||||
// - it *is* but shouldn't be included (dictated by upperInclusive)
|
||||
if (upperPosition.index != lastIndex || !upper.inclusive)
|
||||
{
|
||||
int end = (upperPosition.cmp < 0 || (upperPosition.cmp == 0 && upper.inclusive))
|
||||
? upperPosition.index + 1 : upperPosition.index;
|
||||
|
||||
builder.add(block.getRange(0, end));
|
||||
lastFullBlockIdx = upperBlock - 1;
|
||||
}
|
||||
}
|
||||
|
||||
int totalSuperBlocks = (lastFullBlockIdx - firstFullBlockIdx) / OnDiskIndexBuilder.SUPER_BLOCK_SIZE;
|
||||
|
||||
// if there are no super-blocks, we can simply read all of the block iterators in sequence
|
||||
if (totalSuperBlocks == 0)
|
||||
{
|
||||
for (int i = firstFullBlockIdx; i <= lastFullBlockIdx; i++)
|
||||
builder.add(dataLevel.getBlock(i).getBlockIndex().iterator(keyFetcher));
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
// first get all of the blocks which are aligned before the first super-block in the sequence,
|
||||
// e.g. if the block range was (1, 9) and super-block-size = 4, we need to read 1, 2, 3, 4 - 7 is covered by
|
||||
// super-block, 8, 9 is a remainder.
|
||||
|
||||
int superBlockAlignedStart = firstFullBlockIdx == 0 ? 0 : (int) FBUtilities.align(firstFullBlockIdx, OnDiskIndexBuilder.SUPER_BLOCK_SIZE);
|
||||
for (int blockIdx = firstFullBlockIdx; blockIdx < Math.min(superBlockAlignedStart, lastFullBlockIdx); blockIdx++)
|
||||
builder.add(getBlockIterator(blockIdx));
|
||||
|
||||
// now read all of the super-blocks matched by the request, from the previous comment
|
||||
// it's a block with index 1 (which covers everything from 4 to 7)
|
||||
|
||||
int superBlockIdx = superBlockAlignedStart / OnDiskIndexBuilder.SUPER_BLOCK_SIZE;
|
||||
for (int offset = 0; offset < totalSuperBlocks - 1; offset++)
|
||||
builder.add(dataLevel.getSuperBlock(superBlockIdx++).iterator());
|
||||
|
||||
// now it's time for a remainder read, again from the previous example it's 8, 9 because
|
||||
// we have over-shot previous block but didn't request enough to cover next super-block.
|
||||
|
||||
int lastCoveredBlock = superBlockIdx * OnDiskIndexBuilder.SUPER_BLOCK_SIZE;
|
||||
for (int offset = 0; offset <= (lastFullBlockIdx - lastCoveredBlock); offset++)
|
||||
builder.add(getBlockIterator(lastCoveredBlock + offset));
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private RangeIterator<Long, Token> searchPoint(int lowerBlock, Expression expression)
|
||||
{
|
||||
Iterator<DataTerm> terms = new TermIterator(lowerBlock, expression, IteratorOrder.DESC);
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
|
||||
while (terms.hasNext())
|
||||
{
|
||||
try
|
||||
{
|
||||
builder.add(terms.next().getTokens());
|
||||
}
|
||||
finally
|
||||
{
|
||||
expression.checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
private RangeIterator<Long, Token> getBlockIterator(int blockIdx)
|
||||
{
|
||||
DataBlock block = dataLevel.getBlock(blockIdx);
|
||||
return (block.hasCombinedIndex)
|
||||
? block.getBlockIndex().iterator(keyFetcher)
|
||||
: block.getRange(0, block.termCount());
|
||||
}
|
||||
|
||||
public Iterator<DataTerm> iteratorAt(ByteBuffer query, IteratorOrder order, boolean inclusive)
|
||||
{
|
||||
Expression e = new Expression("", comparator);
|
||||
Expression.Bound bound = new Expression.Bound(query, inclusive);
|
||||
|
||||
switch (order)
|
||||
{
|
||||
case DESC:
|
||||
e.setLower(bound);
|
||||
break;
|
||||
|
||||
case ASC:
|
||||
e.setUpper(bound);
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalArgumentException("Unknown order: " + order);
|
||||
}
|
||||
|
||||
return new TermIterator(levels.length == 0 ? 0 : getBlockIdx(findPointer(query), query), e, order);
|
||||
}
|
||||
|
||||
private int getDataBlock(ByteBuffer query)
|
||||
{
|
||||
return levels.length == 0 ? 0 : getBlockIdx(findPointer(query), query);
|
||||
}
|
||||
|
||||
public Iterator<DataTerm> iterator()
|
||||
{
|
||||
return new TermIterator(0, new Expression("", comparator), IteratorOrder.DESC);
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
FileUtils.closeQuietly(indexFile);
|
||||
}
|
||||
|
||||
private PointerTerm findPointer(ByteBuffer query)
|
||||
{
|
||||
PointerTerm ptr = null;
|
||||
for (PointerLevel level : levels)
|
||||
{
|
||||
if ((ptr = level.getPointer(ptr, query)) == null)
|
||||
return null;
|
||||
}
|
||||
|
||||
return ptr;
|
||||
}
|
||||
|
||||
private SearchResult<DataTerm> searchIndex(ByteBuffer query, int blockIdx)
|
||||
{
|
||||
return dataLevel.getBlock(blockIdx).search(comparator, query);
|
||||
}
|
||||
|
||||
private int getBlockIdx(PointerTerm ptr, ByteBuffer query)
|
||||
{
|
||||
int blockIdx = 0;
|
||||
if (ptr != null)
|
||||
{
|
||||
int cmp = ptr.compareTo(comparator, query);
|
||||
blockIdx = (cmp == 0 || cmp > 0) ? ptr.getBlock() : ptr.getBlock() + 1;
|
||||
}
|
||||
|
||||
return blockIdx;
|
||||
}
|
||||
|
||||
protected class PointerLevel extends Level<PointerBlock>
|
||||
{
|
||||
public PointerLevel(long offset, int count)
|
||||
{
|
||||
super(offset, count);
|
||||
}
|
||||
|
||||
public PointerTerm getPointer(PointerTerm parent, ByteBuffer query)
|
||||
{
|
||||
return getBlock(getBlockIdx(parent, query)).search(comparator, query).result;
|
||||
}
|
||||
|
||||
protected PointerBlock cast(MappedBuffer block)
|
||||
{
|
||||
return new PointerBlock(block);
|
||||
}
|
||||
}
|
||||
|
||||
protected class DataLevel extends Level<DataBlock>
|
||||
{
|
||||
protected final int superBlockCnt;
|
||||
protected final long superBlocksOffset;
|
||||
|
||||
public DataLevel(long offset, int count)
|
||||
{
|
||||
super(offset, count);
|
||||
long baseOffset = blockOffsets + blockCount * 8;
|
||||
superBlockCnt = indexFile.getInt(baseOffset);
|
||||
superBlocksOffset = baseOffset + 4;
|
||||
}
|
||||
|
||||
protected DataBlock cast(MappedBuffer block)
|
||||
{
|
||||
return new DataBlock(block);
|
||||
}
|
||||
|
||||
public OnDiskSuperBlock getSuperBlock(int idx)
|
||||
{
|
||||
assert idx < superBlockCnt : String.format("requested index %d is greater than super block count %d", idx, superBlockCnt);
|
||||
long blockOffset = indexFile.getLong(superBlocksOffset + idx * 8);
|
||||
return new OnDiskSuperBlock(indexFile.duplicate().position(blockOffset));
|
||||
}
|
||||
}
|
||||
|
||||
protected class OnDiskSuperBlock
|
||||
{
|
||||
private final TokenTree tokenTree;
|
||||
|
||||
public OnDiskSuperBlock(MappedBuffer buffer)
|
||||
{
|
||||
tokenTree = new TokenTree(descriptor, buffer);
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> iterator()
|
||||
{
|
||||
return tokenTree.iterator(keyFetcher);
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract class Level<T extends OnDiskBlock>
|
||||
{
|
||||
protected final long blockOffsets;
|
||||
protected final int blockCount;
|
||||
|
||||
public Level(long offsets, int count)
|
||||
{
|
||||
this.blockOffsets = offsets;
|
||||
this.blockCount = count;
|
||||
}
|
||||
|
||||
public T getBlock(int idx) throws FSReadError
|
||||
{
|
||||
assert idx >= 0 && idx < blockCount;
|
||||
|
||||
// calculate block offset and move there
|
||||
// (long is intentional, we'll just need mmap implementation which supports long positions)
|
||||
long blockOffset = indexFile.getLong(blockOffsets + idx * 8);
|
||||
return cast(indexFile.duplicate().position(blockOffset));
|
||||
}
|
||||
|
||||
protected abstract T cast(MappedBuffer block);
|
||||
}
|
||||
|
||||
protected class DataBlock extends OnDiskBlock<DataTerm>
|
||||
{
|
||||
public DataBlock(MappedBuffer data)
|
||||
{
|
||||
super(descriptor, data, BlockType.DATA);
|
||||
}
|
||||
|
||||
protected DataTerm cast(MappedBuffer data)
|
||||
{
|
||||
return new DataTerm(data, termSize, getBlockIndex());
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> getRange(int start, int end)
|
||||
{
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
NavigableMap<Long, Token> sparse = new TreeMap<>();
|
||||
|
||||
for (int i = start; i < end; i++)
|
||||
{
|
||||
DataTerm term = getTerm(i);
|
||||
|
||||
if (term.isSparse())
|
||||
{
|
||||
NavigableMap<Long, Token> tokens = term.getSparseTokens();
|
||||
for (Map.Entry<Long, Token> t : tokens.entrySet())
|
||||
{
|
||||
Token token = sparse.get(t.getKey());
|
||||
if (token == null)
|
||||
sparse.put(t.getKey(), t.getValue());
|
||||
else
|
||||
token.merge(t.getValue());
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
builder.add(term.getTokens());
|
||||
}
|
||||
}
|
||||
|
||||
PrefetchedTokensIterator prefetched = sparse.isEmpty() ? null : new PrefetchedTokensIterator(sparse);
|
||||
|
||||
if (builder.rangeCount() == 0)
|
||||
return prefetched;
|
||||
|
||||
builder.add(prefetched);
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
|
||||
protected class PointerBlock extends OnDiskBlock<PointerTerm>
|
||||
{
|
||||
public PointerBlock(MappedBuffer block)
|
||||
{
|
||||
super(descriptor, block, BlockType.POINTER);
|
||||
}
|
||||
|
||||
protected PointerTerm cast(MappedBuffer data)
|
||||
{
|
||||
return new PointerTerm(data, termSize);
|
||||
}
|
||||
}
|
||||
|
||||
public class DataTerm extends Term implements Comparable<DataTerm>
|
||||
{
|
||||
private final TokenTree perBlockIndex;
|
||||
|
||||
protected DataTerm(MappedBuffer content, OnDiskIndexBuilder.TermSize size, TokenTree perBlockIndex)
|
||||
{
|
||||
super(content, size);
|
||||
this.perBlockIndex = perBlockIndex;
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> getTokens()
|
||||
{
|
||||
final long blockEnd = FBUtilities.align(content.position(), OnDiskIndexBuilder.BLOCK_SIZE);
|
||||
|
||||
if (isSparse())
|
||||
return new PrefetchedTokensIterator(getSparseTokens());
|
||||
|
||||
long offset = blockEnd + 4 + content.getInt(getDataOffset() + 1);
|
||||
return new TokenTree(descriptor, indexFile.duplicate().position(offset)).iterator(keyFetcher);
|
||||
}
|
||||
|
||||
public boolean isSparse()
|
||||
{
|
||||
return content.get(getDataOffset()) > 0;
|
||||
}
|
||||
|
||||
public NavigableMap<Long, Token> getSparseTokens()
|
||||
{
|
||||
long ptrOffset = getDataOffset();
|
||||
|
||||
byte size = content.get(ptrOffset);
|
||||
|
||||
assert size > 0;
|
||||
|
||||
NavigableMap<Long, Token> individualTokens = new TreeMap<>();
|
||||
for (int i = 0; i < size; i++)
|
||||
{
|
||||
Token token = perBlockIndex.get(content.getLong(ptrOffset + 1 + (8 * i)), keyFetcher);
|
||||
|
||||
assert token != null;
|
||||
individualTokens.put(token.get(), token);
|
||||
}
|
||||
|
||||
return individualTokens;
|
||||
}
|
||||
|
||||
public int compareTo(DataTerm other)
|
||||
{
|
||||
return other == null ? 1 : compareTo(comparator, other.getTerm());
|
||||
}
|
||||
}
|
||||
|
||||
protected static class PointerTerm extends Term
|
||||
{
|
||||
public PointerTerm(MappedBuffer content, OnDiskIndexBuilder.TermSize size)
|
||||
{
|
||||
super(content, size);
|
||||
}
|
||||
|
||||
public int getBlock()
|
||||
{
|
||||
return content.getInt(getDataOffset());
|
||||
}
|
||||
}
|
||||
|
||||
private static class PrefetchedTokensIterator extends RangeIterator<Long, Token>
|
||||
{
|
||||
private final NavigableMap<Long, Token> tokens;
|
||||
private PeekingIterator<Token> currentIterator;
|
||||
|
||||
public PrefetchedTokensIterator(NavigableMap<Long, Token> tokens)
|
||||
{
|
||||
super(tokens.firstKey(), tokens.lastKey(), tokens.size());
|
||||
this.tokens = tokens;
|
||||
this.currentIterator = Iterators.peekingIterator(tokens.values().iterator());
|
||||
}
|
||||
|
||||
protected Token computeNext()
|
||||
{
|
||||
return currentIterator != null && currentIterator.hasNext()
|
||||
? currentIterator.next()
|
||||
: endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(Long nextToken)
|
||||
{
|
||||
currentIterator = Iterators.peekingIterator(tokens.tailMap(nextToken, true).values().iterator());
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
endOfData();
|
||||
}
|
||||
}
|
||||
|
||||
public AbstractType<?> getComparator()
|
||||
{
|
||||
return comparator;
|
||||
}
|
||||
|
||||
public String getIndexPath()
|
||||
{
|
||||
return indexPath;
|
||||
}
|
||||
|
||||
private class TermIterator extends AbstractIterator<DataTerm>
|
||||
{
|
||||
private final Expression e;
|
||||
private final IteratorOrder order;
|
||||
|
||||
protected OnDiskBlock<DataTerm> currentBlock;
|
||||
protected int blockIndex, offset;
|
||||
|
||||
private boolean checkLower = true, checkUpper = true;
|
||||
|
||||
public TermIterator(int startBlock, Expression expression, IteratorOrder order)
|
||||
{
|
||||
this.e = expression;
|
||||
this.order = order;
|
||||
this.blockIndex = startBlock;
|
||||
|
||||
nextBlock();
|
||||
}
|
||||
|
||||
protected DataTerm computeNext()
|
||||
{
|
||||
for (;;)
|
||||
{
|
||||
if (currentBlock == null)
|
||||
return endOfData();
|
||||
|
||||
if (offset >= 0 && offset < currentBlock.termCount())
|
||||
{
|
||||
DataTerm currentTerm = currentBlock.getTerm(nextOffset());
|
||||
|
||||
if (checkLower && !e.isLowerSatisfiedBy(currentTerm))
|
||||
continue;
|
||||
|
||||
// flip the flag right on the first bounds match
|
||||
// to avoid expensive comparisons
|
||||
checkLower = false;
|
||||
|
||||
if (checkUpper && !e.isUpperSatisfiedBy(currentTerm))
|
||||
return endOfData();
|
||||
|
||||
return currentTerm;
|
||||
}
|
||||
|
||||
nextBlock();
|
||||
}
|
||||
}
|
||||
|
||||
protected void nextBlock()
|
||||
{
|
||||
currentBlock = null;
|
||||
|
||||
if (blockIndex < 0 || blockIndex >= dataLevel.blockCount)
|
||||
return;
|
||||
|
||||
currentBlock = dataLevel.getBlock(nextBlockIndex());
|
||||
offset = checkLower ? order.startAt(currentBlock, e) : currentBlock.minOffset(order);
|
||||
|
||||
// let's check the last term of the new block right away
|
||||
// if expression's upper bound is satisfied by it such means that we can avoid
|
||||
// doing any expensive upper bound checks for that block.
|
||||
checkUpper = e.hasUpper() && !e.isUpperSatisfiedBy(currentBlock.getTerm(currentBlock.maxOffset(order)));
|
||||
}
|
||||
|
||||
protected int nextBlockIndex()
|
||||
{
|
||||
int current = blockIndex;
|
||||
blockIndex += order.step;
|
||||
return current;
|
||||
}
|
||||
|
||||
protected int nextOffset()
|
||||
{
|
||||
int current = offset;
|
||||
offset += order.step;
|
||||
return current;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,627 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.sa.IntegralSA;
|
||||
import org.apache.cassandra.index.sasi.sa.SA;
|
||||
import org.apache.cassandra.index.sasi.sa.TermIterator;
|
||||
import org.apache.cassandra.index.sasi.sa.SuffixSA;
|
||||
import org.apache.cassandra.db.marshal.*;
|
||||
import org.apache.cassandra.io.FSWriteError;
|
||||
import org.apache.cassandra.io.compress.BufferType;
|
||||
import org.apache.cassandra.io.util.*;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
import com.carrotsearch.hppc.LongArrayList;
|
||||
import com.carrotsearch.hppc.LongSet;
|
||||
import com.carrotsearch.hppc.ShortArrayList;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class OnDiskIndexBuilder
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(OnDiskIndexBuilder.class);
|
||||
|
||||
public enum Mode
|
||||
{
|
||||
PREFIX, CONTAINS, SPARSE;
|
||||
|
||||
public static Mode mode(String mode)
|
||||
{
|
||||
return Mode.valueOf(mode.toUpperCase());
|
||||
}
|
||||
}
|
||||
|
||||
public enum TermSize
|
||||
{
|
||||
INT(4), LONG(8), UUID(16), VARIABLE(-1);
|
||||
|
||||
public final int size;
|
||||
|
||||
TermSize(int size)
|
||||
{
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
public boolean isConstant()
|
||||
{
|
||||
return this != VARIABLE;
|
||||
}
|
||||
|
||||
public static TermSize of(int size)
|
||||
{
|
||||
switch (size)
|
||||
{
|
||||
case -1:
|
||||
return VARIABLE;
|
||||
|
||||
case 4:
|
||||
return INT;
|
||||
|
||||
case 8:
|
||||
return LONG;
|
||||
|
||||
case 16:
|
||||
return UUID;
|
||||
|
||||
default:
|
||||
throw new IllegalStateException("unknown state: " + size);
|
||||
}
|
||||
}
|
||||
|
||||
public static TermSize sizeOf(AbstractType<?> comparator)
|
||||
{
|
||||
if (comparator instanceof Int32Type || comparator instanceof FloatType)
|
||||
return INT;
|
||||
|
||||
if (comparator instanceof LongType || comparator instanceof DoubleType
|
||||
|| comparator instanceof TimestampType || comparator instanceof DateType)
|
||||
return LONG;
|
||||
|
||||
if (comparator instanceof TimeUUIDType || comparator instanceof UUIDType)
|
||||
return UUID;
|
||||
|
||||
return VARIABLE;
|
||||
}
|
||||
}
|
||||
|
||||
public static final int BLOCK_SIZE = 4096;
|
||||
public static final int MAX_TERM_SIZE = 1024;
|
||||
public static final int SUPER_BLOCK_SIZE = 64;
|
||||
|
||||
private final List<MutableLevel<InMemoryPointerTerm>> levels = new ArrayList<>();
|
||||
private MutableLevel<InMemoryDataTerm> dataLevel;
|
||||
|
||||
private final TermSize termSize;
|
||||
|
||||
private final AbstractType<?> keyComparator, termComparator;
|
||||
|
||||
private final Map<ByteBuffer, TokenTreeBuilder> terms;
|
||||
private final Mode mode;
|
||||
|
||||
private ByteBuffer minKey, maxKey;
|
||||
private long estimatedBytes;
|
||||
|
||||
public OnDiskIndexBuilder(AbstractType<?> keyComparator, AbstractType<?> comparator, Mode mode)
|
||||
{
|
||||
this.keyComparator = keyComparator;
|
||||
this.termComparator = comparator;
|
||||
this.terms = new HashMap<>();
|
||||
this.termSize = TermSize.sizeOf(comparator);
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public OnDiskIndexBuilder add(ByteBuffer term, DecoratedKey key, long keyPosition)
|
||||
{
|
||||
if (term.remaining() >= MAX_TERM_SIZE)
|
||||
{
|
||||
logger.error("Rejecting value (value size {}, maximum size {} bytes).", term.remaining(), Short.MAX_VALUE);
|
||||
return this;
|
||||
}
|
||||
|
||||
TokenTreeBuilder tokens = terms.get(term);
|
||||
if (tokens == null)
|
||||
{
|
||||
terms.put(term, (tokens = new TokenTreeBuilder()));
|
||||
|
||||
// on-heap size estimates from jol
|
||||
// 64 bytes for TTB + 48 bytes for TreeMap in TTB + size bytes for the term (map key)
|
||||
estimatedBytes += 64 + 48 + term.remaining();
|
||||
}
|
||||
|
||||
tokens.add((Long) key.getToken().getTokenValue(), keyPosition);
|
||||
|
||||
// calculate key range (based on actual key values) for current index
|
||||
minKey = (minKey == null || keyComparator.compare(minKey, key.getKey()) > 0) ? key.getKey() : minKey;
|
||||
maxKey = (maxKey == null || keyComparator.compare(maxKey, key.getKey()) < 0) ? key.getKey() : maxKey;
|
||||
|
||||
// 60 ((boolean(1)*4) + (long(8)*4) + 24) bytes for the LongOpenHashSet created when the keyPosition was added
|
||||
// + 40 bytes for the TreeMap.Entry + 8 bytes for the token (key).
|
||||
// in the case of hash collision for the token we may overestimate but this is extremely rare
|
||||
estimatedBytes += 60 + 40 + 8;
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public long estimatedMemoryUse()
|
||||
{
|
||||
return estimatedBytes;
|
||||
}
|
||||
|
||||
private void addTerm(InMemoryDataTerm term, SequentialWriter out) throws IOException
|
||||
{
|
||||
InMemoryPointerTerm ptr = dataLevel.add(term);
|
||||
if (ptr == null)
|
||||
return;
|
||||
|
||||
int levelIdx = 0;
|
||||
for (;;)
|
||||
{
|
||||
MutableLevel<InMemoryPointerTerm> level = getIndexLevel(levelIdx++, out);
|
||||
if ((ptr = level.add(ptr)) == null)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isEmpty()
|
||||
{
|
||||
return terms.isEmpty();
|
||||
}
|
||||
|
||||
public void finish(Pair<ByteBuffer, ByteBuffer> range, File file, TermIterator terms)
|
||||
{
|
||||
finish(Descriptor.CURRENT, range, file, terms);
|
||||
}
|
||||
|
||||
/**
|
||||
* Finishes up index building process by creating/populating index file.
|
||||
*
|
||||
* @param indexFile The file to write index contents to.
|
||||
*
|
||||
* @return true if index was written successfully, false otherwise (e.g. if index was empty).
|
||||
*
|
||||
* @throws FSWriteError on I/O error.
|
||||
*/
|
||||
public boolean finish(File indexFile) throws FSWriteError
|
||||
{
|
||||
return finish(Descriptor.CURRENT, indexFile);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected boolean finish(Descriptor descriptor, File file) throws FSWriteError
|
||||
{
|
||||
// no terms means there is nothing to build
|
||||
if (terms.isEmpty())
|
||||
return false;
|
||||
|
||||
// split terms into suffixes only if it's text, otherwise (even if CONTAINS is set) use terms in original form
|
||||
SA sa = ((termComparator instanceof UTF8Type || termComparator instanceof AsciiType) && mode == Mode.CONTAINS)
|
||||
? new SuffixSA(termComparator, mode) : new IntegralSA(termComparator, mode);
|
||||
|
||||
for (Map.Entry<ByteBuffer, TokenTreeBuilder> term : terms.entrySet())
|
||||
sa.add(term.getKey(), term.getValue());
|
||||
|
||||
finish(descriptor, Pair.create(minKey, maxKey), file, sa.finish());
|
||||
return true;
|
||||
}
|
||||
|
||||
protected void finish(Descriptor descriptor, Pair<ByteBuffer, ByteBuffer> range, File file, TermIterator terms)
|
||||
{
|
||||
SequentialWriter out = null;
|
||||
|
||||
try
|
||||
{
|
||||
out = new SequentialWriter(file, BLOCK_SIZE, BufferType.ON_HEAP);
|
||||
|
||||
out.writeUTF(descriptor.version.toString());
|
||||
|
||||
out.writeShort(termSize.size);
|
||||
|
||||
// min, max term (useful to find initial scan range from search expressions)
|
||||
ByteBufferUtil.writeWithShortLength(terms.minTerm(), out);
|
||||
ByteBufferUtil.writeWithShortLength(terms.maxTerm(), out);
|
||||
|
||||
// min, max keys covered by index (useful when searching across multiple indexes)
|
||||
ByteBufferUtil.writeWithShortLength(range.left, out);
|
||||
ByteBufferUtil.writeWithShortLength(range.right, out);
|
||||
|
||||
out.writeUTF(mode.toString());
|
||||
|
||||
out.skipBytes((int) (BLOCK_SIZE - out.position()));
|
||||
|
||||
dataLevel = mode == Mode.SPARSE ? new DataBuilderLevel(out, new MutableDataBlock(mode))
|
||||
: new MutableLevel<>(out, new MutableDataBlock(mode));
|
||||
while (terms.hasNext())
|
||||
{
|
||||
Pair<ByteBuffer, TokenTreeBuilder> term = terms.next();
|
||||
addTerm(new InMemoryDataTerm(term.left, term.right), out);
|
||||
}
|
||||
|
||||
dataLevel.finalFlush();
|
||||
for (MutableLevel l : levels)
|
||||
l.flush(); // flush all of the buffers
|
||||
|
||||
// and finally write levels index
|
||||
final long levelIndexPosition = out.position();
|
||||
|
||||
out.writeInt(levels.size());
|
||||
for (int i = levels.size() - 1; i >= 0; i--)
|
||||
levels.get(i).flushMetadata();
|
||||
|
||||
dataLevel.flushMetadata();
|
||||
|
||||
out.writeLong(levelIndexPosition);
|
||||
|
||||
// sync contents of the output and disk,
|
||||
// since it's not done implicitly on close
|
||||
out.sync();
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new FSWriteError(e, file);
|
||||
}
|
||||
finally
|
||||
{
|
||||
FileUtils.closeQuietly(out);
|
||||
}
|
||||
}
|
||||
|
||||
private MutableLevel<InMemoryPointerTerm> getIndexLevel(int idx, SequentialWriter out)
|
||||
{
|
||||
if (levels.size() == 0)
|
||||
levels.add(new MutableLevel<>(out, new MutableBlock<>()));
|
||||
|
||||
if (levels.size() - 1 < idx)
|
||||
{
|
||||
int toAdd = idx - (levels.size() - 1);
|
||||
for (int i = 0; i < toAdd; i++)
|
||||
levels.add(new MutableLevel<>(out, new MutableBlock<>()));
|
||||
}
|
||||
|
||||
return levels.get(idx);
|
||||
}
|
||||
|
||||
protected static void alignToBlock(SequentialWriter out) throws IOException
|
||||
{
|
||||
long endOfBlock = out.position();
|
||||
if ((endOfBlock & (BLOCK_SIZE - 1)) != 0) // align on the block boundary if needed
|
||||
out.skipBytes((int) (FBUtilities.align(endOfBlock, BLOCK_SIZE) - endOfBlock));
|
||||
}
|
||||
|
||||
private class InMemoryTerm
|
||||
{
|
||||
protected final ByteBuffer term;
|
||||
|
||||
public InMemoryTerm(ByteBuffer term)
|
||||
{
|
||||
this.term = term;
|
||||
}
|
||||
|
||||
public int serializedSize()
|
||||
{
|
||||
return (termSize.isConstant() ? 0 : 2) + term.remaining();
|
||||
}
|
||||
|
||||
public void serialize(DataOutputPlus out) throws IOException
|
||||
{
|
||||
if (termSize.isConstant())
|
||||
out.write(term);
|
||||
else
|
||||
ByteBufferUtil.writeWithShortLength(term, out);
|
||||
}
|
||||
}
|
||||
|
||||
private class InMemoryPointerTerm extends InMemoryTerm
|
||||
{
|
||||
protected final int blockCnt;
|
||||
|
||||
public InMemoryPointerTerm(ByteBuffer term, int blockCnt)
|
||||
{
|
||||
super(term);
|
||||
this.blockCnt = blockCnt;
|
||||
}
|
||||
|
||||
public int serializedSize()
|
||||
{
|
||||
return super.serializedSize() + 4;
|
||||
}
|
||||
|
||||
public void serialize(DataOutputPlus out) throws IOException
|
||||
{
|
||||
super.serialize(out);
|
||||
out.writeInt(blockCnt);
|
||||
}
|
||||
}
|
||||
|
||||
private class InMemoryDataTerm extends InMemoryTerm
|
||||
{
|
||||
private final TokenTreeBuilder keys;
|
||||
|
||||
public InMemoryDataTerm(ByteBuffer term, TokenTreeBuilder keys)
|
||||
{
|
||||
super(term);
|
||||
this.keys = keys;
|
||||
}
|
||||
}
|
||||
|
||||
private class MutableLevel<T extends InMemoryTerm>
|
||||
{
|
||||
private final LongArrayList blockOffsets = new LongArrayList();
|
||||
|
||||
protected final SequentialWriter out;
|
||||
|
||||
private final MutableBlock<T> inProcessBlock;
|
||||
private InMemoryPointerTerm lastTerm;
|
||||
|
||||
public MutableLevel(SequentialWriter out, MutableBlock<T> block)
|
||||
{
|
||||
this.out = out;
|
||||
this.inProcessBlock = block;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return If we flushed a block, return the last term of that block; else, null.
|
||||
*/
|
||||
public InMemoryPointerTerm add(T term) throws IOException
|
||||
{
|
||||
InMemoryPointerTerm toPromote = null;
|
||||
|
||||
if (!inProcessBlock.hasSpaceFor(term))
|
||||
{
|
||||
flush();
|
||||
toPromote = lastTerm;
|
||||
}
|
||||
|
||||
inProcessBlock.add(term);
|
||||
|
||||
lastTerm = new InMemoryPointerTerm(term.term, blockOffsets.size());
|
||||
return toPromote;
|
||||
}
|
||||
|
||||
public void flush() throws IOException
|
||||
{
|
||||
blockOffsets.add(out.position());
|
||||
inProcessBlock.flushAndClear(out);
|
||||
}
|
||||
|
||||
public void finalFlush() throws IOException
|
||||
{
|
||||
flush();
|
||||
}
|
||||
|
||||
public void flushMetadata() throws IOException
|
||||
{
|
||||
flushMetadata(blockOffsets);
|
||||
}
|
||||
|
||||
protected void flushMetadata(LongArrayList longArrayList) throws IOException
|
||||
{
|
||||
out.writeInt(longArrayList.size());
|
||||
for (int i = 0; i < longArrayList.size(); i++)
|
||||
out.writeLong(longArrayList.get(i));
|
||||
}
|
||||
}
|
||||
|
||||
/** builds standard data blocks and super blocks, as well */
|
||||
private class DataBuilderLevel extends MutableLevel<InMemoryDataTerm>
|
||||
{
|
||||
private final LongArrayList superBlockOffsets = new LongArrayList();
|
||||
|
||||
/** count of regular data blocks written since current super block was init'd */
|
||||
private int dataBlocksCnt;
|
||||
private TokenTreeBuilder superBlockTree;
|
||||
|
||||
public DataBuilderLevel(SequentialWriter out, MutableBlock<InMemoryDataTerm> block)
|
||||
{
|
||||
super(out, block);
|
||||
superBlockTree = new TokenTreeBuilder();
|
||||
}
|
||||
|
||||
public InMemoryPointerTerm add(InMemoryDataTerm term) throws IOException
|
||||
{
|
||||
InMemoryPointerTerm ptr = super.add(term);
|
||||
if (ptr != null)
|
||||
{
|
||||
dataBlocksCnt++;
|
||||
flushSuperBlock(false);
|
||||
}
|
||||
superBlockTree.add(term.keys.getTokens());
|
||||
return ptr;
|
||||
}
|
||||
|
||||
public void flushSuperBlock(boolean force) throws IOException
|
||||
{
|
||||
if (dataBlocksCnt == SUPER_BLOCK_SIZE || (force && !superBlockTree.getTokens().isEmpty()))
|
||||
{
|
||||
superBlockOffsets.add(out.position());
|
||||
superBlockTree.finish().write(out);
|
||||
alignToBlock(out);
|
||||
|
||||
dataBlocksCnt = 0;
|
||||
superBlockTree = new TokenTreeBuilder();
|
||||
}
|
||||
}
|
||||
|
||||
public void finalFlush() throws IOException
|
||||
{
|
||||
super.flush();
|
||||
flushSuperBlock(true);
|
||||
}
|
||||
|
||||
public void flushMetadata() throws IOException
|
||||
{
|
||||
super.flushMetadata();
|
||||
flushMetadata(superBlockOffsets);
|
||||
}
|
||||
}
|
||||
|
||||
private static class MutableBlock<T extends InMemoryTerm>
|
||||
{
|
||||
protected final DataOutputBufferFixed buffer;
|
||||
protected final ShortArrayList offsets;
|
||||
|
||||
public MutableBlock()
|
||||
{
|
||||
buffer = new DataOutputBufferFixed(BLOCK_SIZE);
|
||||
offsets = new ShortArrayList();
|
||||
}
|
||||
|
||||
public final void add(T term) throws IOException
|
||||
{
|
||||
offsets.add((short) buffer.position());
|
||||
addInternal(term);
|
||||
}
|
||||
|
||||
protected void addInternal(T term) throws IOException
|
||||
{
|
||||
term.serialize(buffer);
|
||||
}
|
||||
|
||||
public boolean hasSpaceFor(T element)
|
||||
{
|
||||
return sizeAfter(element) < BLOCK_SIZE;
|
||||
}
|
||||
|
||||
protected int sizeAfter(T element)
|
||||
{
|
||||
return getWatermark() + 4 + element.serializedSize();
|
||||
}
|
||||
|
||||
protected int getWatermark()
|
||||
{
|
||||
return 4 + offsets.size() * 2 + (int) buffer.position();
|
||||
}
|
||||
|
||||
public void flushAndClear(SequentialWriter out) throws IOException
|
||||
{
|
||||
out.writeInt(offsets.size());
|
||||
for (int i = 0; i < offsets.size(); i++)
|
||||
out.writeShort(offsets.get(i));
|
||||
|
||||
out.write(buffer.buffer());
|
||||
|
||||
alignToBlock(out);
|
||||
|
||||
offsets.clear();
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
|
||||
private static class MutableDataBlock extends MutableBlock<InMemoryDataTerm>
|
||||
{
|
||||
private final Mode mode;
|
||||
|
||||
private int offset = 0;
|
||||
private int sparseValueTerms = 0;
|
||||
|
||||
private final List<TokenTreeBuilder> containers = new ArrayList<>();
|
||||
private TokenTreeBuilder combinedIndex;
|
||||
|
||||
public MutableDataBlock(Mode mode)
|
||||
{
|
||||
this.mode = mode;
|
||||
this.combinedIndex = new TokenTreeBuilder();
|
||||
}
|
||||
|
||||
protected void addInternal(InMemoryDataTerm term) throws IOException
|
||||
{
|
||||
TokenTreeBuilder keys = term.keys;
|
||||
|
||||
if (mode == Mode.SPARSE && keys.getTokenCount() <= 5)
|
||||
{
|
||||
writeTerm(term, keys);
|
||||
sparseValueTerms++;
|
||||
}
|
||||
else
|
||||
{
|
||||
writeTerm(term, offset);
|
||||
|
||||
offset += keys.serializedSize();
|
||||
containers.add(keys);
|
||||
}
|
||||
|
||||
if (mode == Mode.SPARSE)
|
||||
combinedIndex.add(keys.getTokens());
|
||||
}
|
||||
|
||||
protected int sizeAfter(InMemoryDataTerm element)
|
||||
{
|
||||
return super.sizeAfter(element) + ptrLength(element);
|
||||
}
|
||||
|
||||
public void flushAndClear(SequentialWriter out) throws IOException
|
||||
{
|
||||
super.flushAndClear(out);
|
||||
|
||||
out.writeInt((sparseValueTerms == 0) ? -1 : offset);
|
||||
|
||||
if (containers.size() > 0)
|
||||
{
|
||||
for (TokenTreeBuilder tokens : containers)
|
||||
tokens.write(out);
|
||||
}
|
||||
|
||||
if (sparseValueTerms > 0)
|
||||
{
|
||||
combinedIndex.finish().write(out);
|
||||
}
|
||||
|
||||
alignToBlock(out);
|
||||
|
||||
containers.clear();
|
||||
combinedIndex = new TokenTreeBuilder();
|
||||
|
||||
offset = 0;
|
||||
sparseValueTerms = 0;
|
||||
}
|
||||
|
||||
private int ptrLength(InMemoryDataTerm term)
|
||||
{
|
||||
return (term.keys.getTokenCount() > 5)
|
||||
? 5 // 1 byte type + 4 byte offset to the tree
|
||||
: 1 + (8 * (int) term.keys.getTokenCount()); // 1 byte size + n 8 byte tokens
|
||||
}
|
||||
|
||||
private void writeTerm(InMemoryTerm term, TokenTreeBuilder keys) throws IOException
|
||||
{
|
||||
term.serialize(buffer);
|
||||
buffer.writeByte((byte) keys.getTokenCount());
|
||||
|
||||
Iterator<Pair<Long, LongSet>> tokens = keys.iterator();
|
||||
while (tokens.hasNext())
|
||||
buffer.writeLong(tokens.next().left);
|
||||
}
|
||||
|
||||
private void writeTerm(InMemoryTerm term, int offset) throws IOException
|
||||
{
|
||||
term.serialize(buffer);
|
||||
buffer.writeByte(0x0);
|
||||
buffer.writeInt(offset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,361 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
import org.apache.cassandra.concurrent.JMXEnabledThreadPoolExecutor;
|
||||
import org.apache.cassandra.concurrent.NamedThreadFactory;
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.compaction.OperationType;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.db.rows.Unfiltered;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.utils.CombinedTermIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.TypeUtil;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.io.sstable.Descriptor;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableFlushObserver;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.util.concurrent.Futures;
|
||||
import com.google.common.util.concurrent.Uninterruptibles;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class PerSSTableIndexWriter implements SSTableFlushObserver
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(PerSSTableIndexWriter.class);
|
||||
|
||||
private static final ThreadPoolExecutor INDEX_FLUSHER_MEMTABLE;
|
||||
private static final ThreadPoolExecutor INDEX_FLUSHER_GENERAL;
|
||||
|
||||
static
|
||||
{
|
||||
INDEX_FLUSHER_GENERAL = new JMXEnabledThreadPoolExecutor(1, 8, 60, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(),
|
||||
new NamedThreadFactory("SASI-General"),
|
||||
"internal");
|
||||
INDEX_FLUSHER_GENERAL.allowCoreThreadTimeOut(true);
|
||||
|
||||
INDEX_FLUSHER_MEMTABLE = new JMXEnabledThreadPoolExecutor(1, 8, 60, TimeUnit.SECONDS,
|
||||
new LinkedBlockingQueue<>(),
|
||||
new NamedThreadFactory("SASI-Memtable"),
|
||||
"internal");
|
||||
INDEX_FLUSHER_MEMTABLE.allowCoreThreadTimeOut(true);
|
||||
}
|
||||
|
||||
private final int nowInSec = FBUtilities.nowInSeconds();
|
||||
|
||||
private final Descriptor descriptor;
|
||||
private final OperationType source;
|
||||
|
||||
private final AbstractType<?> keyValidator;
|
||||
private final Map<ColumnDefinition, ColumnIndex> supportedIndexes;
|
||||
|
||||
@VisibleForTesting
|
||||
protected final Map<ColumnDefinition, Index> indexes;
|
||||
|
||||
private DecoratedKey currentKey;
|
||||
private long currentKeyPosition;
|
||||
private boolean isComplete;
|
||||
|
||||
public PerSSTableIndexWriter(AbstractType<?> keyValidator,
|
||||
Descriptor descriptor,
|
||||
OperationType source,
|
||||
Map<ColumnDefinition, ColumnIndex> supportedIndexes)
|
||||
{
|
||||
this.keyValidator = keyValidator;
|
||||
this.descriptor = descriptor;
|
||||
this.source = source;
|
||||
this.supportedIndexes = supportedIndexes;
|
||||
this.indexes = new HashMap<>();
|
||||
}
|
||||
|
||||
public void begin()
|
||||
{}
|
||||
|
||||
public void startPartition(DecoratedKey key, long curPosition)
|
||||
{
|
||||
currentKey = key;
|
||||
currentKeyPosition = curPosition;
|
||||
}
|
||||
|
||||
public void nextUnfilteredCluster(Unfiltered unfiltered)
|
||||
{
|
||||
if (!unfiltered.isRow())
|
||||
return;
|
||||
|
||||
Row row = (Row) unfiltered;
|
||||
|
||||
supportedIndexes.keySet().forEach((column) -> {
|
||||
ByteBuffer value = ColumnIndex.getValueOf(column, row, nowInSec);
|
||||
if (value == null)
|
||||
return;
|
||||
|
||||
ColumnIndex columnIndex = supportedIndexes.get(column);
|
||||
if (columnIndex == null)
|
||||
return;
|
||||
|
||||
Index index = indexes.get(column);
|
||||
if (index == null)
|
||||
indexes.put(column, (index = new Index(columnIndex)));
|
||||
|
||||
index.add(value.duplicate(), currentKey, currentKeyPosition);
|
||||
});
|
||||
}
|
||||
|
||||
public void complete()
|
||||
{
|
||||
if (isComplete)
|
||||
return;
|
||||
|
||||
currentKey = null;
|
||||
|
||||
try
|
||||
{
|
||||
CountDownLatch latch = new CountDownLatch(indexes.size());
|
||||
for (Index index : indexes.values())
|
||||
index.complete(latch);
|
||||
|
||||
Uninterruptibles.awaitUninterruptibly(latch);
|
||||
}
|
||||
finally
|
||||
{
|
||||
indexes.clear();
|
||||
isComplete = true;
|
||||
}
|
||||
}
|
||||
|
||||
public Index getIndex(ColumnDefinition columnDef)
|
||||
{
|
||||
return indexes.get(columnDef);
|
||||
}
|
||||
|
||||
public Descriptor getDescriptor()
|
||||
{
|
||||
return descriptor;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected class Index
|
||||
{
|
||||
private final ColumnIndex columnIndex;
|
||||
private final String outputFile;
|
||||
private final AbstractAnalyzer analyzer;
|
||||
private final long maxMemorySize;
|
||||
|
||||
@VisibleForTesting
|
||||
protected final Set<Future<OnDiskIndex>> segments;
|
||||
private int segmentNumber = 0;
|
||||
|
||||
private OnDiskIndexBuilder currentBuilder;
|
||||
|
||||
public Index(ColumnIndex columnIndex)
|
||||
{
|
||||
this.columnIndex = columnIndex;
|
||||
this.outputFile = descriptor.filenameFor(columnIndex.getComponent());
|
||||
this.analyzer = columnIndex.getAnalyzer();
|
||||
this.segments = new HashSet<>();
|
||||
this.maxMemorySize = maxMemorySize(columnIndex);
|
||||
this.currentBuilder = newIndexBuilder();
|
||||
}
|
||||
|
||||
public void add(ByteBuffer term, DecoratedKey key, long keyPosition)
|
||||
{
|
||||
if (term.remaining() == 0)
|
||||
return;
|
||||
|
||||
boolean isAdded = false;
|
||||
|
||||
analyzer.reset(term);
|
||||
while (analyzer.hasNext())
|
||||
{
|
||||
ByteBuffer token = analyzer.next();
|
||||
int size = token.remaining();
|
||||
|
||||
if (token.remaining() >= OnDiskIndexBuilder.MAX_TERM_SIZE)
|
||||
{
|
||||
logger.info("Rejecting value (size {}, maximum {} bytes) for column {} (analyzed {}) at {} SSTable.",
|
||||
term.remaining(),
|
||||
OnDiskIndexBuilder.MAX_TERM_SIZE,
|
||||
columnIndex.getColumnName(),
|
||||
columnIndex.getMode().isAnalyzed,
|
||||
descriptor);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TypeUtil.isValid(token, columnIndex.getValidator()))
|
||||
{
|
||||
if ((token = TypeUtil.tryUpcast(token, columnIndex.getValidator())) == null)
|
||||
{
|
||||
logger.info("({}) Failed to add {} to index for key: {}, value size was {} bytes, validator is {}.",
|
||||
outputFile,
|
||||
columnIndex.getColumnName(),
|
||||
keyValidator.getString(key.getKey()),
|
||||
size,
|
||||
columnIndex.getValidator());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
currentBuilder.add(token, key, keyPosition);
|
||||
isAdded = true;
|
||||
}
|
||||
|
||||
if (!isAdded || currentBuilder.estimatedMemoryUse() < maxMemorySize)
|
||||
return; // non of the generated tokens were added to the index or memory size wasn't reached
|
||||
|
||||
segments.add(getExecutor().submit(scheduleSegmentFlush(false)));
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected Callable<OnDiskIndex> scheduleSegmentFlush(final boolean isFinal)
|
||||
{
|
||||
final OnDiskIndexBuilder builder = currentBuilder;
|
||||
currentBuilder = newIndexBuilder();
|
||||
|
||||
final String segmentFile = filename(isFinal);
|
||||
|
||||
return () -> {
|
||||
long start1 = System.nanoTime();
|
||||
|
||||
try
|
||||
{
|
||||
File index = new File(segmentFile);
|
||||
return builder.finish(index) ? new OnDiskIndex(index, columnIndex.getValidator(), null) : null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!isFinal)
|
||||
logger.info("Flushed index segment {}, took {} ms.", segmentFile, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start1));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public void complete(final CountDownLatch latch)
|
||||
{
|
||||
logger.info("Scheduling index flush to {}", outputFile);
|
||||
|
||||
getExecutor().submit((Runnable) () -> {
|
||||
long start1 = System.nanoTime();
|
||||
|
||||
OnDiskIndex[] parts = new OnDiskIndex[segments.size() + 1];
|
||||
|
||||
try
|
||||
{
|
||||
// no parts present, build entire index from memory
|
||||
if (segments.isEmpty())
|
||||
{
|
||||
scheduleSegmentFlush(true).call();
|
||||
return;
|
||||
}
|
||||
|
||||
// parts are present but there is something still in memory, let's flush that inline
|
||||
if (!currentBuilder.isEmpty())
|
||||
{
|
||||
OnDiskIndex last = scheduleSegmentFlush(false).call();
|
||||
segments.add(Futures.immediateFuture(last));
|
||||
}
|
||||
|
||||
int index = 0;
|
||||
ByteBuffer combinedMin = null, combinedMax = null;
|
||||
|
||||
for (Future<OnDiskIndex> f : segments)
|
||||
{
|
||||
OnDiskIndex part = Futures.getUnchecked(f);
|
||||
if (part == null)
|
||||
continue;
|
||||
|
||||
parts[index++] = part;
|
||||
combinedMin = (combinedMin == null || keyValidator.compare(combinedMin, part.minKey()) > 0) ? part.minKey() : combinedMin;
|
||||
combinedMax = (combinedMax == null || keyValidator.compare(combinedMax, part.maxKey()) < 0) ? part.maxKey() : combinedMax;
|
||||
}
|
||||
|
||||
OnDiskIndexBuilder builder = newIndexBuilder();
|
||||
builder.finish(Pair.create(combinedMin, combinedMax),
|
||||
new File(outputFile),
|
||||
new CombinedTermIterator(parts));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
logger.error("Failed to flush index {}.", outputFile, e);
|
||||
FileUtils.delete(outputFile);
|
||||
}
|
||||
finally
|
||||
{
|
||||
logger.info("Index flush to {} took {} ms.", outputFile, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start1));
|
||||
|
||||
for (OnDiskIndex part : parts)
|
||||
{
|
||||
if (part == null)
|
||||
continue;
|
||||
|
||||
FileUtils.closeQuietly(part);
|
||||
FileUtils.delete(part.getIndexPath());
|
||||
}
|
||||
|
||||
latch.countDown();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private ExecutorService getExecutor()
|
||||
{
|
||||
return source == OperationType.FLUSH ? INDEX_FLUSHER_MEMTABLE : INDEX_FLUSHER_GENERAL;
|
||||
}
|
||||
|
||||
private OnDiskIndexBuilder newIndexBuilder()
|
||||
{
|
||||
return new OnDiskIndexBuilder(keyValidator, columnIndex.getValidator(), columnIndex.getMode().mode);
|
||||
}
|
||||
|
||||
public String filename(boolean isFinal)
|
||||
{
|
||||
return outputFile + (isFinal ? "" : "_" + segmentNumber++);
|
||||
}
|
||||
}
|
||||
|
||||
protected long maxMemorySize(ColumnIndex columnIndex)
|
||||
{
|
||||
// 1G for memtable and configuration for compaction
|
||||
return source == OperationType.FLUSH ? 1073741824L : columnIndex.getMode().maxCompactionFlushMemoryInMb;
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return descriptor.hashCode();
|
||||
}
|
||||
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
return !(o == null || !(o instanceof PerSSTableIndexWriter)) && descriptor.equals(((PerSSTableIndexWriter) o).descriptor);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import com.google.common.primitives.Longs;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.utils.CombinedValue;
|
||||
|
||||
public abstract class Token implements CombinedValue<Long>, Iterable<DecoratedKey>
|
||||
{
|
||||
protected final long token;
|
||||
|
||||
public Token(long token)
|
||||
{
|
||||
this.token = token;
|
||||
}
|
||||
|
||||
public Long get()
|
||||
{
|
||||
return token;
|
||||
}
|
||||
|
||||
public int compareTo(CombinedValue<Long> o)
|
||||
{
|
||||
return Longs.compare(token, ((Token) o).token);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,519 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.utils.AbstractIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.CombinedValue;
|
||||
import org.apache.cassandra.index.sasi.utils.MappedBuffer;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.utils.MergeIterator;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.Iterators;
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
|
||||
import static org.apache.cassandra.index.sasi.disk.TokenTreeBuilder.EntryType;
|
||||
|
||||
// Note: all of the seek-able offsets contained in TokenTree should be sizeof(long)
|
||||
// even if currently only lower int portion of them if used, because that makes
|
||||
// it possible to switch to mmap implementation which supports long positions
|
||||
// without any on-disk format changes and/or re-indexing if one day we'll have a need to.
|
||||
public class TokenTree
|
||||
{
|
||||
private static final int LONG_BYTES = Long.SIZE / 8;
|
||||
private static final int SHORT_BYTES = Short.SIZE / 8;
|
||||
|
||||
private final Descriptor descriptor;
|
||||
private final MappedBuffer file;
|
||||
private final long startPos;
|
||||
private final long treeMinToken;
|
||||
private final long treeMaxToken;
|
||||
private final long tokenCount;
|
||||
|
||||
@VisibleForTesting
|
||||
protected TokenTree(MappedBuffer tokenTree)
|
||||
{
|
||||
this(Descriptor.CURRENT, tokenTree);
|
||||
}
|
||||
|
||||
public TokenTree(Descriptor d, MappedBuffer tokenTree)
|
||||
{
|
||||
descriptor = d;
|
||||
file = tokenTree;
|
||||
startPos = file.position();
|
||||
|
||||
file.position(startPos + TokenTreeBuilder.SHARED_HEADER_BYTES);
|
||||
|
||||
if (!validateMagic())
|
||||
throw new IllegalArgumentException("invalid token tree");
|
||||
|
||||
tokenCount = file.getLong();
|
||||
treeMinToken = file.getLong();
|
||||
treeMaxToken = file.getLong();
|
||||
}
|
||||
|
||||
public long getCount()
|
||||
{
|
||||
return tokenCount;
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> iterator(Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
return new TokenTreeIterator(file.duplicate(), keyFetcher);
|
||||
}
|
||||
|
||||
public OnDiskToken get(final long searchToken, Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
seekToLeaf(searchToken, file);
|
||||
long leafStart = file.position();
|
||||
short leafSize = file.getShort(leafStart + 1); // skip the info byte
|
||||
|
||||
file.position(leafStart + TokenTreeBuilder.BLOCK_HEADER_BYTES); // skip to tokens
|
||||
short tokenIndex = searchLeaf(searchToken, leafSize);
|
||||
|
||||
file.position(leafStart + TokenTreeBuilder.BLOCK_HEADER_BYTES);
|
||||
|
||||
OnDiskToken token = OnDiskToken.getTokenAt(file, tokenIndex, leafSize, keyFetcher);
|
||||
return token.get().equals(searchToken) ? token : null;
|
||||
}
|
||||
|
||||
private boolean validateMagic()
|
||||
{
|
||||
switch (descriptor.version.toString())
|
||||
{
|
||||
case Descriptor.VERSION_AA:
|
||||
return true;
|
||||
case Descriptor.VERSION_AB:
|
||||
return TokenTreeBuilder.AB_MAGIC == file.getShort();
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// finds leaf that *could* contain token
|
||||
private void seekToLeaf(long token, MappedBuffer file)
|
||||
{
|
||||
// this loop always seeks forward except for the first iteration
|
||||
// where it may seek back to the root
|
||||
long blockStart = startPos;
|
||||
while (true)
|
||||
{
|
||||
file.position(blockStart);
|
||||
|
||||
byte info = file.get();
|
||||
boolean isLeaf = (info & 1) == 1;
|
||||
|
||||
if (isLeaf)
|
||||
{
|
||||
file.position(blockStart);
|
||||
break;
|
||||
}
|
||||
|
||||
short tokenCount = file.getShort();
|
||||
|
||||
long minToken = file.getLong();
|
||||
long maxToken = file.getLong();
|
||||
|
||||
long seekBase = blockStart + TokenTreeBuilder.BLOCK_HEADER_BYTES;
|
||||
if (minToken > token)
|
||||
{
|
||||
// seek to beginning of child offsets to locate first child
|
||||
file.position(seekBase + tokenCount * LONG_BYTES);
|
||||
blockStart = (startPos + (int) file.getLong());
|
||||
}
|
||||
else if (maxToken < token)
|
||||
{
|
||||
// seek to end of child offsets to locate last child
|
||||
file.position(seekBase + (2 * tokenCount) * LONG_BYTES);
|
||||
blockStart = (startPos + (int) file.getLong());
|
||||
}
|
||||
else
|
||||
{
|
||||
// skip to end of block header/start of interior block tokens
|
||||
file.position(seekBase);
|
||||
|
||||
short offsetIndex = searchBlock(token, tokenCount, file);
|
||||
|
||||
// file pointer is now at beginning of offsets
|
||||
if (offsetIndex == tokenCount)
|
||||
file.position(file.position() + (offsetIndex * LONG_BYTES));
|
||||
else
|
||||
file.position(file.position() + ((tokenCount - offsetIndex - 1) + offsetIndex) * LONG_BYTES);
|
||||
|
||||
blockStart = (startPos + (int) file.getLong());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private short searchBlock(long searchToken, short tokenCount, MappedBuffer file)
|
||||
{
|
||||
short offsetIndex = 0;
|
||||
for (int i = 0; i < tokenCount; i++)
|
||||
{
|
||||
long readToken = file.getLong();
|
||||
if (searchToken < readToken)
|
||||
break;
|
||||
|
||||
offsetIndex++;
|
||||
}
|
||||
|
||||
return offsetIndex;
|
||||
}
|
||||
|
||||
private short searchLeaf(long searchToken, short tokenCount)
|
||||
{
|
||||
long base = file.position();
|
||||
|
||||
int start = 0;
|
||||
int end = tokenCount;
|
||||
int middle = 0;
|
||||
|
||||
while (start <= end)
|
||||
{
|
||||
middle = start + ((end - start) >> 1);
|
||||
|
||||
// each entry is 16 bytes wide, token is in bytes 4-11
|
||||
long token = file.getLong(base + (middle * (2 * LONG_BYTES) + 4));
|
||||
|
||||
if (token == searchToken)
|
||||
break;
|
||||
|
||||
if (token < searchToken)
|
||||
start = middle + 1;
|
||||
else
|
||||
end = middle - 1;
|
||||
}
|
||||
|
||||
return (short) middle;
|
||||
}
|
||||
|
||||
public class TokenTreeIterator extends RangeIterator<Long, Token>
|
||||
{
|
||||
private final Function<Long, DecoratedKey> keyFetcher;
|
||||
private final MappedBuffer file;
|
||||
|
||||
private long currentLeafStart;
|
||||
private int currentTokenIndex;
|
||||
|
||||
private long leafMinToken;
|
||||
private long leafMaxToken;
|
||||
private short leafSize;
|
||||
|
||||
protected boolean firstIteration = true;
|
||||
private boolean lastLeaf;
|
||||
|
||||
TokenTreeIterator(MappedBuffer file, Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
super(treeMinToken, treeMaxToken, tokenCount);
|
||||
|
||||
this.file = file;
|
||||
this.keyFetcher = keyFetcher;
|
||||
}
|
||||
|
||||
protected Token computeNext()
|
||||
{
|
||||
maybeFirstIteration();
|
||||
|
||||
if (currentTokenIndex >= leafSize && lastLeaf)
|
||||
return endOfData();
|
||||
|
||||
if (currentTokenIndex < leafSize) // tokens remaining in this leaf
|
||||
{
|
||||
return getTokenAt(currentTokenIndex++);
|
||||
}
|
||||
else // no more tokens remaining in this leaf
|
||||
{
|
||||
assert !lastLeaf;
|
||||
|
||||
seekToNextLeaf();
|
||||
setupBlock();
|
||||
return computeNext();
|
||||
}
|
||||
}
|
||||
|
||||
protected void performSkipTo(Long nextToken)
|
||||
{
|
||||
maybeFirstIteration();
|
||||
|
||||
if (nextToken <= leafMaxToken) // next is in this leaf block
|
||||
{
|
||||
searchLeaf(nextToken);
|
||||
}
|
||||
else // next is in a leaf block that needs to be found
|
||||
{
|
||||
seekToLeaf(nextToken, file);
|
||||
setupBlock();
|
||||
findNearest(nextToken);
|
||||
}
|
||||
}
|
||||
|
||||
private void setupBlock()
|
||||
{
|
||||
currentLeafStart = file.position();
|
||||
currentTokenIndex = 0;
|
||||
|
||||
lastLeaf = (file.get() & (1 << TokenTreeBuilder.LAST_LEAF_SHIFT)) > 0;
|
||||
leafSize = file.getShort();
|
||||
|
||||
leafMinToken = file.getLong();
|
||||
leafMaxToken = file.getLong();
|
||||
|
||||
// seek to end of leaf header/start of data
|
||||
file.position(currentLeafStart + TokenTreeBuilder.BLOCK_HEADER_BYTES);
|
||||
}
|
||||
|
||||
private void findNearest(Long next)
|
||||
{
|
||||
if (next > leafMaxToken && !lastLeaf)
|
||||
{
|
||||
seekToNextLeaf();
|
||||
setupBlock();
|
||||
findNearest(next);
|
||||
}
|
||||
else if (next > leafMinToken)
|
||||
searchLeaf(next);
|
||||
}
|
||||
|
||||
private void searchLeaf(long next)
|
||||
{
|
||||
for (int i = currentTokenIndex; i < leafSize; i++)
|
||||
{
|
||||
if (compareTokenAt(currentTokenIndex, next) >= 0)
|
||||
break;
|
||||
|
||||
currentTokenIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
private int compareTokenAt(int idx, long toToken)
|
||||
{
|
||||
return Long.compare(file.getLong(getTokenPosition(idx)), toToken);
|
||||
}
|
||||
|
||||
private Token getTokenAt(int idx)
|
||||
{
|
||||
return OnDiskToken.getTokenAt(file, idx, leafSize, keyFetcher);
|
||||
}
|
||||
|
||||
private long getTokenPosition(int idx)
|
||||
{
|
||||
// skip 4 byte entry header to get position pointing directly at the entry's token
|
||||
return OnDiskToken.getEntryPosition(idx, file) + (2 * SHORT_BYTES);
|
||||
}
|
||||
|
||||
private void seekToNextLeaf()
|
||||
{
|
||||
file.position(currentLeafStart + TokenTreeBuilder.BLOCK_BYTES);
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
// nothing to do here
|
||||
}
|
||||
|
||||
private void maybeFirstIteration()
|
||||
{
|
||||
// seek to the first token only when requested for the first time,
|
||||
// highly predictable branch and saves us a lot by not traversing the tree
|
||||
// on creation time because it's not at all required.
|
||||
if (!firstIteration)
|
||||
return;
|
||||
|
||||
seekToLeaf(treeMinToken, file);
|
||||
setupBlock();
|
||||
firstIteration = false;
|
||||
}
|
||||
}
|
||||
|
||||
public static class OnDiskToken extends Token
|
||||
{
|
||||
private final Set<TokenInfo> info = new HashSet<>(2);
|
||||
private final Set<DecoratedKey> loadedKeys = new TreeSet<>(DecoratedKey.comparator);
|
||||
|
||||
public OnDiskToken(MappedBuffer buffer, long position, short leafSize, Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
super(buffer.getLong(position + (2 * SHORT_BYTES)));
|
||||
info.add(new TokenInfo(buffer, position, leafSize, keyFetcher));
|
||||
}
|
||||
|
||||
public void merge(CombinedValue<Long> other)
|
||||
{
|
||||
if (!(other instanceof Token))
|
||||
return;
|
||||
|
||||
Token o = (Token) other;
|
||||
if (token != o.token)
|
||||
throw new IllegalArgumentException(String.format("%s != %s", token, o.token));
|
||||
|
||||
if (o instanceof OnDiskToken)
|
||||
{
|
||||
info.addAll(((OnDiskToken) other).info);
|
||||
}
|
||||
else
|
||||
{
|
||||
Iterators.addAll(loadedKeys, o.iterator());
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<DecoratedKey> iterator()
|
||||
{
|
||||
List<Iterator<DecoratedKey>> keys = new ArrayList<>(info.size());
|
||||
|
||||
for (TokenInfo i : info)
|
||||
keys.add(i.iterator());
|
||||
|
||||
if (!loadedKeys.isEmpty())
|
||||
keys.add(loadedKeys.iterator());
|
||||
|
||||
return MergeIterator.get(keys, DecoratedKey.comparator, new MergeIterator.Reducer<DecoratedKey, DecoratedKey>()
|
||||
{
|
||||
DecoratedKey reduced = null;
|
||||
|
||||
public boolean trivialReduceIsTrivial()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
public void reduce(int idx, DecoratedKey current)
|
||||
{
|
||||
reduced = current;
|
||||
}
|
||||
|
||||
protected DecoratedKey getReduced()
|
||||
{
|
||||
return reduced;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public Set<Long> getOffsets()
|
||||
{
|
||||
Set<Long> offsets = new HashSet<>();
|
||||
for (TokenInfo i : info)
|
||||
{
|
||||
for (long offset : i.fetchOffsets())
|
||||
offsets.add(offset);
|
||||
}
|
||||
|
||||
return offsets;
|
||||
}
|
||||
|
||||
public static OnDiskToken getTokenAt(MappedBuffer buffer, int idx, short leafSize, Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
return new OnDiskToken(buffer, getEntryPosition(idx, buffer), leafSize, keyFetcher);
|
||||
}
|
||||
|
||||
private static long getEntryPosition(int idx, MappedBuffer file)
|
||||
{
|
||||
// info (4 bytes) + token (8 bytes) + offset (4 bytes) = 16 bytes
|
||||
return file.position() + (idx * (2 * LONG_BYTES));
|
||||
}
|
||||
}
|
||||
|
||||
private static class TokenInfo
|
||||
{
|
||||
private final MappedBuffer buffer;
|
||||
private final Function<Long, DecoratedKey> keyFetcher;
|
||||
|
||||
private final long position;
|
||||
private final short leafSize;
|
||||
|
||||
public TokenInfo(MappedBuffer buffer, long position, short leafSize, Function<Long, DecoratedKey> keyFetcher)
|
||||
{
|
||||
this.keyFetcher = keyFetcher;
|
||||
this.buffer = buffer;
|
||||
this.position = position;
|
||||
this.leafSize = leafSize;
|
||||
}
|
||||
|
||||
public Iterator<DecoratedKey> iterator()
|
||||
{
|
||||
return new KeyIterator(keyFetcher, fetchOffsets());
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return new HashCodeBuilder().append(keyFetcher).append(position).append(leafSize).build();
|
||||
}
|
||||
|
||||
public boolean equals(Object other)
|
||||
{
|
||||
if (!(other instanceof TokenInfo))
|
||||
return false;
|
||||
|
||||
TokenInfo o = (TokenInfo) other;
|
||||
return keyFetcher == o.keyFetcher && position == o.position;
|
||||
}
|
||||
|
||||
private long[] fetchOffsets()
|
||||
{
|
||||
short info = buffer.getShort(position);
|
||||
short offsetShort = buffer.getShort(position + SHORT_BYTES);
|
||||
int offsetInt = buffer.getInt(position + (2 * SHORT_BYTES) + LONG_BYTES);
|
||||
|
||||
EntryType type = EntryType.of(info & TokenTreeBuilder.ENTRY_TYPE_MASK);
|
||||
|
||||
switch (type)
|
||||
{
|
||||
case SIMPLE:
|
||||
return new long[] { offsetInt };
|
||||
|
||||
case OVERFLOW:
|
||||
long[] offsets = new long[offsetShort]; // offsetShort contains count of tokens
|
||||
long offsetPos = (buffer.position() + (2 * (leafSize * LONG_BYTES)) + (offsetInt * LONG_BYTES));
|
||||
|
||||
for (int i = 0; i < offsetShort; i++)
|
||||
offsets[i] = buffer.getLong(offsetPos + (i * LONG_BYTES));
|
||||
|
||||
return offsets;
|
||||
|
||||
case FACTORED:
|
||||
return new long[] { (((long) offsetInt) << Short.SIZE) + offsetShort };
|
||||
|
||||
case PACKED:
|
||||
return new long[] { offsetShort, offsetInt };
|
||||
|
||||
default:
|
||||
throw new IllegalStateException("Unknown entry type: " + type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class KeyIterator extends AbstractIterator<DecoratedKey>
|
||||
{
|
||||
private final Function<Long, DecoratedKey> keyFetcher;
|
||||
private final long[] offsets;
|
||||
private int index = 0;
|
||||
|
||||
public KeyIterator(Function<Long, DecoratedKey> keyFetcher, long[] offsets)
|
||||
{
|
||||
this.keyFetcher = keyFetcher;
|
||||
this.offsets = offsets;
|
||||
}
|
||||
|
||||
public DecoratedKey computeNext()
|
||||
{
|
||||
return index < offsets.length ? keyFetcher.apply(offsets[index++]) : endOfData();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,839 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.disk;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.io.util.DataOutputPlus;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
import com.carrotsearch.hppc.LongArrayList;
|
||||
import com.carrotsearch.hppc.LongSet;
|
||||
import com.carrotsearch.hppc.cursors.LongCursor;
|
||||
import com.carrotsearch.hppc.LongOpenHashSet;
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
|
||||
public class TokenTreeBuilder
|
||||
{
|
||||
// note: ordinal positions are used here, do not change order
|
||||
enum EntryType
|
||||
{
|
||||
SIMPLE, FACTORED, PACKED, OVERFLOW;
|
||||
|
||||
public static EntryType of(int ordinal)
|
||||
{
|
||||
if (ordinal == SIMPLE.ordinal())
|
||||
return SIMPLE;
|
||||
|
||||
if (ordinal == FACTORED.ordinal())
|
||||
return FACTORED;
|
||||
|
||||
if (ordinal == PACKED.ordinal())
|
||||
return PACKED;
|
||||
|
||||
if (ordinal == OVERFLOW.ordinal())
|
||||
return OVERFLOW;
|
||||
|
||||
throw new IllegalArgumentException("Unknown ordinal: " + ordinal);
|
||||
}
|
||||
}
|
||||
|
||||
public static final int BLOCK_BYTES = 4096;
|
||||
public static final int BLOCK_HEADER_BYTES = 64;
|
||||
public static final int OVERFLOW_TRAILER_BYTES = 64;
|
||||
public static final int OVERFLOW_TRAILER_CAPACITY = OVERFLOW_TRAILER_BYTES / 8;
|
||||
public static final int TOKENS_PER_BLOCK = (BLOCK_BYTES - BLOCK_HEADER_BYTES - OVERFLOW_TRAILER_BYTES) / 16;
|
||||
public static final long MAX_OFFSET = (1L << 47) - 1; // 48 bits for (signed) offset
|
||||
public static final byte LAST_LEAF_SHIFT = 1;
|
||||
public static final byte SHARED_HEADER_BYTES = 19;
|
||||
public static final byte ENTRY_TYPE_MASK = 0x03;
|
||||
public static final short AB_MAGIC = 0x5A51;
|
||||
|
||||
private final SortedMap<Long, LongSet> tokens = new TreeMap<>();
|
||||
private int numBlocks;
|
||||
|
||||
private Node root;
|
||||
private InteriorNode rightmostParent;
|
||||
private Leaf leftmostLeaf;
|
||||
private Leaf rightmostLeaf;
|
||||
private long tokenCount = 0;
|
||||
private long treeMinToken;
|
||||
private long treeMaxToken;
|
||||
|
||||
public TokenTreeBuilder()
|
||||
{}
|
||||
|
||||
public TokenTreeBuilder(SortedMap<Long, LongSet> data)
|
||||
{
|
||||
add(data);
|
||||
}
|
||||
|
||||
public void add(Long token, long keyPosition)
|
||||
{
|
||||
LongSet found = tokens.get(token);
|
||||
if (found == null)
|
||||
tokens.put(token, (found = new LongOpenHashSet(2)));
|
||||
|
||||
found.add(keyPosition);
|
||||
}
|
||||
|
||||
public void add(SortedMap<Long, LongSet> data)
|
||||
{
|
||||
for (Map.Entry<Long, LongSet> newEntry : data.entrySet())
|
||||
{
|
||||
LongSet found = tokens.get(newEntry.getKey());
|
||||
if (found == null)
|
||||
tokens.put(newEntry.getKey(), (found = new LongOpenHashSet(4)));
|
||||
|
||||
for (LongCursor offset : newEntry.getValue())
|
||||
found.add(offset.value);
|
||||
}
|
||||
}
|
||||
|
||||
public TokenTreeBuilder finish()
|
||||
{
|
||||
maybeBulkLoad();
|
||||
return this;
|
||||
}
|
||||
|
||||
public SortedMap<Long, LongSet> getTokens()
|
||||
{
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public long getTokenCount()
|
||||
{
|
||||
return tokenCount;
|
||||
}
|
||||
|
||||
public int serializedSize()
|
||||
{
|
||||
if (numBlocks == 1)
|
||||
return (BLOCK_HEADER_BYTES + ((int) tokenCount * 16));
|
||||
else
|
||||
return numBlocks * BLOCK_BYTES;
|
||||
}
|
||||
|
||||
public void write(DataOutputPlus out) throws IOException
|
||||
{
|
||||
ByteBuffer blockBuffer = ByteBuffer.allocate(BLOCK_BYTES);
|
||||
Iterator<Node> levelIterator = root.levelIterator();
|
||||
long childBlockIndex = 1;
|
||||
|
||||
while (levelIterator != null)
|
||||
{
|
||||
|
||||
Node firstChild = null;
|
||||
while (levelIterator.hasNext())
|
||||
{
|
||||
Node block = levelIterator.next();
|
||||
|
||||
if (firstChild == null && !block.isLeaf())
|
||||
firstChild = ((InteriorNode) block).children.get(0);
|
||||
|
||||
block.serialize(childBlockIndex, blockBuffer);
|
||||
flushBuffer(blockBuffer, out, numBlocks != 1);
|
||||
|
||||
childBlockIndex += block.childCount();
|
||||
}
|
||||
|
||||
levelIterator = (firstChild == null) ? null : firstChild.levelIterator();
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<Pair<Long, LongSet>> iterator()
|
||||
{
|
||||
return new TokenIterator(leftmostLeaf.levelIterator());
|
||||
}
|
||||
|
||||
private void maybeBulkLoad()
|
||||
{
|
||||
if (root == null)
|
||||
bulkLoad();
|
||||
}
|
||||
|
||||
private void flushBuffer(ByteBuffer buffer, DataOutputPlus o, boolean align) throws IOException
|
||||
{
|
||||
// seek to end of last block before flushing
|
||||
if (align)
|
||||
alignBuffer(buffer, BLOCK_BYTES);
|
||||
|
||||
buffer.flip();
|
||||
o.write(buffer);
|
||||
buffer.clear();
|
||||
}
|
||||
|
||||
private static void alignBuffer(ByteBuffer buffer, int blockSize)
|
||||
{
|
||||
long curPos = buffer.position();
|
||||
if ((curPos & (blockSize - 1)) != 0) // align on the block boundary if needed
|
||||
buffer.position((int) FBUtilities.align(curPos, blockSize));
|
||||
}
|
||||
|
||||
private void bulkLoad()
|
||||
{
|
||||
tokenCount = tokens.size();
|
||||
treeMinToken = tokens.firstKey();
|
||||
treeMaxToken = tokens.lastKey();
|
||||
numBlocks = 1;
|
||||
|
||||
// special case the tree that only has a single block in it (so we don't create a useless root)
|
||||
if (tokenCount <= TOKENS_PER_BLOCK)
|
||||
{
|
||||
leftmostLeaf = new Leaf(tokens);
|
||||
rightmostLeaf = leftmostLeaf;
|
||||
root = leftmostLeaf;
|
||||
}
|
||||
else
|
||||
{
|
||||
root = new InteriorNode();
|
||||
rightmostParent = (InteriorNode) root;
|
||||
|
||||
int i = 0;
|
||||
Leaf lastLeaf = null;
|
||||
Long firstToken = tokens.firstKey();
|
||||
Long finalToken = tokens.lastKey();
|
||||
Long lastToken;
|
||||
for (Long token : tokens.keySet())
|
||||
{
|
||||
if (i == 0 || (i % TOKENS_PER_BLOCK != 0 && i != (tokenCount - 1)))
|
||||
{
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
lastToken = token;
|
||||
Leaf leaf = (i != (tokenCount - 1) || token.equals(finalToken)) ?
|
||||
new Leaf(tokens.subMap(firstToken, lastToken)) : new Leaf(tokens.tailMap(firstToken));
|
||||
|
||||
if (i == TOKENS_PER_BLOCK)
|
||||
leftmostLeaf = leaf;
|
||||
else
|
||||
lastLeaf.next = leaf;
|
||||
|
||||
rightmostParent.add(leaf);
|
||||
lastLeaf = leaf;
|
||||
rightmostLeaf = leaf;
|
||||
firstToken = lastToken;
|
||||
i++;
|
||||
numBlocks++;
|
||||
|
||||
if (token.equals(finalToken))
|
||||
{
|
||||
Leaf finalLeaf = new Leaf(tokens.tailMap(token));
|
||||
lastLeaf.next = finalLeaf;
|
||||
rightmostParent.add(finalLeaf);
|
||||
rightmostLeaf = finalLeaf;
|
||||
numBlocks++;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private abstract class Node
|
||||
{
|
||||
protected InteriorNode parent;
|
||||
protected Node next;
|
||||
protected Long nodeMinToken, nodeMaxToken;
|
||||
|
||||
public abstract void serialize(long childBlockIndex, ByteBuffer buf);
|
||||
public abstract int childCount();
|
||||
public abstract int tokenCount();
|
||||
public abstract Long smallestToken();
|
||||
|
||||
public Iterator<Node> levelIterator()
|
||||
{
|
||||
return new LevelIterator(this);
|
||||
}
|
||||
|
||||
public boolean isLeaf()
|
||||
{
|
||||
return (this instanceof Leaf);
|
||||
}
|
||||
|
||||
protected boolean isLastLeaf()
|
||||
{
|
||||
return this == rightmostLeaf;
|
||||
}
|
||||
|
||||
protected boolean isRoot()
|
||||
{
|
||||
return this == root;
|
||||
}
|
||||
|
||||
protected void updateTokenRange(long token)
|
||||
{
|
||||
nodeMinToken = nodeMinToken == null ? token : Math.min(nodeMinToken, token);
|
||||
nodeMaxToken = nodeMaxToken == null ? token : Math.max(nodeMaxToken, token);
|
||||
}
|
||||
|
||||
protected void serializeHeader(ByteBuffer buf)
|
||||
{
|
||||
Header header;
|
||||
if (isRoot())
|
||||
header = new RootHeader();
|
||||
else if (!isLeaf())
|
||||
header = new InteriorNodeHeader();
|
||||
else
|
||||
header = new LeafHeader();
|
||||
|
||||
header.serialize(buf);
|
||||
alignBuffer(buf, BLOCK_HEADER_BYTES);
|
||||
}
|
||||
|
||||
private abstract class Header
|
||||
{
|
||||
public void serialize(ByteBuffer buf)
|
||||
{
|
||||
buf.put(infoByte())
|
||||
.putShort((short) (tokenCount()))
|
||||
.putLong(nodeMinToken)
|
||||
.putLong(nodeMaxToken);
|
||||
}
|
||||
|
||||
protected abstract byte infoByte();
|
||||
}
|
||||
|
||||
private class RootHeader extends Header
|
||||
{
|
||||
public void serialize(ByteBuffer buf)
|
||||
{
|
||||
super.serialize(buf);
|
||||
writeMagic(buf);
|
||||
buf.putLong(tokenCount)
|
||||
.putLong(treeMinToken)
|
||||
.putLong(treeMaxToken);
|
||||
}
|
||||
|
||||
protected byte infoByte()
|
||||
{
|
||||
// if leaf, set leaf indicator and last leaf indicator (bits 0 & 1)
|
||||
// if not leaf, clear both bits
|
||||
return (byte) ((isLeaf()) ? 3 : 0);
|
||||
}
|
||||
|
||||
protected void writeMagic(ByteBuffer buf)
|
||||
{
|
||||
switch (Descriptor.CURRENT_VERSION)
|
||||
{
|
||||
case Descriptor.VERSION_AB:
|
||||
buf.putShort(AB_MAGIC);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private class InteriorNodeHeader extends Header
|
||||
{
|
||||
// bit 0 (leaf indicator) & bit 1 (last leaf indicator) cleared
|
||||
protected byte infoByte()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private class LeafHeader extends Header
|
||||
{
|
||||
// bit 0 set as leaf indicator
|
||||
// bit 1 set if this is last leaf of data
|
||||
protected byte infoByte()
|
||||
{
|
||||
byte infoByte = 1;
|
||||
infoByte |= (isLastLeaf()) ? (1 << LAST_LEAF_SHIFT) : 0;
|
||||
|
||||
return infoByte;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private class Leaf extends Node
|
||||
{
|
||||
private final SortedMap<Long, LongSet> tokens;
|
||||
private LongArrayList overflowCollisions;
|
||||
|
||||
Leaf(SortedMap<Long, LongSet> data)
|
||||
{
|
||||
nodeMinToken = data.firstKey();
|
||||
nodeMaxToken = data.lastKey();
|
||||
tokens = data;
|
||||
}
|
||||
|
||||
public Long largestToken()
|
||||
{
|
||||
return nodeMaxToken;
|
||||
}
|
||||
|
||||
public void serialize(long childBlockIndex, ByteBuffer buf)
|
||||
{
|
||||
serializeHeader(buf);
|
||||
serializeData(buf);
|
||||
serializeOverflowCollisions(buf);
|
||||
}
|
||||
|
||||
public int childCount()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
public int tokenCount()
|
||||
{
|
||||
return tokens.size();
|
||||
}
|
||||
|
||||
public Long smallestToken()
|
||||
{
|
||||
return nodeMinToken;
|
||||
}
|
||||
|
||||
public Iterator<Map.Entry<Long, LongSet>> tokenIterator()
|
||||
{
|
||||
return tokens.entrySet().iterator();
|
||||
}
|
||||
|
||||
private void serializeData(ByteBuffer buf)
|
||||
{
|
||||
for (Map.Entry<Long, LongSet> entry : tokens.entrySet())
|
||||
createEntry(entry.getKey(), entry.getValue()).serialize(buf);
|
||||
}
|
||||
|
||||
private void serializeOverflowCollisions(ByteBuffer buf)
|
||||
{
|
||||
if (overflowCollisions != null)
|
||||
for (LongCursor offset : overflowCollisions)
|
||||
buf.putLong(offset.value);
|
||||
}
|
||||
|
||||
|
||||
private LeafEntry createEntry(final long tok, final LongSet offsets)
|
||||
{
|
||||
int offsetCount = offsets.size();
|
||||
switch (offsetCount)
|
||||
{
|
||||
case 0:
|
||||
throw new AssertionError("no offsets for token " + tok);
|
||||
case 1:
|
||||
long offset = offsets.toArray()[0];
|
||||
if (offset > MAX_OFFSET)
|
||||
throw new AssertionError("offset " + offset + " cannot be greater than " + MAX_OFFSET);
|
||||
else if (offset <= Integer.MAX_VALUE)
|
||||
return new SimpleLeafEntry(tok, offset);
|
||||
else
|
||||
return new FactoredOffsetLeafEntry(tok, offset);
|
||||
case 2:
|
||||
long[] rawOffsets = offsets.toArray();
|
||||
if (rawOffsets[0] <= Integer.MAX_VALUE && rawOffsets[1] <= Integer.MAX_VALUE &&
|
||||
(rawOffsets[0] <= Short.MAX_VALUE || rawOffsets[1] <= Short.MAX_VALUE))
|
||||
return new PackedCollisionLeafEntry(tok, rawOffsets);
|
||||
else
|
||||
return createOverflowEntry(tok, offsetCount, offsets);
|
||||
default:
|
||||
return createOverflowEntry(tok, offsetCount, offsets);
|
||||
}
|
||||
}
|
||||
|
||||
private LeafEntry createOverflowEntry(final long tok, final int offsetCount, final LongSet offsets)
|
||||
{
|
||||
if (overflowCollisions == null)
|
||||
overflowCollisions = new LongArrayList();
|
||||
|
||||
LeafEntry entry = new OverflowCollisionLeafEntry(tok, (short) overflowCollisions.size(), (short) offsetCount);
|
||||
for (LongCursor o : offsets) {
|
||||
if (overflowCollisions.size() == OVERFLOW_TRAILER_CAPACITY)
|
||||
throw new AssertionError("cannot have more than " + OVERFLOW_TRAILER_CAPACITY + " overflow collisions per leaf");
|
||||
else
|
||||
overflowCollisions.add(o.value);
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
private abstract class LeafEntry
|
||||
{
|
||||
protected final long token;
|
||||
|
||||
abstract public EntryType type();
|
||||
abstract public int offsetData();
|
||||
abstract public short offsetExtra();
|
||||
|
||||
public LeafEntry(final long tok)
|
||||
{
|
||||
token = tok;
|
||||
}
|
||||
|
||||
public void serialize(ByteBuffer buf)
|
||||
{
|
||||
buf.putShort((short) type().ordinal())
|
||||
.putShort(offsetExtra())
|
||||
.putLong(token)
|
||||
.putInt(offsetData());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
// assumes there is a single offset and the offset is <= Integer.MAX_VALUE
|
||||
private class SimpleLeafEntry extends LeafEntry
|
||||
{
|
||||
private final long offset;
|
||||
|
||||
public SimpleLeafEntry(final long tok, final long off)
|
||||
{
|
||||
super(tok);
|
||||
offset = off;
|
||||
}
|
||||
|
||||
public EntryType type()
|
||||
{
|
||||
return EntryType.SIMPLE;
|
||||
}
|
||||
|
||||
public int offsetData()
|
||||
{
|
||||
return (int) offset;
|
||||
}
|
||||
|
||||
public short offsetExtra()
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// assumes there is a single offset and Integer.MAX_VALUE < offset <= MAX_OFFSET
|
||||
// take the middle 32 bits of offset (or the top 32 when considering offset is max 48 bits)
|
||||
// and store where offset is normally stored. take bottom 16 bits of offset and store in entry header
|
||||
private class FactoredOffsetLeafEntry extends LeafEntry
|
||||
{
|
||||
private final long offset;
|
||||
|
||||
public FactoredOffsetLeafEntry(final long tok, final long off)
|
||||
{
|
||||
super(tok);
|
||||
offset = off;
|
||||
}
|
||||
|
||||
public EntryType type()
|
||||
{
|
||||
return EntryType.FACTORED;
|
||||
}
|
||||
|
||||
public int offsetData()
|
||||
{
|
||||
return (int) (offset >>> Short.SIZE);
|
||||
}
|
||||
|
||||
public short offsetExtra()
|
||||
{
|
||||
return (short) offset;
|
||||
}
|
||||
}
|
||||
|
||||
// holds an entry with two offsets that can be packed in an int & a short
|
||||
// the int offset is stored where offset is normally stored. short offset is
|
||||
// stored in entry header
|
||||
private class PackedCollisionLeafEntry extends LeafEntry
|
||||
{
|
||||
private short smallerOffset;
|
||||
private int largerOffset;
|
||||
|
||||
public PackedCollisionLeafEntry(final long tok, final long[] offs)
|
||||
{
|
||||
super(tok);
|
||||
|
||||
smallerOffset = (short) Math.min(offs[0], offs[1]);
|
||||
largerOffset = (int) Math.max(offs[0], offs[1]);
|
||||
}
|
||||
|
||||
public EntryType type()
|
||||
{
|
||||
return EntryType.PACKED;
|
||||
}
|
||||
|
||||
public int offsetData()
|
||||
{
|
||||
return largerOffset;
|
||||
}
|
||||
|
||||
public short offsetExtra()
|
||||
{
|
||||
return smallerOffset;
|
||||
}
|
||||
}
|
||||
|
||||
// holds an entry with three or more offsets, or two offsets that cannot
|
||||
// be packed into an int & a short. the index into the overflow list
|
||||
// is stored where the offset is normally stored. the number of overflowed offsets
|
||||
// for the entry is stored in the entry header
|
||||
private class OverflowCollisionLeafEntry extends LeafEntry
|
||||
{
|
||||
private final short startIndex;
|
||||
private final short count;
|
||||
|
||||
public OverflowCollisionLeafEntry(final long tok, final short collisionStartIndex, final short collisionCount)
|
||||
{
|
||||
super(tok);
|
||||
startIndex = collisionStartIndex;
|
||||
count = collisionCount;
|
||||
}
|
||||
|
||||
public EntryType type()
|
||||
{
|
||||
return EntryType.OVERFLOW;
|
||||
}
|
||||
|
||||
public int offsetData()
|
||||
{
|
||||
return startIndex;
|
||||
}
|
||||
|
||||
public short offsetExtra()
|
||||
{
|
||||
return count;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private class InteriorNode extends Node
|
||||
{
|
||||
private List<Long> tokens = new ArrayList<>(TOKENS_PER_BLOCK);
|
||||
private List<Node> children = new ArrayList<>(TOKENS_PER_BLOCK + 1);
|
||||
private int position = 0; // TODO (jwest): can get rid of this and use array size
|
||||
|
||||
|
||||
public void serialize(long childBlockIndex, ByteBuffer buf)
|
||||
{
|
||||
serializeHeader(buf);
|
||||
serializeTokens(buf);
|
||||
serializeChildOffsets(childBlockIndex, buf);
|
||||
}
|
||||
|
||||
public int childCount()
|
||||
{
|
||||
return children.size();
|
||||
}
|
||||
|
||||
public int tokenCount()
|
||||
{
|
||||
return tokens.size();
|
||||
}
|
||||
|
||||
public Long smallestToken()
|
||||
{
|
||||
return tokens.get(0);
|
||||
}
|
||||
|
||||
protected void add(Long token, InteriorNode leftChild, InteriorNode rightChild)
|
||||
{
|
||||
int pos = tokens.size();
|
||||
if (pos == TOKENS_PER_BLOCK)
|
||||
{
|
||||
InteriorNode sibling = split();
|
||||
sibling.add(token, leftChild, rightChild);
|
||||
|
||||
}
|
||||
else {
|
||||
if (leftChild != null)
|
||||
children.add(pos, leftChild);
|
||||
|
||||
if (rightChild != null)
|
||||
{
|
||||
children.add(pos + 1, rightChild);
|
||||
rightChild.parent = this;
|
||||
}
|
||||
|
||||
updateTokenRange(token);
|
||||
tokens.add(pos, token);
|
||||
}
|
||||
}
|
||||
|
||||
protected void add(Leaf node)
|
||||
{
|
||||
|
||||
if (position == (TOKENS_PER_BLOCK + 1))
|
||||
{
|
||||
rightmostParent = split();
|
||||
rightmostParent.add(node);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
||||
node.parent = this;
|
||||
children.add(position, node);
|
||||
position++;
|
||||
|
||||
// the first child is referenced only during bulk load. we don't take a value
|
||||
// to store into the tree, one is subtracted since position has already been incremented
|
||||
// for the next node to be added
|
||||
if (position - 1 == 0)
|
||||
return;
|
||||
|
||||
|
||||
// tokens are inserted one behind the current position, but 2 is subtracted because
|
||||
// position has already been incremented for the next add
|
||||
Long smallestToken = node.smallestToken();
|
||||
updateTokenRange(smallestToken);
|
||||
tokens.add(position - 2, smallestToken);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected InteriorNode split()
|
||||
{
|
||||
Pair<Long, InteriorNode> splitResult = splitBlock();
|
||||
Long middleValue = splitResult.left;
|
||||
InteriorNode sibling = splitResult.right;
|
||||
InteriorNode leftChild = null;
|
||||
|
||||
// create a new root if necessary
|
||||
if (parent == null)
|
||||
{
|
||||
parent = new InteriorNode();
|
||||
root = parent;
|
||||
sibling.parent = parent;
|
||||
leftChild = this;
|
||||
numBlocks++;
|
||||
}
|
||||
|
||||
parent.add(middleValue, leftChild, sibling);
|
||||
|
||||
return sibling;
|
||||
}
|
||||
|
||||
protected Pair<Long, InteriorNode> splitBlock()
|
||||
{
|
||||
final int splitPosition = TOKENS_PER_BLOCK - 2;
|
||||
InteriorNode sibling = new InteriorNode();
|
||||
sibling.parent = parent;
|
||||
next = sibling;
|
||||
|
||||
Long middleValue = tokens.get(splitPosition);
|
||||
|
||||
for (int i = splitPosition; i < TOKENS_PER_BLOCK; i++)
|
||||
{
|
||||
if (i != TOKENS_PER_BLOCK && i != splitPosition)
|
||||
{
|
||||
long token = tokens.get(i);
|
||||
sibling.updateTokenRange(token);
|
||||
sibling.tokens.add(token);
|
||||
}
|
||||
|
||||
Node child = children.get(i + 1);
|
||||
child.parent = sibling;
|
||||
sibling.children.add(child);
|
||||
sibling.position++;
|
||||
}
|
||||
|
||||
for (int i = TOKENS_PER_BLOCK; i >= splitPosition; i--)
|
||||
{
|
||||
if (i != TOKENS_PER_BLOCK)
|
||||
tokens.remove(i);
|
||||
|
||||
if (i != splitPosition)
|
||||
children.remove(i);
|
||||
}
|
||||
|
||||
nodeMinToken = smallestToken();
|
||||
nodeMaxToken = tokens.get(tokens.size() - 1);
|
||||
numBlocks++;
|
||||
|
||||
return Pair.create(middleValue, sibling);
|
||||
}
|
||||
|
||||
protected boolean isFull()
|
||||
{
|
||||
return (position >= TOKENS_PER_BLOCK + 1);
|
||||
}
|
||||
|
||||
private void serializeTokens(ByteBuffer buf)
|
||||
{
|
||||
for (Long token : tokens)
|
||||
buf.putLong(token);
|
||||
}
|
||||
|
||||
|
||||
private void serializeChildOffsets(long childBlockIndex, ByteBuffer buf)
|
||||
{
|
||||
for (int i = 0; i < children.size(); i++)
|
||||
buf.putLong((childBlockIndex + i) * BLOCK_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
public static class LevelIterator extends AbstractIterator<Node>
|
||||
{
|
||||
private Node currentNode;
|
||||
|
||||
LevelIterator(Node first)
|
||||
{
|
||||
currentNode = first;
|
||||
}
|
||||
|
||||
public Node computeNext()
|
||||
{
|
||||
if (currentNode == null)
|
||||
return endOfData();
|
||||
|
||||
Node returnNode = currentNode;
|
||||
currentNode = returnNode.next;
|
||||
|
||||
return returnNode;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
public static class TokenIterator extends AbstractIterator<Pair<Long, LongSet>>
|
||||
{
|
||||
private Iterator<Node> levelIterator;
|
||||
private Iterator<Map.Entry<Long, LongSet>> currentIterator;
|
||||
|
||||
TokenIterator(Iterator<Node> level)
|
||||
{
|
||||
levelIterator = level;
|
||||
if (levelIterator.hasNext())
|
||||
currentIterator = ((Leaf) levelIterator.next()).tokenIterator();
|
||||
}
|
||||
|
||||
public Pair<Long, LongSet> computeNext()
|
||||
{
|
||||
if (currentIterator != null && currentIterator.hasNext())
|
||||
{
|
||||
Map.Entry<Long, LongSet> next = currentIterator.next();
|
||||
return Pair.create(next.getKey(), next.getValue());
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!levelIterator.hasNext())
|
||||
return endOfData();
|
||||
else
|
||||
{
|
||||
currentIterator = ((Leaf) levelIterator.next()).tokenIterator();
|
||||
return computeNext();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.exceptions;
|
||||
|
||||
public class TimeQuotaExceededException extends RuntimeException
|
||||
{}
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.memory;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.TypeUtil;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class IndexMemtable
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(IndexMemtable.class);
|
||||
|
||||
private final MemIndex index;
|
||||
|
||||
public IndexMemtable(ColumnIndex columnIndex)
|
||||
{
|
||||
this.index = MemIndex.forColumn(columnIndex.keyValidator(), columnIndex);
|
||||
}
|
||||
|
||||
public long index(DecoratedKey key, ByteBuffer value)
|
||||
{
|
||||
if (value == null || value.remaining() == 0)
|
||||
return 0;
|
||||
|
||||
AbstractType<?> validator = index.columnIndex.getValidator();
|
||||
if (!TypeUtil.isValid(value, validator))
|
||||
{
|
||||
int size = value.remaining();
|
||||
if ((value = TypeUtil.tryUpcast(value, validator)) == null)
|
||||
{
|
||||
logger.error("Can't add column {} to index for key: {}, value size {} bytes, validator: {}.",
|
||||
index.columnIndex.getColumnName(),
|
||||
index.columnIndex.keyValidator().getString(key.getKey()),
|
||||
size,
|
||||
validator);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return index.add(key, value);
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> search(Expression expression)
|
||||
{
|
||||
return index == null ? null : index.search(expression);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.memory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
import java.util.SortedSet;
|
||||
import java.util.TreeSet;
|
||||
import java.util.concurrent.ConcurrentSkipListSet;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.utils.AbstractIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.CombinedValue;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
|
||||
public class KeyRangeIterator extends RangeIterator<Long, Token>
|
||||
{
|
||||
private final DKIterator iterator;
|
||||
|
||||
public KeyRangeIterator(ConcurrentSkipListSet<DecoratedKey> keys)
|
||||
{
|
||||
super((Long) keys.first().getToken().getTokenValue(), (Long) keys.last().getToken().getTokenValue(), keys.size());
|
||||
this.iterator = new DKIterator(keys.iterator());
|
||||
}
|
||||
|
||||
protected Token computeNext()
|
||||
{
|
||||
return iterator.hasNext() ? new DKToken(iterator.next()) : endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(Long nextToken)
|
||||
{
|
||||
while (iterator.hasNext())
|
||||
{
|
||||
DecoratedKey key = iterator.peek();
|
||||
if (Long.compare((long) key.getToken().getTokenValue(), nextToken) >= 0)
|
||||
break;
|
||||
|
||||
// consume smaller key
|
||||
iterator.next();
|
||||
}
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{}
|
||||
|
||||
private static class DKIterator extends AbstractIterator<DecoratedKey> implements PeekingIterator<DecoratedKey>
|
||||
{
|
||||
private final Iterator<DecoratedKey> keys;
|
||||
|
||||
public DKIterator(Iterator<DecoratedKey> keys)
|
||||
{
|
||||
this.keys = keys;
|
||||
}
|
||||
|
||||
protected DecoratedKey computeNext()
|
||||
{
|
||||
return keys.hasNext() ? keys.next() : endOfData();
|
||||
}
|
||||
}
|
||||
|
||||
private static class DKToken extends Token
|
||||
{
|
||||
private final SortedSet<DecoratedKey> keys;
|
||||
|
||||
public DKToken(final DecoratedKey key)
|
||||
{
|
||||
super((long) key.getToken().getTokenValue());
|
||||
|
||||
keys = new TreeSet<DecoratedKey>(DecoratedKey.comparator)
|
||||
{{
|
||||
add(key);
|
||||
}};
|
||||
}
|
||||
|
||||
public void merge(CombinedValue<Long> other)
|
||||
{
|
||||
if (!(other instanceof Token))
|
||||
return;
|
||||
|
||||
Token o = (Token) other;
|
||||
assert o.get().equals(token);
|
||||
|
||||
if (o instanceof DKToken)
|
||||
{
|
||||
keys.addAll(((DKToken) o).keys);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (DecoratedKey key : o)
|
||||
keys.add(key);
|
||||
}
|
||||
}
|
||||
|
||||
public Iterator<DecoratedKey> iterator()
|
||||
{
|
||||
return keys.iterator();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.memory;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
import org.github.jamm.MemoryMeter;
|
||||
|
||||
public abstract class MemIndex
|
||||
{
|
||||
protected final AbstractType<?> keyValidator;
|
||||
protected final ColumnIndex columnIndex;
|
||||
|
||||
protected MemIndex(AbstractType<?> keyValidator, ColumnIndex columnIndex)
|
||||
{
|
||||
this.keyValidator = keyValidator;
|
||||
this.columnIndex = columnIndex;
|
||||
}
|
||||
|
||||
public abstract long add(DecoratedKey key, ByteBuffer value);
|
||||
public abstract RangeIterator<Long, Token> search(Expression expression);
|
||||
|
||||
public static MemIndex forColumn(AbstractType<?> keyValidator, ColumnIndex columnIndex)
|
||||
{
|
||||
return columnIndex.isLiteral()
|
||||
? new TrieMemIndex(keyValidator, columnIndex)
|
||||
: new SkipListMemIndex(keyValidator, columnIndex);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.memory;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentSkipListMap;
|
||||
import java.util.concurrent.ConcurrentSkipListSet;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public class SkipListMemIndex extends MemIndex
|
||||
{
|
||||
public static final int CSLM_OVERHEAD = 128; // average overhead of CSLM
|
||||
|
||||
private final ConcurrentSkipListMap<ByteBuffer, ConcurrentSkipListSet<DecoratedKey>> index;
|
||||
|
||||
public SkipListMemIndex(AbstractType<?> keyValidator, ColumnIndex columnIndex)
|
||||
{
|
||||
super(keyValidator, columnIndex);
|
||||
index = new ConcurrentSkipListMap<>(columnIndex.getValidator());
|
||||
}
|
||||
|
||||
public long add(DecoratedKey key, ByteBuffer value)
|
||||
{
|
||||
long overhead = CSLM_OVERHEAD; // DKs are shared
|
||||
ConcurrentSkipListSet<DecoratedKey> keys = index.get(value);
|
||||
|
||||
if (keys == null)
|
||||
{
|
||||
ConcurrentSkipListSet<DecoratedKey> newKeys = new ConcurrentSkipListSet<>(DecoratedKey.comparator);
|
||||
keys = index.putIfAbsent(value, newKeys);
|
||||
if (keys == null)
|
||||
{
|
||||
overhead += CSLM_OVERHEAD + value.remaining();
|
||||
keys = newKeys;
|
||||
}
|
||||
}
|
||||
|
||||
keys.add(key);
|
||||
|
||||
return overhead;
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> search(Expression expression)
|
||||
{
|
||||
ByteBuffer min = expression.lower == null ? null : expression.lower.value;
|
||||
ByteBuffer max = expression.upper == null ? null : expression.upper.value;
|
||||
|
||||
SortedMap<ByteBuffer, ConcurrentSkipListSet<DecoratedKey>> search;
|
||||
|
||||
if (min == null && max == null)
|
||||
{
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
if (min != null && max != null)
|
||||
{
|
||||
search = index.subMap(min, expression.lower.inclusive, max, expression.upper.inclusive);
|
||||
}
|
||||
else if (min == null)
|
||||
{
|
||||
search = index.headMap(max, expression.upper.inclusive);
|
||||
}
|
||||
else
|
||||
{
|
||||
search = index.tailMap(min, expression.lower.inclusive);
|
||||
}
|
||||
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
search.values().stream()
|
||||
.filter(keys -> !keys.isEmpty())
|
||||
.forEach(keys -> builder.add(new KeyRangeIterator(keys)));
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,254 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.memory;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentSkipListSet;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
|
||||
import com.googlecode.concurrenttrees.suffix.ConcurrentSuffixTree;
|
||||
import com.googlecode.concurrenttrees.radix.node.concrete.SmartArrayBasedNodeFactory;
|
||||
import com.googlecode.concurrenttrees.radix.node.Node;
|
||||
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.cassandra.index.sasi.memory.SkipListMemIndex.CSLM_OVERHEAD;
|
||||
|
||||
public class TrieMemIndex extends MemIndex
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(TrieMemIndex.class);
|
||||
|
||||
private final ConcurrentTrie index;
|
||||
|
||||
public TrieMemIndex(AbstractType<?> keyValidator, ColumnIndex columnIndex)
|
||||
{
|
||||
super(keyValidator, columnIndex);
|
||||
|
||||
switch (columnIndex.getMode().mode)
|
||||
{
|
||||
case CONTAINS:
|
||||
index = new ConcurrentSuffixTrie(columnIndex.getDefinition());
|
||||
break;
|
||||
|
||||
case PREFIX:
|
||||
index = new ConcurrentPrefixTrie(columnIndex.getDefinition());
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalStateException("Unsupported mode: " + columnIndex.getMode().mode);
|
||||
}
|
||||
}
|
||||
|
||||
public long add(DecoratedKey key, ByteBuffer value)
|
||||
{
|
||||
AbstractAnalyzer analyzer = columnIndex.getAnalyzer();
|
||||
analyzer.reset(value.duplicate());
|
||||
|
||||
long size = 0;
|
||||
while (analyzer.hasNext())
|
||||
{
|
||||
ByteBuffer term = analyzer.next();
|
||||
|
||||
if (term.remaining() >= OnDiskIndexBuilder.MAX_TERM_SIZE)
|
||||
{
|
||||
logger.info("Can't add term of column {} to index for key: {}, term size {} bytes, max allowed size {} bytes, use analyzed = true (if not yet set) for that column.",
|
||||
columnIndex.getColumnName(),
|
||||
keyValidator.getString(key.getKey()),
|
||||
term.remaining(),
|
||||
OnDiskIndexBuilder.MAX_TERM_SIZE);
|
||||
continue;
|
||||
}
|
||||
|
||||
size += index.add(columnIndex.getValidator().getString(term), key);
|
||||
}
|
||||
|
||||
return size;
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> search(Expression expression)
|
||||
{
|
||||
return index.search(expression);
|
||||
}
|
||||
|
||||
private static abstract class ConcurrentTrie
|
||||
{
|
||||
public static final SizeEstimatingNodeFactory NODE_FACTORY = new SizeEstimatingNodeFactory();
|
||||
|
||||
protected final ColumnDefinition definition;
|
||||
|
||||
public ConcurrentTrie(ColumnDefinition column)
|
||||
{
|
||||
definition = column;
|
||||
}
|
||||
|
||||
public long add(String value, DecoratedKey key)
|
||||
{
|
||||
long overhead = CSLM_OVERHEAD;
|
||||
ConcurrentSkipListSet<DecoratedKey> keys = get(value);
|
||||
if (keys == null)
|
||||
{
|
||||
ConcurrentSkipListSet<DecoratedKey> newKeys = new ConcurrentSkipListSet<>(DecoratedKey.comparator);
|
||||
keys = putIfAbsent(value, newKeys);
|
||||
if (keys == null)
|
||||
{
|
||||
overhead += CSLM_OVERHEAD + value.length();
|
||||
keys = newKeys;
|
||||
}
|
||||
}
|
||||
|
||||
keys.add(key);
|
||||
|
||||
// get and reset new memory size allocated by current thread
|
||||
overhead += NODE_FACTORY.currentUpdateSize();
|
||||
NODE_FACTORY.reset();
|
||||
|
||||
return overhead;
|
||||
}
|
||||
|
||||
public RangeIterator<Long, Token> search(Expression expression)
|
||||
{
|
||||
assert expression.getOp() == Expression.Op.EQ; // means that min == max
|
||||
|
||||
ByteBuffer prefix = expression.lower == null ? null : expression.lower.value;
|
||||
|
||||
Iterable<ConcurrentSkipListSet<DecoratedKey>> search = search(definition.cellValueType().getString(prefix));
|
||||
|
||||
RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
|
||||
for (ConcurrentSkipListSet<DecoratedKey> keys : search)
|
||||
{
|
||||
if (!keys.isEmpty())
|
||||
builder.add(new KeyRangeIterator(keys));
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
protected abstract ConcurrentSkipListSet<DecoratedKey> get(String value);
|
||||
protected abstract Iterable<ConcurrentSkipListSet<DecoratedKey>> search(String value);
|
||||
protected abstract ConcurrentSkipListSet<DecoratedKey> putIfAbsent(String value, ConcurrentSkipListSet<DecoratedKey> key);
|
||||
}
|
||||
|
||||
protected static class ConcurrentPrefixTrie extends ConcurrentTrie
|
||||
{
|
||||
private final ConcurrentRadixTree<ConcurrentSkipListSet<DecoratedKey>> trie;
|
||||
|
||||
private ConcurrentPrefixTrie(ColumnDefinition column)
|
||||
{
|
||||
super(column);
|
||||
trie = new ConcurrentRadixTree<>(NODE_FACTORY);
|
||||
}
|
||||
|
||||
public ConcurrentSkipListSet<DecoratedKey> get(String value)
|
||||
{
|
||||
return trie.getValueForExactKey(value);
|
||||
}
|
||||
|
||||
public ConcurrentSkipListSet<DecoratedKey> putIfAbsent(String value, ConcurrentSkipListSet<DecoratedKey> newKeys)
|
||||
{
|
||||
return trie.putIfAbsent(value, newKeys);
|
||||
}
|
||||
|
||||
public Iterable<ConcurrentSkipListSet<DecoratedKey>> search(String value)
|
||||
{
|
||||
return trie.getValuesForKeysStartingWith(value);
|
||||
}
|
||||
}
|
||||
|
||||
protected static class ConcurrentSuffixTrie extends ConcurrentTrie
|
||||
{
|
||||
private final ConcurrentSuffixTree<ConcurrentSkipListSet<DecoratedKey>> trie;
|
||||
|
||||
private ConcurrentSuffixTrie(ColumnDefinition column)
|
||||
{
|
||||
super(column);
|
||||
trie = new ConcurrentSuffixTree<>(NODE_FACTORY);
|
||||
}
|
||||
|
||||
public ConcurrentSkipListSet<DecoratedKey> get(String value)
|
||||
{
|
||||
return trie.getValueForExactKey(value);
|
||||
}
|
||||
|
||||
public ConcurrentSkipListSet<DecoratedKey> putIfAbsent(String value, ConcurrentSkipListSet<DecoratedKey> newKeys)
|
||||
{
|
||||
return trie.putIfAbsent(value, newKeys);
|
||||
}
|
||||
|
||||
public Iterable<ConcurrentSkipListSet<DecoratedKey>> search(String value)
|
||||
{
|
||||
return trie.getValuesForKeysContaining(value);
|
||||
}
|
||||
}
|
||||
|
||||
// This relies on the fact that all of the tree updates are done under exclusive write lock,
|
||||
// method would overestimate in certain circumstances e.g. when nodes are replaced in place,
|
||||
// but it's still better comparing to underestimate since it gives more breathing room for other memory users.
|
||||
private static class SizeEstimatingNodeFactory extends SmartArrayBasedNodeFactory
|
||||
{
|
||||
private final ThreadLocal<Long> updateSize = ThreadLocal.withInitial(() -> 0L);
|
||||
|
||||
public Node createNode(CharSequence edgeCharacters, Object value, List<Node> childNodes, boolean isRoot)
|
||||
{
|
||||
Node node = super.createNode(edgeCharacters, value, childNodes, isRoot);
|
||||
updateSize.set(updateSize.get() + measure(node));
|
||||
return node;
|
||||
}
|
||||
|
||||
public long currentUpdateSize()
|
||||
{
|
||||
return updateSize.get();
|
||||
}
|
||||
|
||||
public void reset()
|
||||
{
|
||||
updateSize.set(0L);
|
||||
}
|
||||
|
||||
private long measure(Node node)
|
||||
{
|
||||
// node with max overhead is CharArrayNodeLeafWithValue = 24B
|
||||
long overhead = 24;
|
||||
|
||||
// array of chars (2 bytes) + CharSequence overhead
|
||||
overhead += 24 + node.getIncomingEdge().length() * 2;
|
||||
|
||||
if (node.getOutgoingEdges() != null)
|
||||
{
|
||||
// 16 bytes for AtomicReferenceArray
|
||||
overhead += 16;
|
||||
overhead += 24 * node.getOutgoingEdges().size();
|
||||
}
|
||||
|
||||
return overhead;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,340 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.plan;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.cql3.Operator;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex;
|
||||
import org.apache.cassandra.index.sasi.utils.TypeUtil;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.db.marshal.UTF8Type;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
|
||||
import org.apache.commons.lang3.builder.HashCodeBuilder;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.Iterators;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class Expression
|
||||
{
|
||||
private static final Logger logger = LoggerFactory.getLogger(Expression.class);
|
||||
|
||||
public enum Op
|
||||
{
|
||||
EQ, NOT_EQ, RANGE
|
||||
}
|
||||
|
||||
private final QueryController controller;
|
||||
|
||||
public final AbstractAnalyzer analyzer;
|
||||
|
||||
public final ColumnIndex index;
|
||||
public final AbstractType<?> validator;
|
||||
public final boolean isLiteral;
|
||||
|
||||
@VisibleForTesting
|
||||
protected Op operation;
|
||||
|
||||
public Bound lower, upper;
|
||||
public List<ByteBuffer> exclusions = new ArrayList<>();
|
||||
|
||||
public Expression(Expression other)
|
||||
{
|
||||
this(other.controller, other.index);
|
||||
operation = other.operation;
|
||||
}
|
||||
|
||||
public Expression(QueryController controller, ColumnIndex columnIndex)
|
||||
{
|
||||
this.controller = controller;
|
||||
this.index = columnIndex;
|
||||
this.analyzer = columnIndex.getAnalyzer();
|
||||
this.validator = columnIndex.getValidator();
|
||||
this.isLiteral = columnIndex.isLiteral();
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
public Expression(String name, AbstractType<?> validator)
|
||||
{
|
||||
this(null, new ColumnIndex(UTF8Type.instance, ColumnDefinition.regularDef("sasi", "internal", name, validator), null));
|
||||
}
|
||||
|
||||
public Expression setLower(Bound newLower)
|
||||
{
|
||||
lower = newLower == null ? null : new Bound(newLower.value, newLower.inclusive);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Expression setUpper(Bound newUpper)
|
||||
{
|
||||
upper = newUpper == null ? null : new Bound(newUpper.value, newUpper.inclusive);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Expression setOp(Op op)
|
||||
{
|
||||
this.operation = op;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Expression add(Operator op, ByteBuffer value)
|
||||
{
|
||||
boolean lowerInclusive = false, upperInclusive = false;
|
||||
switch (op)
|
||||
{
|
||||
case EQ:
|
||||
lower = new Bound(value, true);
|
||||
upper = lower;
|
||||
operation = Op.EQ;
|
||||
break;
|
||||
|
||||
case NEQ:
|
||||
// index expressions are priority sorted
|
||||
// and NOT_EQ is the lowest priority, which means that operation type
|
||||
// is always going to be set before reaching it in case of RANGE or EQ.
|
||||
if (operation == null)
|
||||
{
|
||||
operation = Op.NOT_EQ;
|
||||
lower = new Bound(value, true);
|
||||
upper = lower;
|
||||
}
|
||||
else
|
||||
exclusions.add(value);
|
||||
break;
|
||||
|
||||
case LTE:
|
||||
upperInclusive = true;
|
||||
case LT:
|
||||
operation = Op.RANGE;
|
||||
upper = new Bound(value, upperInclusive);
|
||||
break;
|
||||
|
||||
case GTE:
|
||||
lowerInclusive = true;
|
||||
case GT:
|
||||
operation = Op.RANGE;
|
||||
lower = new Bound(value, lowerInclusive);
|
||||
break;
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public Expression addExclusion(ByteBuffer value)
|
||||
{
|
||||
exclusions.add(value);
|
||||
return this;
|
||||
}
|
||||
|
||||
public boolean contains(ByteBuffer value)
|
||||
{
|
||||
if (!TypeUtil.isValid(value, validator))
|
||||
{
|
||||
int size = value.remaining();
|
||||
if ((value = TypeUtil.tryUpcast(value, validator)) == null)
|
||||
{
|
||||
logger.error("Can't cast value for {} to size accepted by {}, value size is {} bytes.",
|
||||
index.getColumnName(),
|
||||
validator,
|
||||
size);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (lower != null)
|
||||
{
|
||||
// suffix check
|
||||
if (isLiteral)
|
||||
{
|
||||
if (!validateStringValue(value, lower.value))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// range or (not-)equals - (mainly) for numeric values
|
||||
int cmp = validator.compare(lower.value, value);
|
||||
|
||||
// in case of (NOT_)EQ lower == upper
|
||||
if (operation == Op.EQ || operation == Op.NOT_EQ)
|
||||
return cmp == 0;
|
||||
|
||||
if (cmp > 0 || (cmp == 0 && !lower.inclusive))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (upper != null && lower != upper)
|
||||
{
|
||||
// string (prefix or suffix) check
|
||||
if (isLiteral)
|
||||
{
|
||||
if (!validateStringValue(value, upper.value))
|
||||
return false;
|
||||
}
|
||||
else
|
||||
{
|
||||
// range - mainly for numeric values
|
||||
int cmp = validator.compare(upper.value, value);
|
||||
if (cmp < 0 || (cmp == 0 && !upper.inclusive))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// as a last step let's check exclusions for the given field,
|
||||
// this covers EQ/RANGE with exclusions.
|
||||
for (ByteBuffer term : exclusions)
|
||||
{
|
||||
if (isLiteral && validateStringValue(value, term))
|
||||
return false;
|
||||
else if (validator.compare(term, value) == 0)
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean validateStringValue(ByteBuffer columnValue, ByteBuffer requestedValue)
|
||||
{
|
||||
analyzer.reset(columnValue.duplicate());
|
||||
while (analyzer.hasNext())
|
||||
{
|
||||
ByteBuffer term = analyzer.next();
|
||||
if (ByteBufferUtil.contains(term, requestedValue))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public Op getOp()
|
||||
{
|
||||
return operation;
|
||||
}
|
||||
|
||||
public void checkpoint()
|
||||
{
|
||||
if (controller == null)
|
||||
return;
|
||||
|
||||
controller.checkpoint();
|
||||
}
|
||||
|
||||
public boolean hasLower()
|
||||
{
|
||||
return lower != null;
|
||||
}
|
||||
|
||||
public boolean hasUpper()
|
||||
{
|
||||
return upper != null;
|
||||
}
|
||||
|
||||
public boolean isLowerSatisfiedBy(OnDiskIndex.DataTerm term)
|
||||
{
|
||||
if (!hasLower())
|
||||
return true;
|
||||
|
||||
int cmp = term.compareTo(validator, lower.value, false);
|
||||
return cmp > 0 || cmp == 0 && lower.inclusive;
|
||||
}
|
||||
|
||||
public boolean isUpperSatisfiedBy(OnDiskIndex.DataTerm term)
|
||||
{
|
||||
if (!hasUpper())
|
||||
return true;
|
||||
|
||||
int cmp = term.compareTo(validator, upper.value, false);
|
||||
return cmp < 0 || cmp == 0 && upper.inclusive;
|
||||
}
|
||||
|
||||
public boolean isIndexed()
|
||||
{
|
||||
return index.isIndexed();
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return String.format("Expression{name: %s, op: %s, lower: (%s, %s), upper: (%s, %s), exclusions: %s}",
|
||||
index.getColumnName(),
|
||||
operation,
|
||||
lower == null ? "null" : validator.getString(lower.value),
|
||||
lower != null && lower.inclusive,
|
||||
upper == null ? "null" : validator.getString(upper.value),
|
||||
upper != null && upper.inclusive,
|
||||
Iterators.toString(Iterators.transform(exclusions.iterator(), validator::getString)));
|
||||
}
|
||||
|
||||
public int hashCode()
|
||||
{
|
||||
return new HashCodeBuilder().append(index.getColumnName())
|
||||
.append(operation)
|
||||
.append(validator)
|
||||
.append(lower).append(upper)
|
||||
.append(exclusions).build();
|
||||
}
|
||||
|
||||
public boolean equals(Object other)
|
||||
{
|
||||
if (!(other instanceof Expression))
|
||||
return false;
|
||||
|
||||
if (this == other)
|
||||
return true;
|
||||
|
||||
Expression o = (Expression) other;
|
||||
|
||||
return Objects.equals(index.getColumnName(), o.index.getColumnName())
|
||||
&& validator.equals(o.validator)
|
||||
&& operation == o.operation
|
||||
&& Objects.equals(lower, o.lower)
|
||||
&& Objects.equals(upper, o.upper)
|
||||
&& exclusions.equals(o.exclusions);
|
||||
}
|
||||
|
||||
|
||||
public static class Bound
|
||||
{
|
||||
public final ByteBuffer value;
|
||||
public final boolean inclusive;
|
||||
|
||||
public Bound(ByteBuffer value, boolean inclusive)
|
||||
{
|
||||
this.value = value;
|
||||
this.inclusive = inclusive;
|
||||
}
|
||||
|
||||
public boolean equals(Object other)
|
||||
{
|
||||
if (!(other instanceof Bound))
|
||||
return false;
|
||||
|
||||
Bound o = (Bound) other;
|
||||
return value.equals(o.value) && inclusive == o.inclusive;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,477 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.plan;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.config.ColumnDefinition;
|
||||
import org.apache.cassandra.config.ColumnDefinition.Kind;
|
||||
import org.apache.cassandra.cql3.Operator;
|
||||
import org.apache.cassandra.db.filter.RowFilter;
|
||||
import org.apache.cassandra.db.rows.Row;
|
||||
import org.apache.cassandra.db.rows.Unfiltered;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.analyzer.AbstractAnalyzer;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Expression.Op;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIntersectionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.collect.*;
|
||||
import org.apache.cassandra.utils.FBUtilities;
|
||||
|
||||
public class Operation extends RangeIterator<Long, Token>
|
||||
{
|
||||
public enum OperationType
|
||||
{
|
||||
AND, OR;
|
||||
|
||||
public boolean apply(boolean a, boolean b)
|
||||
{
|
||||
switch (this)
|
||||
{
|
||||
case OR:
|
||||
return a | b;
|
||||
|
||||
case AND:
|
||||
return a & b;
|
||||
|
||||
default:
|
||||
throw new AssertionError();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private final QueryController controller;
|
||||
|
||||
protected final OperationType op;
|
||||
protected final ListMultimap<ColumnDefinition, Expression> expressions;
|
||||
protected final RangeIterator<Long, Token> range;
|
||||
|
||||
protected Operation left, right;
|
||||
|
||||
private Operation(OperationType operation,
|
||||
QueryController controller,
|
||||
ListMultimap<ColumnDefinition, Expression> expressions,
|
||||
RangeIterator<Long, Token> range,
|
||||
Operation left, Operation right)
|
||||
{
|
||||
super(range);
|
||||
|
||||
this.op = operation;
|
||||
this.controller = controller;
|
||||
this.expressions = expressions;
|
||||
this.range = range;
|
||||
|
||||
this.left = left;
|
||||
this.right = right;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursive "satisfies" checks based on operation
|
||||
* and data from the lower level members using depth-first search
|
||||
* and bubbling the results back to the top level caller.
|
||||
*
|
||||
* Most of the work here is done by {@link #localSatisfiedBy(Unfiltered, boolean)}
|
||||
* see it's comment for details, if there are no local expressions
|
||||
* assigned to Operation it will call satisfiedBy(Row) on it's children.
|
||||
*
|
||||
* Query: first_name = X AND (last_name = Y OR address = XYZ AND street = IL AND city = C) OR (state = 'CA' AND country = 'US')
|
||||
* Row: key1: (first_name: X, last_name: Z, address: XYZ, street: IL, city: C, state: NY, country:US)
|
||||
*
|
||||
* #1 OR
|
||||
* / \
|
||||
* #2 (first_name) AND AND (state, country)
|
||||
* \
|
||||
* #3 (last_name) OR
|
||||
* \
|
||||
* #4 AND (address, street, city)
|
||||
*
|
||||
*
|
||||
* Evaluation of the key1 is top-down depth-first search:
|
||||
*
|
||||
* --- going down ---
|
||||
* Level #1 is evaluated, OR expression has to pull results from it's children which are at level #2 and OR them together,
|
||||
* Level #2 AND (state, country) could be be evaluated right away, AND (first_name) refers to it's "right" child from level #3
|
||||
* Level #3 OR (last_name) requests results from level #4
|
||||
* Level #4 AND (address, street, city) does logical AND between it's 3 fields, returns result back to level #3.
|
||||
* --- bubbling up ---
|
||||
* Level #3 computes OR between AND (address, street, city) result and it's "last_name" expression
|
||||
* Level #2 computes AND between "first_name" and result of level #3, AND (state, country) which is already computed
|
||||
* Level #1 does OR between results of AND (first_name) and AND (state, country) and returns final result.
|
||||
*
|
||||
* @param row The row to check.
|
||||
* @return true if give Row satisfied all of the expressions in the tree,
|
||||
* false otherwise.
|
||||
*/
|
||||
public boolean satisfiedBy(Unfiltered row, boolean allowMissingColumns)
|
||||
{
|
||||
boolean sideL, sideR;
|
||||
|
||||
if (expressions == null || expressions.isEmpty())
|
||||
{
|
||||
sideL = left != null && left.satisfiedBy(row, allowMissingColumns);
|
||||
sideR = right != null && right.satisfiedBy(row, allowMissingColumns);
|
||||
|
||||
// one of the expressions was skipped
|
||||
// because it had no indexes attached
|
||||
if (left == null)
|
||||
return sideR;
|
||||
}
|
||||
else
|
||||
{
|
||||
sideL = localSatisfiedBy(row, allowMissingColumns);
|
||||
|
||||
// if there is no right it means that this expression
|
||||
// is last in the sequence, we can just return result from local expressions
|
||||
if (right == null)
|
||||
return sideL;
|
||||
|
||||
sideR = right.satisfiedBy(row, allowMissingColumns);
|
||||
}
|
||||
|
||||
|
||||
return op.apply(sideL, sideR);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check every expression in the analyzed list to figure out if the
|
||||
* columns in the give row match all of the based on the operation
|
||||
* set to the current operation node.
|
||||
*
|
||||
* The algorithm is as follows: for every given expression from analyzed
|
||||
* list get corresponding column from the Row:
|
||||
* - apply {@link Expression#contains(ByteBuffer)}
|
||||
* method to figure out if it's satisfied;
|
||||
* - apply logical operation between boolean accumulator and current boolean result;
|
||||
* - if result == false and node's operation is AND return right away;
|
||||
*
|
||||
* After all of the expressions have been evaluated return resulting accumulator variable.
|
||||
*
|
||||
* Example:
|
||||
*
|
||||
* Operation = (op: AND, columns: [first_name = p, 5 < age < 7, last_name: y])
|
||||
* Row = (first_name: pavel, last_name: y, age: 6, timestamp: 15)
|
||||
*
|
||||
* #1 get "first_name" = p (expressions)
|
||||
* - row-get "first_name" => "pavel"
|
||||
* - compare "pavel" against "p" => true (current)
|
||||
* - set accumulator current => true (because this is expression #1)
|
||||
*
|
||||
* #2 get "last_name" = y (expressions)
|
||||
* - row-get "last_name" => "y"
|
||||
* - compare "y" against "y" => true (current)
|
||||
* - set accumulator to accumulator & current => true
|
||||
*
|
||||
* #3 get 5 < "age" < 7 (expressions)
|
||||
* - row-get "age" => "6"
|
||||
* - compare 5 < 6 < 7 => true (current)
|
||||
* - set accumulator to accumulator & current => true
|
||||
*
|
||||
* #4 return accumulator => true (row satisfied all of the conditions)
|
||||
*
|
||||
* @param row The row to check.
|
||||
* @return true if give Row satisfied all of the analyzed expressions,
|
||||
* false otherwise.
|
||||
*/
|
||||
private boolean localSatisfiedBy(Unfiltered row, boolean allowMissingColumns)
|
||||
{
|
||||
if (row == null || !row.isRow())
|
||||
return false;
|
||||
|
||||
final int now = FBUtilities.nowInSeconds();
|
||||
boolean result = false;
|
||||
int idx = 0;
|
||||
|
||||
for (ColumnDefinition column : expressions.keySet())
|
||||
{
|
||||
if (column.kind == Kind.PARTITION_KEY)
|
||||
continue;
|
||||
|
||||
ByteBuffer value = ColumnIndex.getValueOf(column, (Row) row, now);
|
||||
boolean isMissingColumn = value == null;
|
||||
|
||||
if (!allowMissingColumns && isMissingColumn)
|
||||
throw new IllegalStateException("All indexed columns should be included into the column slice, missing: " + column);
|
||||
|
||||
boolean isMatch = false;
|
||||
// If there is a column with multiple expressions that effectively means an OR
|
||||
// e.g. comment = 'x y z' could be split into 'comment' EQ 'x', 'comment' EQ 'y', 'comment' EQ 'z'
|
||||
// by analyzer, in situation like that we only need to check if at least one of expressions matches,
|
||||
// and there is no hit on the NOT_EQ (if any) which are always at the end of the filter list.
|
||||
// Loop always starts from the end of the list, which makes it possible to break after the last
|
||||
// NOT_EQ condition on first EQ/RANGE condition satisfied, instead of checking every
|
||||
// single expression in the column filter list.
|
||||
List<Expression> filters = expressions.get(column);
|
||||
for (int i = filters.size() - 1; i >= 0; i--)
|
||||
{
|
||||
Expression expression = filters.get(i);
|
||||
isMatch = !isMissingColumn && expression.contains(value);
|
||||
if (expression.getOp() == Op.NOT_EQ)
|
||||
{
|
||||
// since this is NOT_EQ operation we have to
|
||||
// inverse match flag (to check against other expressions),
|
||||
// and break in case of negative inverse because that means
|
||||
// that it's a positive hit on the not-eq clause.
|
||||
isMatch = !isMatch;
|
||||
if (!isMatch)
|
||||
break;
|
||||
} // if it was a match on EQ/RANGE or column is missing
|
||||
else if (isMatch || isMissingColumn)
|
||||
break;
|
||||
}
|
||||
|
||||
if (idx++ == 0)
|
||||
{
|
||||
result = isMatch;
|
||||
continue;
|
||||
}
|
||||
|
||||
result = op.apply(result, isMatch);
|
||||
|
||||
// exit early because we already got a single false
|
||||
if (op == OperationType.AND && !result)
|
||||
return false;
|
||||
}
|
||||
|
||||
return idx == 0 || result;
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected static ListMultimap<ColumnDefinition, Expression> analyzeGroup(QueryController controller,
|
||||
OperationType op,
|
||||
List<RowFilter.Expression> expressions)
|
||||
{
|
||||
ListMultimap<ColumnDefinition, Expression> analyzed = ArrayListMultimap.create();
|
||||
|
||||
// sort all of the expressions in the operation by name and priority of the logical operator
|
||||
// this gives us an efficient way to handle inequality and combining into ranges without extra processing
|
||||
// and converting expressions from one type to another.
|
||||
Collections.sort(expressions, (a, b) -> {
|
||||
int cmp = a.column().compareTo(b.column());
|
||||
return cmp == 0 ? -Integer.compare(getPriority(a.operator()), getPriority(b.operator())) : cmp;
|
||||
});
|
||||
|
||||
for (final RowFilter.Expression e : expressions)
|
||||
{
|
||||
ColumnIndex columnIndex = controller.getIndex(e);
|
||||
List<Expression> perColumn = analyzed.get(e.column());
|
||||
|
||||
if (columnIndex == null)
|
||||
columnIndex = new ColumnIndex(controller.getKeyValidator(), e.column(), null);
|
||||
|
||||
AbstractAnalyzer analyzer = columnIndex.getAnalyzer();
|
||||
analyzer.reset(e.getIndexValue());
|
||||
|
||||
// EQ/NOT_EQ can have multiple expressions e.g. text = "Hello World",
|
||||
// becomes text = "Hello" OR text = "World" because "space" is always interpreted as a split point (by analyzer),
|
||||
// NOT_EQ is made an independent expression only in case of pre-existing multiple EQ expressions, or
|
||||
// if there is no EQ operations and NOT_EQ is met or a single NOT_EQ expression present,
|
||||
// in such case we know exactly that there would be no more EQ/RANGE expressions for given column
|
||||
// since NOT_EQ has the lowest priority.
|
||||
if (e.operator() == Operator.EQ
|
||||
|| (e.operator() == Operator.NEQ
|
||||
&& (perColumn.size() == 0 || perColumn.size() > 1
|
||||
|| (perColumn.size() == 1 && perColumn.get(0).getOp() == Op.NOT_EQ))))
|
||||
{
|
||||
while (analyzer.hasNext())
|
||||
{
|
||||
final ByteBuffer token = analyzer.next();
|
||||
perColumn.add(new Expression(controller, columnIndex).add(e.operator(), token));
|
||||
}
|
||||
}
|
||||
else
|
||||
// "range" or not-equals operator, combines both bounds together into the single expression,
|
||||
// iff operation of the group is AND, otherwise we are forced to create separate expressions,
|
||||
// not-equals is combined with the range iff operator is AND.
|
||||
{
|
||||
Expression range;
|
||||
if (perColumn.size() == 0 || op != OperationType.AND)
|
||||
perColumn.add((range = new Expression(controller, columnIndex)));
|
||||
else
|
||||
range = Iterables.getLast(perColumn);
|
||||
|
||||
while (analyzer.hasNext())
|
||||
range.add(e.operator(), analyzer.next());
|
||||
}
|
||||
}
|
||||
|
||||
return analyzed;
|
||||
}
|
||||
|
||||
private static int getPriority(Operator op)
|
||||
{
|
||||
switch (op)
|
||||
{
|
||||
case EQ:
|
||||
return 4;
|
||||
|
||||
case GTE:
|
||||
case GT:
|
||||
return 3;
|
||||
|
||||
case LTE:
|
||||
case LT:
|
||||
return 2;
|
||||
|
||||
case NEQ:
|
||||
return 1;
|
||||
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
protected Token computeNext()
|
||||
{
|
||||
return range != null && range.hasNext() ? range.next() : endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(Long nextToken)
|
||||
{
|
||||
if (range != null)
|
||||
range.skipTo(nextToken);
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
controller.releaseIndexes(this);
|
||||
}
|
||||
|
||||
public static class Builder
|
||||
{
|
||||
private final QueryController controller;
|
||||
|
||||
protected final OperationType op;
|
||||
protected final List<RowFilter.Expression> expressions;
|
||||
|
||||
protected Builder left, right;
|
||||
|
||||
public Builder(OperationType operation, QueryController controller, RowFilter.Expression... columns)
|
||||
{
|
||||
this.op = operation;
|
||||
this.controller = controller;
|
||||
this.expressions = new ArrayList<>();
|
||||
Collections.addAll(expressions, columns);
|
||||
}
|
||||
|
||||
public Builder setRight(Builder operation)
|
||||
{
|
||||
this.right = operation;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setLeft(Builder operation)
|
||||
{
|
||||
this.left = operation;
|
||||
return this;
|
||||
}
|
||||
|
||||
public void add(RowFilter.Expression e)
|
||||
{
|
||||
expressions.add(e);
|
||||
}
|
||||
|
||||
public void add(Collection<RowFilter.Expression> newExpressions)
|
||||
{
|
||||
if (expressions != null)
|
||||
expressions.addAll(newExpressions);
|
||||
}
|
||||
|
||||
public Operation complete()
|
||||
{
|
||||
if (!expressions.isEmpty())
|
||||
{
|
||||
ListMultimap<ColumnDefinition, Expression> analyzedExpressions = analyzeGroup(controller, op, expressions);
|
||||
RangeIterator.Builder<Long, Token> range = controller.getIndexes(op, analyzedExpressions.values());
|
||||
|
||||
Operation rightOp = null;
|
||||
if (right != null)
|
||||
{
|
||||
rightOp = right.complete();
|
||||
range.add(rightOp);
|
||||
}
|
||||
|
||||
return new Operation(op, controller, analyzedExpressions, range.build(), null, rightOp);
|
||||
}
|
||||
else
|
||||
{
|
||||
Operation leftOp = null, rightOp = null;
|
||||
boolean leftIndexes = false, rightIndexes = false;
|
||||
|
||||
if (left != null)
|
||||
{
|
||||
leftOp = left.complete();
|
||||
leftIndexes = leftOp != null && leftOp.range != null;
|
||||
}
|
||||
|
||||
if (right != null)
|
||||
{
|
||||
rightOp = right.complete();
|
||||
rightIndexes = rightOp != null && rightOp.range != null;
|
||||
}
|
||||
|
||||
RangeIterator<Long, Token> join;
|
||||
/**
|
||||
* Operation should allow one of it's sub-trees to wrap no indexes, that is related to the fact that we
|
||||
* have to accept defined-but-not-indexed columns as well as key range as IndexExpressions.
|
||||
*
|
||||
* Two cases are possible:
|
||||
*
|
||||
* only left child produced indexed iterators, that could happen when there are two columns
|
||||
* or key range on the right:
|
||||
*
|
||||
* AND
|
||||
* / \
|
||||
* OR \
|
||||
* / \ AND
|
||||
* a b / \
|
||||
* key key
|
||||
*
|
||||
* only right child produced indexed iterators:
|
||||
*
|
||||
* AND
|
||||
* / \
|
||||
* AND a
|
||||
* / \
|
||||
* key key
|
||||
*/
|
||||
if (leftIndexes && !rightIndexes)
|
||||
join = leftOp;
|
||||
else if (!leftIndexes && rightIndexes)
|
||||
join = rightOp;
|
||||
else if (leftIndexes)
|
||||
{
|
||||
RangeIterator.Builder<Long, Token> builder = op == OperationType.OR
|
||||
? RangeUnionIterator.<Long, Token>builder()
|
||||
: RangeIntersectionIterator.<Long, Token>builder();
|
||||
|
||||
join = builder.add(leftOp).add(rightOp).build();
|
||||
}
|
||||
else
|
||||
throw new AssertionError("both sub-trees have 0 indexes.");
|
||||
|
||||
return new Operation(op, controller, null, join, leftOp, rightOp);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,261 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.plan;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.cassandra.config.CFMetaData;
|
||||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore.RefViewFragment;
|
||||
import org.apache.cassandra.db.filter.DataLimits;
|
||||
import org.apache.cassandra.db.filter.RowFilter;
|
||||
import org.apache.cassandra.db.lifecycle.SSTableSet;
|
||||
import org.apache.cassandra.db.rows.UnfilteredRowIterator;
|
||||
import org.apache.cassandra.index.Index;
|
||||
import org.apache.cassandra.index.sasi.SASIIndex;
|
||||
import org.apache.cassandra.index.sasi.SSTableIndex;
|
||||
import org.apache.cassandra.index.sasi.TermIterator;
|
||||
import org.apache.cassandra.index.sasi.conf.ColumnIndex;
|
||||
import org.apache.cassandra.index.sasi.conf.view.View;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.exceptions.TimeQuotaExceededException;
|
||||
import org.apache.cassandra.index.sasi.plan.Operation.OperationType;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIntersectionIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeIterator;
|
||||
import org.apache.cassandra.index.sasi.utils.RangeUnionIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
public class QueryController
|
||||
{
|
||||
private final long executionQuota;
|
||||
private final long executionStart;
|
||||
|
||||
private final ColumnFamilyStore cfs;
|
||||
private final PartitionRangeReadCommand command;
|
||||
private final Map<Collection<Expression>, List<RangeIterator<Long, Token>>> resources = new HashMap<>();
|
||||
private final RefViewFragment scope;
|
||||
private final Set<SSTableReader> sstables;
|
||||
|
||||
public QueryController(ColumnFamilyStore cfs, PartitionRangeReadCommand command, long timeQuotaMs)
|
||||
{
|
||||
this.cfs = cfs;
|
||||
this.command = command;
|
||||
this.executionQuota = TimeUnit.MILLISECONDS.toNanos(timeQuotaMs);
|
||||
this.executionStart = System.nanoTime();
|
||||
this.scope = getSSTableScope(cfs, command);
|
||||
this.sstables = new HashSet<>(scope.sstables);
|
||||
}
|
||||
|
||||
public boolean isForThrift()
|
||||
{
|
||||
return command.isForThrift();
|
||||
}
|
||||
|
||||
public CFMetaData metadata()
|
||||
{
|
||||
return command.metadata();
|
||||
}
|
||||
|
||||
public Collection<RowFilter.Expression> getExpressions()
|
||||
{
|
||||
return command.rowFilter().getExpressions();
|
||||
}
|
||||
|
||||
public DataRange dataRange()
|
||||
{
|
||||
return command.dataRange();
|
||||
}
|
||||
|
||||
public AbstractType<?> getKeyValidator()
|
||||
{
|
||||
return cfs.metadata.getKeyValidator();
|
||||
}
|
||||
|
||||
public ColumnIndex getIndex(RowFilter.Expression expression)
|
||||
{
|
||||
Optional<Index> index = cfs.indexManager.getBestIndexFor(expression);
|
||||
return index.isPresent() ? ((SASIIndex) index.get()).getIndex() : null;
|
||||
}
|
||||
|
||||
|
||||
public UnfilteredRowIterator getPartition(DecoratedKey key, ReadExecutionController executionController)
|
||||
{
|
||||
if (key == null)
|
||||
throw new NullPointerException();
|
||||
try
|
||||
{
|
||||
SinglePartitionReadCommand partition = SinglePartitionReadCommand.create(command.isForThrift(),
|
||||
cfs.metadata,
|
||||
command.nowInSec(),
|
||||
command.columnFilter(),
|
||||
command.rowFilter().withoutExpressions(),
|
||||
DataLimits.NONE,
|
||||
key,
|
||||
command.clusteringIndexFilter(key));
|
||||
|
||||
return partition.queryMemtableAndDisk(cfs, executionController.baseReadOpOrderGroup());
|
||||
}
|
||||
finally
|
||||
{
|
||||
checkpoint();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a range iterator from the given list of expressions by applying given operation (OR/AND).
|
||||
* Building of such iterator involves index search, results of which are persisted in the internal resources list
|
||||
* and can be released later via {@link QueryController#releaseIndexes(Operation)}.
|
||||
*
|
||||
* @param op The operation type to coalesce expressions with.
|
||||
* @param expressions The expressions to build range iterator from (expressions with not results are ignored).
|
||||
*
|
||||
* @return The range builder based on given expressions and operation type.
|
||||
*/
|
||||
public RangeIterator.Builder<Long, Token> getIndexes(OperationType op, Collection<Expression> expressions)
|
||||
{
|
||||
if (resources.containsKey(expressions))
|
||||
throw new IllegalArgumentException("Can't process the same expressions multiple times.");
|
||||
|
||||
RangeIterator.Builder<Long, Token> builder = op == OperationType.OR
|
||||
? RangeUnionIterator.<Long, Token>builder()
|
||||
: RangeIntersectionIterator.<Long, Token>builder();
|
||||
|
||||
List<RangeIterator<Long, Token>> perIndexUnions = new ArrayList<>();
|
||||
|
||||
for (Map.Entry<Expression, Set<SSTableIndex>> e : getView(op, expressions).entrySet())
|
||||
{
|
||||
RangeIterator<Long, Token> index = TermIterator.build(e.getKey(), e.getValue());
|
||||
|
||||
if (index == null)
|
||||
continue;
|
||||
|
||||
builder.add(index);
|
||||
perIndexUnions.add(index);
|
||||
}
|
||||
|
||||
resources.put(expressions, perIndexUnions);
|
||||
return builder;
|
||||
}
|
||||
|
||||
public void checkpoint()
|
||||
{
|
||||
if ((System.nanoTime() - executionStart) >= executionQuota)
|
||||
throw new TimeQuotaExceededException();
|
||||
}
|
||||
|
||||
public void releaseIndexes(Operation operation)
|
||||
{
|
||||
if (operation.expressions != null)
|
||||
releaseIndexes(resources.remove(operation.expressions.values()));
|
||||
}
|
||||
|
||||
private void releaseIndexes(List<RangeIterator<Long, Token>> indexes)
|
||||
{
|
||||
if (indexes == null)
|
||||
return;
|
||||
|
||||
indexes.forEach(FileUtils::closeQuietly);
|
||||
}
|
||||
|
||||
public void finish()
|
||||
{
|
||||
try
|
||||
{
|
||||
resources.values().forEach(this::releaseIndexes);
|
||||
}
|
||||
finally
|
||||
{
|
||||
scope.release();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<Expression, Set<SSTableIndex>> getView(OperationType op, Collection<Expression> expressions)
|
||||
{
|
||||
// first let's determine the primary expression if op is AND
|
||||
Pair<Expression, Set<SSTableIndex>> primary = (op == OperationType.AND) ? calculatePrimary(expressions) : null;
|
||||
|
||||
Map<Expression, Set<SSTableIndex>> indexes = new HashMap<>();
|
||||
for (Expression e : expressions)
|
||||
{
|
||||
// NO_EQ and non-index column query should only act as FILTER BY for satisfiedBy(Row) method
|
||||
// because otherwise it likely to go through the whole index.
|
||||
if (!e.isIndexed() || e.getOp() == Expression.Op.NOT_EQ)
|
||||
continue;
|
||||
|
||||
// primary expression, we'll have to add as is
|
||||
if (primary != null && e.equals(primary.left))
|
||||
{
|
||||
indexes.put(primary.left, primary.right);
|
||||
continue;
|
||||
}
|
||||
|
||||
View view = e.index.getView();
|
||||
if (view == null)
|
||||
continue;
|
||||
|
||||
Set<SSTableIndex> readers = new HashSet<>();
|
||||
if (primary != null && primary.right.size() > 0)
|
||||
{
|
||||
for (SSTableIndex index : primary.right)
|
||||
readers.addAll(view.match(index.minKey(), index.maxKey()));
|
||||
}
|
||||
else
|
||||
{
|
||||
readers.addAll(view.match(sstables, e));
|
||||
}
|
||||
|
||||
indexes.put(e, readers);
|
||||
}
|
||||
|
||||
return indexes;
|
||||
}
|
||||
|
||||
private Pair<Expression, Set<SSTableIndex>> calculatePrimary(Collection<Expression> expressions)
|
||||
{
|
||||
Expression expression = null;
|
||||
Set<SSTableIndex> primaryIndexes = Collections.emptySet();
|
||||
|
||||
for (Expression e : expressions)
|
||||
{
|
||||
if (!e.isIndexed())
|
||||
continue;
|
||||
|
||||
View view = e.index.getView();
|
||||
if (view == null)
|
||||
continue;
|
||||
|
||||
Set<SSTableIndex> indexes = view.match(sstables, e);
|
||||
if (primaryIndexes.size() > indexes.size())
|
||||
{
|
||||
primaryIndexes = indexes;
|
||||
expression = e;
|
||||
}
|
||||
}
|
||||
|
||||
return expression == null ? null : Pair.create(expression, primaryIndexes);
|
||||
}
|
||||
|
||||
private static RefViewFragment getSSTableScope(ColumnFamilyStore cfs, PartitionRangeReadCommand command)
|
||||
{
|
||||
return cfs.selectAndReference(org.apache.cassandra.db.lifecycle.View.select(SSTableSet.CANONICAL, command.dataRange().keyRange()));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.plan;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.cassandra.config.CFMetaData;
|
||||
import org.apache.cassandra.db.*;
|
||||
import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator;
|
||||
import org.apache.cassandra.db.rows.*;
|
||||
import org.apache.cassandra.dht.AbstractBounds;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.plan.Operation.OperationType;
|
||||
import org.apache.cassandra.exceptions.RequestTimeoutException;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.utils.AbstractIterator;
|
||||
|
||||
public class QueryPlan
|
||||
{
|
||||
private final QueryController controller;
|
||||
|
||||
public QueryPlan(ColumnFamilyStore cfs, ReadCommand command, long executionQuotaMs)
|
||||
{
|
||||
this.controller = new QueryController(cfs, (PartitionRangeReadCommand) command, executionQuotaMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts expressions into operation tree (which is currently just a single AND).
|
||||
*
|
||||
* Operation tree allows us to do a couple of important optimizations
|
||||
* namely, group flattening for AND operations (query rewrite), expression bounds checks,
|
||||
* "satisfies by" checks for resulting rows with an early exit.
|
||||
*
|
||||
* @return root of the operations tree.
|
||||
*/
|
||||
private Operation analyze()
|
||||
{
|
||||
try
|
||||
{
|
||||
Operation.Builder and = new Operation.Builder(OperationType.AND, controller);
|
||||
controller.getExpressions().forEach(and::add);
|
||||
return and.complete();
|
||||
}
|
||||
catch (Exception | Error e)
|
||||
{
|
||||
controller.finish();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public UnfilteredPartitionIterator execute(ReadExecutionController executionController) throws RequestTimeoutException
|
||||
{
|
||||
return new ResultIterator(analyze(), controller, executionController);
|
||||
}
|
||||
|
||||
private static class ResultIterator extends AbstractIterator<UnfilteredRowIterator> implements UnfilteredPartitionIterator
|
||||
{
|
||||
private final AbstractBounds<PartitionPosition> keyRange;
|
||||
private final Operation operationTree;
|
||||
private final QueryController controller;
|
||||
private final ReadExecutionController executionController;
|
||||
|
||||
private Iterator<DecoratedKey> currentKeys = null;
|
||||
|
||||
public ResultIterator(Operation operationTree, QueryController controller, ReadExecutionController executionController)
|
||||
{
|
||||
this.keyRange = controller.dataRange().keyRange();
|
||||
this.operationTree = operationTree;
|
||||
this.controller = controller;
|
||||
this.executionController = executionController;
|
||||
if (operationTree != null)
|
||||
operationTree.skipTo((Long) keyRange.left.getToken().getTokenValue());
|
||||
}
|
||||
|
||||
protected UnfilteredRowIterator computeNext()
|
||||
{
|
||||
if (operationTree == null)
|
||||
return endOfData();
|
||||
|
||||
for (;;)
|
||||
{
|
||||
if (currentKeys == null || !currentKeys.hasNext())
|
||||
{
|
||||
if (!operationTree.hasNext())
|
||||
return endOfData();
|
||||
|
||||
Token token = operationTree.next();
|
||||
currentKeys = token.iterator();
|
||||
}
|
||||
|
||||
while (currentKeys.hasNext())
|
||||
{
|
||||
DecoratedKey key = currentKeys.next();
|
||||
|
||||
if (!keyRange.right.isMinimum() && keyRange.right.compareTo(key) < 0)
|
||||
return endOfData();
|
||||
|
||||
try (UnfilteredRowIterator partition = controller.getPartition(key, executionController))
|
||||
{
|
||||
List<Unfiltered> clusters = new ArrayList<>();
|
||||
while (partition.hasNext())
|
||||
{
|
||||
Unfiltered row = partition.next();
|
||||
if (operationTree.satisfiedBy(row, true))
|
||||
clusters.add(row);
|
||||
}
|
||||
|
||||
if (!clusters.isEmpty())
|
||||
return new PartitionIterator(partition, clusters);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class PartitionIterator extends AbstractUnfilteredRowIterator
|
||||
{
|
||||
private final Iterator<Unfiltered> rows;
|
||||
|
||||
public PartitionIterator(UnfilteredRowIterator partition, Collection<Unfiltered> content)
|
||||
{
|
||||
super(partition.metadata(),
|
||||
partition.partitionKey(),
|
||||
partition.partitionLevelDeletion(),
|
||||
partition.columns(),
|
||||
partition.staticRow(),
|
||||
partition.isReverseOrder(),
|
||||
partition.stats());
|
||||
|
||||
rows = content.iterator();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Unfiltered computeNext()
|
||||
{
|
||||
return rows.hasNext() ? rows.next() : endOfData();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean isForThrift()
|
||||
{
|
||||
return controller.isForThrift();
|
||||
}
|
||||
|
||||
public CFMetaData metadata()
|
||||
{
|
||||
return controller.metadata();
|
||||
}
|
||||
|
||||
public void close()
|
||||
{
|
||||
FileUtils.closeQuietly(operationTree);
|
||||
controller.finish();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public class ByteTerm extends Term<ByteBuffer>
|
||||
{
|
||||
public ByteTerm(int position, ByteBuffer value, TokenTreeBuilder tokens)
|
||||
{
|
||||
super(position, value, tokens);
|
||||
}
|
||||
|
||||
public ByteBuffer getTerm()
|
||||
{
|
||||
return value.duplicate();
|
||||
}
|
||||
|
||||
public ByteBuffer getSuffix(int start)
|
||||
{
|
||||
return (ByteBuffer) value.duplicate().position(value.position() + start);
|
||||
}
|
||||
|
||||
public int compareTo(AbstractType<?> comparator, Term other)
|
||||
{
|
||||
return comparator.compare(value, (ByteBuffer) other.value);
|
||||
}
|
||||
|
||||
public int length()
|
||||
{
|
||||
return value.remaining();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.CharBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
|
||||
public class CharTerm extends Term<CharBuffer>
|
||||
{
|
||||
public CharTerm(int position, CharBuffer value, TokenTreeBuilder tokens)
|
||||
{
|
||||
super(position, value, tokens);
|
||||
}
|
||||
|
||||
public ByteBuffer getTerm()
|
||||
{
|
||||
return Charsets.UTF_8.encode(value.duplicate());
|
||||
}
|
||||
|
||||
public ByteBuffer getSuffix(int start)
|
||||
{
|
||||
return Charsets.UTF_8.encode(value.subSequence(value.position() + start, value.remaining()));
|
||||
}
|
||||
|
||||
public int compareTo(AbstractType<?> comparator, Term other)
|
||||
{
|
||||
return value.compareTo((CharBuffer) other.value);
|
||||
}
|
||||
|
||||
public int length()
|
||||
{
|
||||
return value.length();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
public class IntegralSA extends SA<ByteBuffer>
|
||||
{
|
||||
public IntegralSA(AbstractType<?> comparator, OnDiskIndexBuilder.Mode mode)
|
||||
{
|
||||
super(comparator, mode);
|
||||
}
|
||||
|
||||
public Term<ByteBuffer> getTerm(ByteBuffer termValue, TokenTreeBuilder tokens)
|
||||
{
|
||||
return new ByteTerm(charCount, termValue, tokens);
|
||||
}
|
||||
|
||||
public TermIterator finish()
|
||||
{
|
||||
return new IntegralSuffixIterator();
|
||||
}
|
||||
|
||||
|
||||
private class IntegralSuffixIterator extends TermIterator
|
||||
{
|
||||
private final Iterator<Term<ByteBuffer>> termIterator;
|
||||
|
||||
public IntegralSuffixIterator()
|
||||
{
|
||||
Collections.sort(terms, new Comparator<Term<?>>()
|
||||
{
|
||||
public int compare(Term<?> a, Term<?> b)
|
||||
{
|
||||
return a.compareTo(comparator, b);
|
||||
}
|
||||
});
|
||||
|
||||
termIterator = terms.iterator();
|
||||
}
|
||||
|
||||
public ByteBuffer minTerm()
|
||||
{
|
||||
return terms.get(0).getTerm();
|
||||
}
|
||||
|
||||
public ByteBuffer maxTerm()
|
||||
{
|
||||
return terms.get(terms.size() - 1).getTerm();
|
||||
}
|
||||
|
||||
protected Pair<ByteBuffer, TokenTreeBuilder> computeNext()
|
||||
{
|
||||
if (!termIterator.hasNext())
|
||||
return endOfData();
|
||||
|
||||
Term<ByteBuffer> term = termIterator.next();
|
||||
return Pair.create(term.getTerm(), term.getTokens().finish());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.Buffer;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder.Mode;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public abstract class SA<T extends Buffer>
|
||||
{
|
||||
protected final AbstractType<?> comparator;
|
||||
protected final Mode mode;
|
||||
|
||||
protected final List<Term<T>> terms = new ArrayList<>();
|
||||
protected int charCount = 0;
|
||||
|
||||
public SA(AbstractType<?> comparator, Mode mode)
|
||||
{
|
||||
this.comparator = comparator;
|
||||
this.mode = mode;
|
||||
}
|
||||
|
||||
public Mode getMode()
|
||||
{
|
||||
return mode;
|
||||
}
|
||||
|
||||
public void add(ByteBuffer termValue, TokenTreeBuilder tokens)
|
||||
{
|
||||
Term<T> term = getTerm(termValue, tokens);
|
||||
terms.add(term);
|
||||
charCount += term.length();
|
||||
}
|
||||
|
||||
public abstract TermIterator finish();
|
||||
|
||||
protected abstract Term<T> getTerm(ByteBuffer termValue, TokenTreeBuilder tokens);
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.CharBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
import net.mintern.primitive.Primitive;
|
||||
|
||||
public class SuffixSA extends SA<CharBuffer>
|
||||
{
|
||||
public SuffixSA(AbstractType<?> comparator, OnDiskIndexBuilder.Mode mode)
|
||||
{
|
||||
super(comparator, mode);
|
||||
}
|
||||
|
||||
protected Term<CharBuffer> getTerm(ByteBuffer termValue, TokenTreeBuilder tokens)
|
||||
{
|
||||
return new CharTerm(charCount, Charsets.UTF_8.decode(termValue.duplicate()), tokens);
|
||||
}
|
||||
|
||||
public TermIterator finish()
|
||||
{
|
||||
return new SASuffixIterator();
|
||||
}
|
||||
|
||||
private class SASuffixIterator extends TermIterator
|
||||
{
|
||||
private final long[] suffixes;
|
||||
|
||||
private int current = 0;
|
||||
private ByteBuffer lastProcessedSuffix;
|
||||
private TokenTreeBuilder container;
|
||||
|
||||
public SASuffixIterator()
|
||||
{
|
||||
// each element has term index and char position encoded as two 32-bit integers
|
||||
// to avoid binary search per suffix while sorting suffix array.
|
||||
suffixes = new long[charCount];
|
||||
|
||||
long termIndex = -1, currentTermLength = -1;
|
||||
for (int i = 0; i < charCount; i++)
|
||||
{
|
||||
if (i >= currentTermLength || currentTermLength == -1)
|
||||
{
|
||||
Term currentTerm = terms.get((int) ++termIndex);
|
||||
currentTermLength = currentTerm.getPosition() + currentTerm.length();
|
||||
}
|
||||
|
||||
suffixes[i] = (termIndex << 32) | i;
|
||||
}
|
||||
|
||||
Primitive.sort(suffixes, (a, b) -> {
|
||||
Term aTerm = terms.get((int) (a >>> 32));
|
||||
Term bTerm = terms.get((int) (b >>> 32));
|
||||
return comparator.compare(aTerm.getSuffix(((int) a) - aTerm.getPosition()),
|
||||
bTerm.getSuffix(((int) b) - bTerm.getPosition()));
|
||||
});
|
||||
}
|
||||
|
||||
private Pair<ByteBuffer, TokenTreeBuilder> suffixAt(int position)
|
||||
{
|
||||
long index = suffixes[position];
|
||||
Term term = terms.get((int) (index >>> 32));
|
||||
return Pair.create(term.getSuffix(((int) index) - term.getPosition()), term.getTokens());
|
||||
}
|
||||
|
||||
public ByteBuffer minTerm()
|
||||
{
|
||||
return suffixAt(0).left;
|
||||
}
|
||||
|
||||
public ByteBuffer maxTerm()
|
||||
{
|
||||
return suffixAt(suffixes.length - 1).left;
|
||||
}
|
||||
|
||||
protected Pair<ByteBuffer, TokenTreeBuilder> computeNext()
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (current >= suffixes.length)
|
||||
{
|
||||
if (lastProcessedSuffix == null)
|
||||
return endOfData();
|
||||
|
||||
Pair<ByteBuffer, TokenTreeBuilder> result = finishSuffix();
|
||||
|
||||
lastProcessedSuffix = null;
|
||||
return result;
|
||||
}
|
||||
|
||||
Pair<ByteBuffer, TokenTreeBuilder> suffix = suffixAt(current++);
|
||||
|
||||
if (lastProcessedSuffix == null)
|
||||
{
|
||||
lastProcessedSuffix = suffix.left;
|
||||
container = new TokenTreeBuilder(suffix.right.getTokens());
|
||||
}
|
||||
else if (comparator.compare(lastProcessedSuffix, suffix.left) == 0)
|
||||
{
|
||||
lastProcessedSuffix = suffix.left;
|
||||
container.add(suffix.right.getTokens());
|
||||
}
|
||||
else
|
||||
{
|
||||
Pair<ByteBuffer, TokenTreeBuilder> result = finishSuffix();
|
||||
|
||||
lastProcessedSuffix = suffix.left;
|
||||
container = new TokenTreeBuilder(suffix.right.getTokens());
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private Pair<ByteBuffer, TokenTreeBuilder> finishSuffix()
|
||||
{
|
||||
return Pair.create(lastProcessedSuffix, container.finish());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.Buffer;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public abstract class Term<T extends Buffer>
|
||||
{
|
||||
protected final int position;
|
||||
protected final T value;
|
||||
protected TokenTreeBuilder tokens;
|
||||
|
||||
|
||||
public Term(int position, T value, TokenTreeBuilder tokens)
|
||||
{
|
||||
this.position = position;
|
||||
this.value = value;
|
||||
this.tokens = tokens;
|
||||
}
|
||||
|
||||
public int getPosition()
|
||||
{
|
||||
return position;
|
||||
}
|
||||
|
||||
public abstract ByteBuffer getTerm();
|
||||
public abstract ByteBuffer getSuffix(int start);
|
||||
|
||||
public TokenTreeBuilder getTokens()
|
||||
{
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public abstract int compareTo(AbstractType<?> comparator, Term other);
|
||||
|
||||
public abstract int length();
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.sa;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
import com.google.common.collect.AbstractIterator;
|
||||
|
||||
public abstract class TermIterator extends AbstractIterator<Pair<ByteBuffer, TokenTreeBuilder>>
|
||||
{
|
||||
public abstract ByteBuffer minTerm();
|
||||
public abstract ByteBuffer maxTerm();
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
/*
|
||||
* Copyright (C) 2007 The Guava Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.util.NoSuchElementException;
|
||||
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
|
||||
// This is fork of the Guava AbstractIterator, the only difference
|
||||
// is that state & next variables are now protected, this was required
|
||||
// for SkippableIterator.skipTo(..) to void all previous state.
|
||||
public abstract class AbstractIterator<T> implements PeekingIterator<T>
|
||||
{
|
||||
protected State state = State.NOT_READY;
|
||||
|
||||
/** Constructor for use by subclasses. */
|
||||
protected AbstractIterator() {}
|
||||
|
||||
protected enum State
|
||||
{
|
||||
/** We have computed the next element and haven't returned it yet. */
|
||||
READY,
|
||||
|
||||
/** We haven't yet computed or have already returned the element. */
|
||||
NOT_READY,
|
||||
|
||||
/** We have reached the end of the data and are finished. */
|
||||
DONE,
|
||||
|
||||
/** We've suffered an exception and are kaput. */
|
||||
FAILED,
|
||||
}
|
||||
|
||||
protected T next;
|
||||
|
||||
/**
|
||||
* Returns the next element. <b>Note:</b> the implementation must call {@link
|
||||
* #endOfData()} when there are no elements left in the iteration. Failure to
|
||||
* do so could result in an infinite loop.
|
||||
*
|
||||
* <p>The initial invocation of {@link #hasNext()} or {@link #next()} calls
|
||||
* this method, as does the first invocation of {@code hasNext} or {@code
|
||||
* next} following each successful call to {@code next}. Once the
|
||||
* implementation either invokes {@code endOfData} or throws an exception,
|
||||
* {@code computeNext} is guaranteed to never be called again.
|
||||
*
|
||||
* <p>If this method throws an exception, it will propagate outward to the
|
||||
* {@code hasNext} or {@code next} invocation that invoked this method. Any
|
||||
* further attempts to use the iterator will result in an {@link
|
||||
* IllegalStateException}.
|
||||
*
|
||||
* <p>The implementation of this method may not invoke the {@code hasNext},
|
||||
* {@code next}, or {@link #peek()} methods on this instance; if it does, an
|
||||
* {@code IllegalStateException} will result.
|
||||
*
|
||||
* @return the next element if there was one. If {@code endOfData} was called
|
||||
* during execution, the return value will be ignored.
|
||||
* @throws RuntimeException if any unrecoverable error happens. This exception
|
||||
* will propagate outward to the {@code hasNext()}, {@code next()}, or
|
||||
* {@code peek()} invocation that invoked this method. Any further
|
||||
* attempts to use the iterator will result in an
|
||||
* {@link IllegalStateException}.
|
||||
*/
|
||||
protected abstract T computeNext();
|
||||
|
||||
/**
|
||||
* Implementations of {@link #computeNext} <b>must</b> invoke this method when
|
||||
* there are no elements left in the iteration.
|
||||
*
|
||||
* @return {@code null}; a convenience so your {@code computeNext}
|
||||
* implementation can use the simple statement {@code return endOfData();}
|
||||
*/
|
||||
protected final T endOfData()
|
||||
{
|
||||
state = State.DONE;
|
||||
return null;
|
||||
}
|
||||
|
||||
public final boolean hasNext()
|
||||
{
|
||||
checkState(state != State.FAILED);
|
||||
|
||||
switch (state)
|
||||
{
|
||||
case DONE:
|
||||
return false;
|
||||
|
||||
case READY:
|
||||
return true;
|
||||
|
||||
default:
|
||||
}
|
||||
|
||||
return tryToComputeNext();
|
||||
}
|
||||
|
||||
protected boolean tryToComputeNext()
|
||||
{
|
||||
state = State.FAILED; // temporary pessimism
|
||||
next = computeNext();
|
||||
|
||||
if (state != State.DONE)
|
||||
{
|
||||
state = State.READY;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public final T next()
|
||||
{
|
||||
if (!hasNext())
|
||||
throw new NoSuchElementException();
|
||||
|
||||
state = State.NOT_READY;
|
||||
return next;
|
||||
}
|
||||
|
||||
public void remove()
|
||||
{
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the next element in the iteration without advancing the iteration,
|
||||
* according to the contract of {@link PeekingIterator#peek()}.
|
||||
*
|
||||
* <p>Implementations of {@code AbstractIterator} that wish to expose this
|
||||
* functionality should implement {@code PeekingIterator}.
|
||||
*/
|
||||
public final T peek()
|
||||
{
|
||||
if (!hasNext())
|
||||
throw new NoSuchElementException();
|
||||
|
||||
return next;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex.DataTerm;
|
||||
import org.apache.cassandra.index.sasi.disk.Token;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTree;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
import com.carrotsearch.hppc.LongOpenHashSet;
|
||||
import com.carrotsearch.hppc.LongSet;
|
||||
import com.carrotsearch.hppc.cursors.LongCursor;
|
||||
|
||||
public class CombinedTerm implements CombinedValue<DataTerm>
|
||||
{
|
||||
private final AbstractType<?> comparator;
|
||||
private final DataTerm term;
|
||||
private final TreeMap<Long, LongSet> tokens;
|
||||
|
||||
public CombinedTerm(AbstractType<?> comparator, DataTerm term)
|
||||
{
|
||||
this.comparator = comparator;
|
||||
this.term = term;
|
||||
this.tokens = new TreeMap<>();
|
||||
|
||||
RangeIterator<Long, Token> tokens = term.getTokens();
|
||||
while (tokens.hasNext())
|
||||
{
|
||||
Token current = tokens.next();
|
||||
LongSet offsets = this.tokens.get(current.get());
|
||||
if (offsets == null)
|
||||
this.tokens.put(current.get(), (offsets = new LongOpenHashSet()));
|
||||
|
||||
for (Long offset : ((TokenTree.OnDiskToken) current).getOffsets())
|
||||
offsets.add(offset);
|
||||
}
|
||||
}
|
||||
|
||||
public ByteBuffer getTerm()
|
||||
{
|
||||
return term.getTerm();
|
||||
}
|
||||
|
||||
public Map<Long, LongSet> getTokens()
|
||||
{
|
||||
return tokens;
|
||||
}
|
||||
|
||||
public TokenTreeBuilder getTokenTreeBuilder()
|
||||
{
|
||||
return new TokenTreeBuilder(tokens).finish();
|
||||
}
|
||||
|
||||
public void merge(CombinedValue<DataTerm> other)
|
||||
{
|
||||
if (!(other instanceof CombinedTerm))
|
||||
return;
|
||||
|
||||
CombinedTerm o = (CombinedTerm) other;
|
||||
|
||||
assert comparator == o.comparator;
|
||||
|
||||
for (Map.Entry<Long, LongSet> token : o.tokens.entrySet())
|
||||
{
|
||||
LongSet offsets = this.tokens.get(token.getKey());
|
||||
if (offsets == null)
|
||||
this.tokens.put(token.getKey(), (offsets = new LongOpenHashSet()));
|
||||
|
||||
for (LongCursor offset : token.getValue())
|
||||
offsets.add(offset.value);
|
||||
}
|
||||
}
|
||||
|
||||
public DataTerm get()
|
||||
{
|
||||
return term;
|
||||
}
|
||||
|
||||
public int compareTo(CombinedValue<DataTerm> o)
|
||||
{
|
||||
return term.compareTo(comparator, o.get().getTerm());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.Descriptor;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.TokenTreeBuilder;
|
||||
import org.apache.cassandra.index.sasi.sa.TermIterator;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.utils.Pair;
|
||||
|
||||
public class CombinedTermIterator extends TermIterator
|
||||
{
|
||||
final Descriptor descriptor;
|
||||
final RangeIterator<OnDiskIndex.DataTerm, CombinedTerm> union;
|
||||
final ByteBuffer min;
|
||||
final ByteBuffer max;
|
||||
|
||||
public CombinedTermIterator(OnDiskIndex... sas)
|
||||
{
|
||||
this(Descriptor.CURRENT, sas);
|
||||
}
|
||||
|
||||
public CombinedTermIterator(Descriptor d, OnDiskIndex... parts)
|
||||
{
|
||||
descriptor = d;
|
||||
union = OnDiskIndexIterator.union(parts);
|
||||
|
||||
AbstractType<?> comparator = parts[0].getComparator(); // assumes all SAs have same comparator
|
||||
ByteBuffer minimum = parts[0].minTerm();
|
||||
ByteBuffer maximum = parts[0].maxTerm();
|
||||
|
||||
for (int i = 1; i < parts.length; i++)
|
||||
{
|
||||
OnDiskIndex part = parts[i];
|
||||
if (part == null)
|
||||
continue;
|
||||
|
||||
minimum = comparator.compare(minimum, part.minTerm()) > 0 ? part.minTerm() : minimum;
|
||||
maximum = comparator.compare(maximum, part.maxTerm()) < 0 ? part.maxTerm() : maximum;
|
||||
}
|
||||
|
||||
min = minimum;
|
||||
max = maximum;
|
||||
}
|
||||
|
||||
public ByteBuffer minTerm()
|
||||
{
|
||||
return min;
|
||||
}
|
||||
|
||||
public ByteBuffer maxTerm()
|
||||
{
|
||||
return max;
|
||||
}
|
||||
|
||||
protected Pair<ByteBuffer, TokenTreeBuilder> computeNext()
|
||||
{
|
||||
if (!union.hasNext())
|
||||
{
|
||||
return endOfData();
|
||||
}
|
||||
else
|
||||
{
|
||||
CombinedTerm term = union.next();
|
||||
return Pair.create(term.getTerm(), term.getTokenTreeBuilder());
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
public interface CombinedValue<V> extends Comparable<CombinedValue<V>>
|
||||
{
|
||||
void merge(CombinedValue<V> other);
|
||||
|
||||
V get();
|
||||
}
|
||||
|
|
@ -0,0 +1,253 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.MappedByteBuffer;
|
||||
import java.nio.channels.FileChannel.MapMode;
|
||||
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
import org.apache.cassandra.io.util.ChannelProxy;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
import org.apache.cassandra.io.util.RandomAccessReader;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
public class MappedBuffer implements Closeable
|
||||
{
|
||||
private final MappedByteBuffer[] pages;
|
||||
|
||||
private long position, limit;
|
||||
private final long capacity;
|
||||
private final int pageSize, sizeBits;
|
||||
|
||||
private MappedBuffer(MappedBuffer other)
|
||||
{
|
||||
this.sizeBits = other.sizeBits;
|
||||
this.pageSize = other.pageSize;
|
||||
this.position = other.position;
|
||||
this.limit = other.limit;
|
||||
this.capacity = other.capacity;
|
||||
this.pages = other.pages;
|
||||
}
|
||||
|
||||
public MappedBuffer(RandomAccessReader file)
|
||||
{
|
||||
this(file.getChannel(), 30);
|
||||
}
|
||||
|
||||
public MappedBuffer(ChannelProxy file)
|
||||
{
|
||||
this(file, 30);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected MappedBuffer(ChannelProxy file, int numPageBits)
|
||||
{
|
||||
if (numPageBits > Integer.SIZE - 1)
|
||||
throw new IllegalArgumentException("page size can't be bigger than 1G");
|
||||
|
||||
sizeBits = numPageBits;
|
||||
pageSize = 1 << sizeBits;
|
||||
position = 0;
|
||||
limit = capacity = file.size();
|
||||
pages = new MappedByteBuffer[(int) (file.size() / pageSize) + 1];
|
||||
|
||||
try
|
||||
{
|
||||
long offset = 0;
|
||||
for (int i = 0; i < pages.length; i++)
|
||||
{
|
||||
long pageSize = Math.min(this.pageSize, (capacity - offset));
|
||||
pages[i] = file.map(MapMode.READ_ONLY, offset, pageSize);
|
||||
offset += pageSize;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
|
||||
public int comparePageTo(long offset, int length, AbstractType<?> comparator, ByteBuffer other)
|
||||
{
|
||||
return comparator.compare(getPageRegion(offset, length), other);
|
||||
}
|
||||
|
||||
public long capacity()
|
||||
{
|
||||
return capacity;
|
||||
}
|
||||
|
||||
public long position()
|
||||
{
|
||||
return position;
|
||||
}
|
||||
|
||||
public MappedBuffer position(long newPosition)
|
||||
{
|
||||
if (newPosition < 0 || newPosition > limit)
|
||||
throw new IllegalArgumentException("position: " + newPosition + ", limit: " + limit);
|
||||
|
||||
position = newPosition;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long limit()
|
||||
{
|
||||
return limit;
|
||||
}
|
||||
|
||||
public MappedBuffer limit(long newLimit)
|
||||
{
|
||||
if (newLimit < position || newLimit > capacity)
|
||||
throw new IllegalArgumentException();
|
||||
|
||||
limit = newLimit;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long remaining()
|
||||
{
|
||||
return limit - position;
|
||||
}
|
||||
|
||||
public boolean hasRemaining()
|
||||
{
|
||||
return remaining() > 0;
|
||||
}
|
||||
|
||||
public byte get()
|
||||
{
|
||||
return get(position++);
|
||||
}
|
||||
|
||||
public byte get(long pos)
|
||||
{
|
||||
return pages[getPage(pos)].get(getPageOffset(pos));
|
||||
}
|
||||
|
||||
public short getShort()
|
||||
{
|
||||
short value = getShort(position);
|
||||
position += 2;
|
||||
return value;
|
||||
}
|
||||
|
||||
public short getShort(long pos)
|
||||
{
|
||||
if (isPageAligned(pos, 2))
|
||||
return pages[getPage(pos)].getShort(getPageOffset(pos));
|
||||
|
||||
int ch1 = get(pos) & 0xff;
|
||||
int ch2 = get(pos + 1) & 0xff;
|
||||
return (short) ((ch1 << 8) + ch2);
|
||||
}
|
||||
|
||||
public int getInt()
|
||||
{
|
||||
int value = getInt(position);
|
||||
position += 4;
|
||||
return value;
|
||||
}
|
||||
|
||||
public int getInt(long pos)
|
||||
{
|
||||
if (isPageAligned(pos, 4))
|
||||
return pages[getPage(pos)].getInt(getPageOffset(pos));
|
||||
|
||||
int ch1 = get(pos) & 0xff;
|
||||
int ch2 = get(pos + 1) & 0xff;
|
||||
int ch3 = get(pos + 2) & 0xff;
|
||||
int ch4 = get(pos + 3) & 0xff;
|
||||
|
||||
return ((ch1 << 24) + (ch2 << 16) + (ch3 << 8) + ch4);
|
||||
}
|
||||
|
||||
public long getLong()
|
||||
{
|
||||
long value = getLong(position);
|
||||
position += 8;
|
||||
return value;
|
||||
}
|
||||
|
||||
|
||||
public long getLong(long pos)
|
||||
{
|
||||
// fast path if the long could be retrieved from a single page
|
||||
// that would avoid multiple expensive look-ups into page array.
|
||||
return (isPageAligned(pos, 8))
|
||||
? pages[getPage(pos)].getLong(getPageOffset(pos))
|
||||
: ((long) (getInt(pos)) << 32) + (getInt(pos + 4) & 0xFFFFFFFFL);
|
||||
}
|
||||
|
||||
public ByteBuffer getPageRegion(long position, int length)
|
||||
{
|
||||
if (!isPageAligned(position, length))
|
||||
throw new IllegalArgumentException(String.format("range: %s-%s wraps more than one page", position, length));
|
||||
|
||||
ByteBuffer slice = pages[getPage(position)].duplicate();
|
||||
|
||||
int pageOffset = getPageOffset(position);
|
||||
slice.position(pageOffset).limit(pageOffset + length);
|
||||
|
||||
return slice;
|
||||
}
|
||||
|
||||
public MappedBuffer duplicate()
|
||||
{
|
||||
return new MappedBuffer(this);
|
||||
}
|
||||
|
||||
public void close()
|
||||
{
|
||||
if (!FileUtils.isCleanerAvailable())
|
||||
return;
|
||||
|
||||
/*
|
||||
* Try forcing the unmapping of pages using undocumented unsafe sun APIs.
|
||||
* If this fails (non Sun JVM), we'll have to wait for the GC to finalize the mapping.
|
||||
* If this works and a thread tries to access any page, hell will unleash on earth.
|
||||
*/
|
||||
try
|
||||
{
|
||||
for (MappedByteBuffer segment : pages)
|
||||
FileUtils.clean(segment);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
// This is not supposed to happen
|
||||
}
|
||||
}
|
||||
|
||||
private int getPage(long position)
|
||||
{
|
||||
return (int) (position >> sizeBits);
|
||||
}
|
||||
|
||||
private int getPageOffset(long position)
|
||||
{
|
||||
return (int) (position & pageSize - 1);
|
||||
}
|
||||
|
||||
private boolean isPageAligned(long position, int length)
|
||||
{
|
||||
return pageSize - (getPageOffset(position) + length) > 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex;
|
||||
import org.apache.cassandra.index.sasi.disk.OnDiskIndex.DataTerm;
|
||||
import org.apache.cassandra.db.marshal.AbstractType;
|
||||
|
||||
public class OnDiskIndexIterator extends RangeIterator<DataTerm, CombinedTerm>
|
||||
{
|
||||
private final AbstractType<?> comparator;
|
||||
private final Iterator<DataTerm> terms;
|
||||
|
||||
public OnDiskIndexIterator(OnDiskIndex index)
|
||||
{
|
||||
super(index.min(), index.max(), Long.MAX_VALUE);
|
||||
|
||||
this.comparator = index.getComparator();
|
||||
this.terms = index.iterator();
|
||||
}
|
||||
|
||||
public static RangeIterator<DataTerm, CombinedTerm> union(OnDiskIndex... union)
|
||||
{
|
||||
RangeUnionIterator.Builder<DataTerm, CombinedTerm> builder = RangeUnionIterator.builder();
|
||||
for (OnDiskIndex e : union)
|
||||
{
|
||||
if (e != null)
|
||||
builder.add(new OnDiskIndexIterator(e));
|
||||
}
|
||||
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
protected CombinedTerm computeNext()
|
||||
{
|
||||
return terms.hasNext() ? new CombinedTerm(comparator, terms.next()) : endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(DataTerm nextToken)
|
||||
{
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{}
|
||||
}
|
||||
|
|
@ -0,0 +1,281 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
import com.google.common.collect.Iterators;
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
public class RangeIntersectionIterator
|
||||
{
|
||||
protected enum Strategy
|
||||
{
|
||||
BOUNCE, LOOKUP, ADAPTIVE
|
||||
}
|
||||
|
||||
public static <K extends Comparable<K>, D extends CombinedValue<K>> Builder<K, D> builder()
|
||||
{
|
||||
return builder(Strategy.ADAPTIVE);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected static <K extends Comparable<K>, D extends CombinedValue<K>> Builder<K, D> builder(Strategy strategy)
|
||||
{
|
||||
return new Builder<>(strategy);
|
||||
}
|
||||
|
||||
public static class Builder<K extends Comparable<K>, D extends CombinedValue<K>> extends RangeIterator.Builder<K, D>
|
||||
{
|
||||
private final Strategy strategy;
|
||||
|
||||
public Builder(Strategy strategy)
|
||||
{
|
||||
super(IteratorType.INTERSECTION);
|
||||
this.strategy = strategy;
|
||||
}
|
||||
|
||||
protected RangeIterator<K, D> buildIterator()
|
||||
{
|
||||
// if the range is disjoint we can simply return empty
|
||||
// iterator of any type, because it's not going to produce any results.
|
||||
if (statistics.isDisjoint())
|
||||
return new BounceIntersectionIterator<>(statistics, new PriorityQueue<RangeIterator<K, D>>(1));
|
||||
|
||||
switch (strategy)
|
||||
{
|
||||
case LOOKUP:
|
||||
return new LookupIntersectionIterator<>(statistics, ranges);
|
||||
|
||||
case BOUNCE:
|
||||
return new BounceIntersectionIterator<>(statistics, ranges);
|
||||
|
||||
case ADAPTIVE:
|
||||
return statistics.sizeRatio() <= 0.01d
|
||||
? new LookupIntersectionIterator<>(statistics, ranges)
|
||||
: new BounceIntersectionIterator<>(statistics, ranges);
|
||||
|
||||
default:
|
||||
throw new IllegalStateException("Unknown strategy: " + strategy);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static abstract class AbstractIntersectionIterator<K extends Comparable<K>, D extends CombinedValue<K>> extends RangeIterator<K, D>
|
||||
{
|
||||
protected final PriorityQueue<RangeIterator<K, D>> ranges;
|
||||
|
||||
private AbstractIntersectionIterator(Builder.Statistics<K, D> statistics, PriorityQueue<RangeIterator<K, D>> ranges)
|
||||
{
|
||||
super(statistics);
|
||||
this.ranges = ranges;
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
for (RangeIterator<K, D> range : ranges)
|
||||
FileUtils.closeQuietly(range);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterator which performs intersection of multiple ranges by using bouncing (merge-join) technique to identify
|
||||
* common elements in the given ranges. Aforementioned "bounce" works as follows: range queue is poll'ed for the
|
||||
* range with the smallest current token (main loop), that token is used to {@link RangeIterator#skipTo(Comparable)}
|
||||
* other ranges, if token produced by {@link RangeIterator#skipTo(Comparable)} is equal to current "candidate" token,
|
||||
* both get merged together and the same operation is repeated for next range from the queue, if returned token
|
||||
* is not equal than candidate, candidate's range gets put back into the queue and the main loop gets repeated until
|
||||
* next intersection token is found or at least one iterator runs out of tokens.
|
||||
*
|
||||
* This technique is every efficient to jump over gaps in the ranges.
|
||||
*
|
||||
* @param <K> The type used to sort ranges.
|
||||
* @param <D> The container type which is going to be returned by {@link Iterator#next()}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
protected static class BounceIntersectionIterator<K extends Comparable<K>, D extends CombinedValue<K>> extends AbstractIntersectionIterator<K, D>
|
||||
{
|
||||
private BounceIntersectionIterator(Builder.Statistics<K, D> statistics, PriorityQueue<RangeIterator<K, D>> ranges)
|
||||
{
|
||||
super(statistics, ranges);
|
||||
}
|
||||
|
||||
protected D computeNext()
|
||||
{
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
RangeIterator<K, D> head = ranges.poll();
|
||||
|
||||
// jump right to the beginning of the intersection or return next element
|
||||
if (head.getCurrent().compareTo(getMinimum()) < 0)
|
||||
head.skipTo(getMinimum());
|
||||
|
||||
D candidate = head.hasNext() ? head.next() : null;
|
||||
if (candidate == null || candidate.get().compareTo(getMaximum()) > 0)
|
||||
{
|
||||
ranges.add(head);
|
||||
return endOfData();
|
||||
}
|
||||
|
||||
List<RangeIterator<K, D>> processed = new ArrayList<>();
|
||||
|
||||
boolean intersectsAll = true, exhausted = false;
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
RangeIterator<K, D> range = ranges.poll();
|
||||
|
||||
// found a range which doesn't overlap with one (or possibly more) other range(s)
|
||||
if (!isOverlapping(head, range))
|
||||
{
|
||||
exhausted = true;
|
||||
intersectsAll = false;
|
||||
break;
|
||||
}
|
||||
|
||||
D point = range.skipTo(candidate.get());
|
||||
|
||||
if (point == null) // other range is exhausted
|
||||
{
|
||||
exhausted = true;
|
||||
intersectsAll = false;
|
||||
break;
|
||||
}
|
||||
|
||||
processed.add(range);
|
||||
|
||||
if (candidate.get().equals(point.get()))
|
||||
{
|
||||
candidate.merge(point);
|
||||
// advance skipped range to the next element if any
|
||||
Iterators.getNext(range, null);
|
||||
}
|
||||
else
|
||||
{
|
||||
intersectsAll = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ranges.add(head);
|
||||
|
||||
for (RangeIterator<K, D> range : processed)
|
||||
ranges.add(range);
|
||||
|
||||
if (exhausted)
|
||||
return endOfData();
|
||||
|
||||
if (intersectsAll)
|
||||
return candidate;
|
||||
}
|
||||
|
||||
return endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(K nextToken)
|
||||
{
|
||||
List<RangeIterator<K, D>> skipped = new ArrayList<>();
|
||||
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
RangeIterator<K, D> range = ranges.poll();
|
||||
range.skipTo(nextToken);
|
||||
skipped.add(range);
|
||||
}
|
||||
|
||||
for (RangeIterator<K, D> range : skipped)
|
||||
ranges.add(range);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterator which performs a linear scan over a primary range (the smallest of the ranges)
|
||||
* and O(log(n)) lookup into secondary ranges using values from the primary iterator.
|
||||
* This technique is efficient when one of the intersection ranges is smaller than others
|
||||
* e.g. ratio 0.01d (default), in such situation scan + lookup is more efficient comparing
|
||||
* to "bounce" merge because "bounce" distance is never going to be big.
|
||||
*
|
||||
* @param <K> The type used to sort ranges.
|
||||
* @param <D> The container type which is going to be returned by {@link Iterator#next()}.
|
||||
*/
|
||||
@VisibleForTesting
|
||||
protected static class LookupIntersectionIterator<K extends Comparable<K>, D extends CombinedValue<K>> extends AbstractIntersectionIterator<K, D>
|
||||
{
|
||||
private final RangeIterator<K, D> smallestIterator;
|
||||
|
||||
private LookupIntersectionIterator(Builder.Statistics<K, D> statistics, PriorityQueue<RangeIterator<K, D>> ranges)
|
||||
{
|
||||
super(statistics, ranges);
|
||||
|
||||
smallestIterator = statistics.minRange;
|
||||
|
||||
if (smallestIterator.getCurrent().compareTo(getMinimum()) < 0)
|
||||
smallestIterator.skipTo(getMinimum());
|
||||
}
|
||||
|
||||
protected D computeNext()
|
||||
{
|
||||
while (smallestIterator.hasNext())
|
||||
{
|
||||
D candidate = smallestIterator.next();
|
||||
K token = candidate.get();
|
||||
|
||||
boolean intersectsAll = true;
|
||||
for (RangeIterator<K, D> range : ranges)
|
||||
{
|
||||
// avoid checking against self, much cheaper than changing queue comparator
|
||||
// to compare based on the size and re-populating such queue.
|
||||
if (range.equals(smallestIterator))
|
||||
continue;
|
||||
|
||||
// found a range which doesn't overlap with one (or possibly more) other range(s)
|
||||
if (!isOverlapping(smallestIterator, range))
|
||||
return endOfData();
|
||||
|
||||
D point = range.skipTo(token);
|
||||
|
||||
if (point == null) // one of the iterators is exhausted
|
||||
return endOfData();
|
||||
|
||||
if (!point.get().equals(token))
|
||||
{
|
||||
intersectsAll = false;
|
||||
break;
|
||||
}
|
||||
|
||||
candidate.merge(point);
|
||||
}
|
||||
|
||||
if (intersectsAll)
|
||||
return candidate;
|
||||
}
|
||||
|
||||
return endOfData();
|
||||
}
|
||||
|
||||
protected void performSkipTo(K nextToken)
|
||||
{
|
||||
smallestIterator.skipTo(nextToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,279 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.io.Closeable;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
|
||||
public abstract class RangeIterator<K extends Comparable<K>, T extends CombinedValue<K>> extends AbstractIterator<T> implements Closeable
|
||||
{
|
||||
private final K min, max;
|
||||
private final long count;
|
||||
private K current;
|
||||
|
||||
protected RangeIterator(Builder.Statistics<K, T> statistics)
|
||||
{
|
||||
this(statistics.min, statistics.max, statistics.tokenCount);
|
||||
}
|
||||
|
||||
public RangeIterator(RangeIterator<K, T> range)
|
||||
{
|
||||
this(range == null ? null : range.min, range == null ? null : range.max, range == null ? -1 : range.count);
|
||||
}
|
||||
|
||||
public RangeIterator(K min, K max, long count)
|
||||
{
|
||||
this.min = min;
|
||||
this.current = min;
|
||||
this.max = max;
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public final K getMinimum()
|
||||
{
|
||||
return min;
|
||||
}
|
||||
|
||||
public final K getCurrent()
|
||||
{
|
||||
return current;
|
||||
}
|
||||
|
||||
public final K getMaximum()
|
||||
{
|
||||
return max;
|
||||
}
|
||||
|
||||
public final long getCount()
|
||||
{
|
||||
return count;
|
||||
}
|
||||
|
||||
/**
|
||||
* When called, this iterators current position should
|
||||
* be skipped forwards until finding either:
|
||||
* 1) an element equal to or bigger than next
|
||||
* 2) the end of the iterator
|
||||
*
|
||||
* @param nextToken value to skip the iterator forward until matching
|
||||
*
|
||||
* @return The next current token after the skip was performed
|
||||
*/
|
||||
public final T skipTo(K nextToken)
|
||||
{
|
||||
if (min == null || max == null)
|
||||
return endOfData();
|
||||
|
||||
if (current.compareTo(nextToken) >= 0)
|
||||
return next == null ? recomputeNext() : next;
|
||||
|
||||
if (max.compareTo(nextToken) < 0)
|
||||
return endOfData();
|
||||
|
||||
performSkipTo(nextToken);
|
||||
return recomputeNext();
|
||||
}
|
||||
|
||||
protected abstract void performSkipTo(K nextToken);
|
||||
|
||||
protected T recomputeNext()
|
||||
{
|
||||
return tryToComputeNext() ? peek() : endOfData();
|
||||
}
|
||||
|
||||
protected boolean tryToComputeNext()
|
||||
{
|
||||
boolean hasNext = super.tryToComputeNext();
|
||||
current = hasNext ? next.get() : getMaximum();
|
||||
return hasNext;
|
||||
}
|
||||
|
||||
public static abstract class Builder<K extends Comparable<K>, D extends CombinedValue<K>>
|
||||
{
|
||||
public enum IteratorType
|
||||
{
|
||||
UNION, INTERSECTION
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected final Statistics<K, D> statistics;
|
||||
|
||||
@VisibleForTesting
|
||||
protected final PriorityQueue<RangeIterator<K, D>> ranges;
|
||||
|
||||
public Builder(IteratorType type)
|
||||
{
|
||||
statistics = new Statistics<>(type);
|
||||
ranges = new PriorityQueue<>(16, (Comparator<RangeIterator<K, D>>) (a, b) -> a.getCurrent().compareTo(b.getCurrent()));
|
||||
}
|
||||
|
||||
public K getMinimum()
|
||||
{
|
||||
return statistics.min;
|
||||
}
|
||||
|
||||
public K getMaximum()
|
||||
{
|
||||
return statistics.max;
|
||||
}
|
||||
|
||||
public long getTokenCount()
|
||||
{
|
||||
return statistics.tokenCount;
|
||||
}
|
||||
|
||||
public int rangeCount()
|
||||
{
|
||||
return ranges.size();
|
||||
}
|
||||
|
||||
public Builder<K, D> add(RangeIterator<K, D> range)
|
||||
{
|
||||
if (range == null || range.getMinimum() == null || range.getMaximum() == null)
|
||||
return this;
|
||||
|
||||
ranges.add(range);
|
||||
statistics.update(range);
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder<K, D> add(List<RangeIterator<K, D>> ranges)
|
||||
{
|
||||
if (ranges == null || ranges.isEmpty())
|
||||
return this;
|
||||
|
||||
ranges.forEach(this::add);
|
||||
return this;
|
||||
}
|
||||
|
||||
public final RangeIterator<K, D> build()
|
||||
{
|
||||
switch (rangeCount())
|
||||
{
|
||||
case 0:
|
||||
return null;
|
||||
|
||||
case 1:
|
||||
return ranges.poll();
|
||||
|
||||
default:
|
||||
return buildIterator();
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract RangeIterator<K, D> buildIterator();
|
||||
|
||||
public static class Statistics<K extends Comparable<K>, D extends CombinedValue<K>>
|
||||
{
|
||||
protected final IteratorType iteratorType;
|
||||
|
||||
protected K min, max;
|
||||
protected long tokenCount;
|
||||
|
||||
// iterator with the least number of items
|
||||
protected RangeIterator<K, D> minRange;
|
||||
// iterator with the most number of items
|
||||
protected RangeIterator<K, D> maxRange;
|
||||
|
||||
// tracks if all of the added ranges overlap, which is useful in case of intersection,
|
||||
// as it gives direct answer as to such iterator is going to produce any results.
|
||||
protected boolean isOverlapping = true;
|
||||
|
||||
public Statistics(IteratorType iteratorType)
|
||||
{
|
||||
this.iteratorType = iteratorType;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update statistics information with the given range.
|
||||
*
|
||||
* Updates min/max of the combined range, token count and
|
||||
* tracks range with the least/most number of tokens.
|
||||
*
|
||||
* @param range The range to update statistics with.
|
||||
*/
|
||||
public void update(RangeIterator<K, D> range)
|
||||
{
|
||||
switch (iteratorType)
|
||||
{
|
||||
case UNION:
|
||||
min = min == null || min.compareTo(range.getMinimum()) > 0 ? range.getMinimum() : min;
|
||||
max = max == null || max.compareTo(range.getMaximum()) < 0 ? range.getMaximum() : max;
|
||||
break;
|
||||
|
||||
case INTERSECTION:
|
||||
// minimum of the intersection is the biggest minimum of individual iterators
|
||||
min = min == null || min.compareTo(range.getMinimum()) < 0 ? range.getMinimum() : min;
|
||||
// maximum of the intersection is the smallest maximum of individual iterators
|
||||
max = max == null || max.compareTo(range.getMaximum()) > 0 ? range.getMaximum() : max;
|
||||
break;
|
||||
|
||||
default:
|
||||
throw new IllegalStateException("Unknown iterator type: " + iteratorType);
|
||||
}
|
||||
|
||||
// check if new range is disjoint with already added ranges, which means that this intersection
|
||||
// is not going to produce any results, so we can cleanup range storage and never added anything to it.
|
||||
isOverlapping &= isOverlapping(min, max, range);
|
||||
|
||||
minRange = minRange == null ? range : min(minRange, range);
|
||||
maxRange = maxRange == null ? range : max(maxRange, range);
|
||||
|
||||
tokenCount += range.getCount();
|
||||
|
||||
}
|
||||
|
||||
private RangeIterator<K, D> min(RangeIterator<K, D> a, RangeIterator<K, D> b)
|
||||
{
|
||||
return a.getCount() > b.getCount() ? b : a;
|
||||
}
|
||||
|
||||
private RangeIterator<K, D> max(RangeIterator<K, D> a, RangeIterator<K, D> b)
|
||||
{
|
||||
return a.getCount() > b.getCount() ? a : b;
|
||||
}
|
||||
|
||||
public boolean isDisjoint()
|
||||
{
|
||||
return !isOverlapping;
|
||||
}
|
||||
|
||||
public double sizeRatio()
|
||||
{
|
||||
return minRange.getCount() * 1d / maxRange.getCount();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected static <K extends Comparable<K>, D extends CombinedValue<K>> boolean isOverlapping(RangeIterator<K, D> a, RangeIterator<K, D> b)
|
||||
{
|
||||
return isOverlapping(a.getCurrent(), a.getMaximum(), b);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected static <K extends Comparable<K>, D extends CombinedValue<K>> boolean isOverlapping(K min, K max, RangeIterator<K, D> b)
|
||||
{
|
||||
return min.compareTo(b.getMaximum()) <= 0 && b.getCurrent().compareTo(max) <= 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.cassandra.io.util.FileUtils;
|
||||
|
||||
/**
|
||||
* Range Union Iterator is used to return sorted stream of elements from multiple RangeIterator instances.
|
||||
*
|
||||
* PriorityQueue is used as a sorting mechanism for the ranges, where each computeNext() operation would poll
|
||||
* from the queue (and push when done), which returns range that contains the smallest element, because
|
||||
* sorting is done on the moving window of range iteration {@link RangeIterator#getCurrent()}. Once retrieved
|
||||
* the smallest element (return candidate) is attempted to be merged with other ranges, because there could
|
||||
* be equal elements in adjacent ranges, such ranges are poll'ed only if their {@link RangeIterator#getCurrent()}
|
||||
* equals to the return candidate.
|
||||
*
|
||||
* @param <K> The type used to sort ranges.
|
||||
* @param <D> The container type which is going to be returned by {@link Iterator#next()}.
|
||||
*/
|
||||
public class RangeUnionIterator<K extends Comparable<K>, D extends CombinedValue<K>> extends RangeIterator<K, D>
|
||||
{
|
||||
private final PriorityQueue<RangeIterator<K, D>> ranges;
|
||||
|
||||
private RangeUnionIterator(Builder.Statistics<K, D> statistics, PriorityQueue<RangeIterator<K, D>> ranges)
|
||||
{
|
||||
super(statistics);
|
||||
this.ranges = ranges;
|
||||
}
|
||||
|
||||
public D computeNext()
|
||||
{
|
||||
RangeIterator<K, D> head = null;
|
||||
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
head = ranges.poll();
|
||||
if (head.hasNext())
|
||||
break;
|
||||
|
||||
FileUtils.closeQuietly(head);
|
||||
}
|
||||
|
||||
if (head == null || !head.hasNext())
|
||||
return endOfData();
|
||||
|
||||
D candidate = head.next();
|
||||
|
||||
List<RangeIterator<K, D>> processedRanges = new ArrayList<>();
|
||||
|
||||
if (head.hasNext())
|
||||
processedRanges.add(head);
|
||||
else
|
||||
FileUtils.closeQuietly(head);
|
||||
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
// peek here instead of poll is an optimization
|
||||
// so we can re-insert less ranges back if candidate
|
||||
// is less than head of the current range.
|
||||
RangeIterator<K, D> range = ranges.peek();
|
||||
|
||||
int cmp = candidate.get().compareTo(range.getCurrent());
|
||||
|
||||
assert cmp <= 0;
|
||||
|
||||
if (cmp < 0)
|
||||
{
|
||||
break; // candidate is smaller than next token, return immediately
|
||||
}
|
||||
else if (cmp == 0)
|
||||
{
|
||||
candidate.merge(range.next()); // consume and merge
|
||||
|
||||
range = ranges.poll();
|
||||
// re-prioritize changed range
|
||||
|
||||
if (range.hasNext())
|
||||
processedRanges.add(range);
|
||||
else
|
||||
FileUtils.closeQuietly(range);
|
||||
}
|
||||
}
|
||||
|
||||
ranges.addAll(processedRanges);
|
||||
return candidate;
|
||||
}
|
||||
|
||||
protected void performSkipTo(K nextToken)
|
||||
{
|
||||
List<RangeIterator<K, D>> changedRanges = new ArrayList<>();
|
||||
|
||||
while (!ranges.isEmpty())
|
||||
{
|
||||
if (ranges.peek().getCurrent().compareTo(nextToken) >= 0)
|
||||
break;
|
||||
|
||||
RangeIterator<K, D> head = ranges.poll();
|
||||
|
||||
if (head.getMaximum().compareTo(nextToken) >= 0)
|
||||
{
|
||||
head.skipTo(nextToken);
|
||||
changedRanges.add(head);
|
||||
continue;
|
||||
}
|
||||
|
||||
FileUtils.closeQuietly(head);
|
||||
}
|
||||
|
||||
ranges.addAll(changedRanges.stream().collect(Collectors.toList()));
|
||||
}
|
||||
|
||||
public void close() throws IOException
|
||||
{
|
||||
ranges.forEach(FileUtils::closeQuietly);
|
||||
}
|
||||
|
||||
public static <K extends Comparable<K>, D extends CombinedValue<K>> Builder<K, D> builder()
|
||||
{
|
||||
return new Builder<>();
|
||||
}
|
||||
|
||||
public static <K extends Comparable<K>, D extends CombinedValue<K>> RangeIterator<K, D> build(List<RangeIterator<K, D>> tokens)
|
||||
{
|
||||
return new Builder<K, D>().add(tokens).build();
|
||||
}
|
||||
|
||||
public static class Builder<K extends Comparable<K>, D extends CombinedValue<K>> extends RangeIterator.Builder<K, D>
|
||||
{
|
||||
public Builder()
|
||||
{
|
||||
super(IteratorType.UNION);
|
||||
}
|
||||
|
||||
protected RangeIterator<K, D> buildIterator()
|
||||
{
|
||||
return new RangeUnionIterator<>(statistics, ranges);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.index.sasi.utils;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.apache.cassandra.db.marshal.*;
|
||||
import org.apache.cassandra.serializers.MarshalException;
|
||||
|
||||
public class TypeUtil
|
||||
{
|
||||
public static boolean isValid(ByteBuffer term, AbstractType<?> validator)
|
||||
{
|
||||
try
|
||||
{
|
||||
validator.validate(term);
|
||||
return true;
|
||||
}
|
||||
catch (MarshalException e)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static ByteBuffer tryUpcast(ByteBuffer term, AbstractType<?> validator)
|
||||
{
|
||||
if (term.remaining() == 0)
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
if (validator instanceof Int32Type && term.remaining() == 2)
|
||||
{
|
||||
return Int32Type.instance.decompose((int) term.getShort(term.position()));
|
||||
}
|
||||
else if (validator instanceof LongType)
|
||||
{
|
||||
long upcastToken;
|
||||
|
||||
switch (term.remaining())
|
||||
{
|
||||
case 2:
|
||||
upcastToken = (long) term.getShort(term.position());
|
||||
break;
|
||||
|
||||
case 4:
|
||||
upcastToken = (long) Int32Type.instance.compose(term);
|
||||
break;
|
||||
|
||||
default:
|
||||
upcastToken = Long.valueOf(UTF8Type.instance.getString(term));
|
||||
}
|
||||
|
||||
return LongType.instance.decompose(upcastToken);
|
||||
}
|
||||
else if (validator instanceof DoubleType && term.remaining() == 4)
|
||||
{
|
||||
return DoubleType.instance.decompose((double) FloatType.instance.compose(term));
|
||||
}
|
||||
|
||||
// maybe it was a string after all
|
||||
return validator.fromString(UTF8Type.instance.getString(term));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,230 @@
|
|||
/*
|
||||
* Copyright 2005-2010 Roger Kapsi, Sam Berlin
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils.trie;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.AbstractMap;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* This class is taken from https://github.com/rkapsi/patricia-trie (v0.6), and slightly modified
|
||||
* to correspond to Cassandra code style, as the only Patricia Trie implementation,
|
||||
* which supports pluggable key comparators (e.g. commons-collections PatriciaTrie (which is based
|
||||
* on rkapsi/patricia-trie project) only supports String keys)
|
||||
* but unfortunately is not deployed to the maven central as a downloadable artifact.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This class provides some basic {@link Trie} functionality and
|
||||
* utility methods for actual {@link Trie} implementations.
|
||||
*/
|
||||
abstract class AbstractTrie<K, V> extends AbstractMap<K, V> implements Serializable, Trie<K, V>
|
||||
{
|
||||
private static final long serialVersionUID = -6358111100045408883L;
|
||||
|
||||
/**
|
||||
* The {@link KeyAnalyzer} that's being used to build the
|
||||
* PATRICIA {@link Trie}
|
||||
*/
|
||||
protected final KeyAnalyzer<? super K> keyAnalyzer;
|
||||
|
||||
/**
|
||||
* Constructs a new {@link Trie} using the given {@link KeyAnalyzer}
|
||||
*/
|
||||
public AbstractTrie(KeyAnalyzer<? super K> keyAnalyzer)
|
||||
{
|
||||
this.keyAnalyzer = Tries.notNull(keyAnalyzer, "keyAnalyzer");
|
||||
}
|
||||
|
||||
@Override
|
||||
public K selectKey(K key)
|
||||
{
|
||||
Map.Entry<K, V> entry = select(key);
|
||||
return entry != null ? entry.getKey() : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public V selectValue(K key)
|
||||
{
|
||||
Map.Entry<K, V> entry = select(key);
|
||||
return entry != null ? entry.getValue() : null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
buffer.append("Trie[").append(size()).append("]={\n");
|
||||
for (Map.Entry<K, V> entry : entrySet()) {
|
||||
buffer.append(" ").append(entry).append("\n");
|
||||
}
|
||||
buffer.append("}\n");
|
||||
return buffer.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the length of the given key in bits
|
||||
*
|
||||
* @see KeyAnalyzer#lengthInBits(Object)
|
||||
*/
|
||||
final int lengthInBits(K key)
|
||||
{
|
||||
return key == null ? 0 : keyAnalyzer.lengthInBits(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns whether or not the given bit on the
|
||||
* key is set or false if the key is null.
|
||||
*
|
||||
* @see KeyAnalyzer#isBitSet(Object, int)
|
||||
*/
|
||||
final boolean isBitSet(K key, int bitIndex)
|
||||
{
|
||||
return key != null && keyAnalyzer.isBitSet(key, bitIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method for calling {@link KeyAnalyzer#bitIndex(Object, Object)}
|
||||
*/
|
||||
final int bitIndex(K key, K otherKey)
|
||||
{
|
||||
if (key != null && otherKey != null)
|
||||
{
|
||||
return keyAnalyzer.bitIndex(key, otherKey);
|
||||
}
|
||||
else if (key != null)
|
||||
{
|
||||
return bitIndex(key);
|
||||
}
|
||||
else if (otherKey != null)
|
||||
{
|
||||
return bitIndex(otherKey);
|
||||
}
|
||||
|
||||
return KeyAnalyzer.NULL_BIT_KEY;
|
||||
}
|
||||
|
||||
private int bitIndex(K key)
|
||||
{
|
||||
int lengthInBits = lengthInBits(key);
|
||||
for (int i = 0; i < lengthInBits; i++)
|
||||
{
|
||||
if (isBitSet(key, i))
|
||||
return i;
|
||||
}
|
||||
|
||||
return KeyAnalyzer.NULL_BIT_KEY;
|
||||
}
|
||||
|
||||
/**
|
||||
* An utility method for calling {@link KeyAnalyzer#compare(Object, Object)}
|
||||
*/
|
||||
final boolean compareKeys(K key, K other)
|
||||
{
|
||||
if (key == null)
|
||||
{
|
||||
return (other == null);
|
||||
}
|
||||
else if (other == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return keyAnalyzer.compare(key, other) == 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* A basic implementation of {@link Entry}
|
||||
*/
|
||||
abstract static class BasicEntry<K, V> implements Map.Entry<K, V>, Serializable
|
||||
{
|
||||
private static final long serialVersionUID = -944364551314110330L;
|
||||
|
||||
protected K key;
|
||||
|
||||
protected V value;
|
||||
|
||||
private transient int hashCode = 0;
|
||||
|
||||
public BasicEntry(K key, V value)
|
||||
{
|
||||
this.key = key;
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the current key and value with the provided
|
||||
* key & value
|
||||
*/
|
||||
public V setKeyValue(K key, V value)
|
||||
{
|
||||
this.key = key;
|
||||
this.hashCode = 0;
|
||||
return setValue(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public K getKey()
|
||||
{
|
||||
return key;
|
||||
}
|
||||
|
||||
@Override
|
||||
public V getValue()
|
||||
{
|
||||
return value;
|
||||
}
|
||||
|
||||
@Override
|
||||
public V setValue(V value)
|
||||
{
|
||||
V previous = this.value;
|
||||
this.value = value;
|
||||
return previous;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode()
|
||||
{
|
||||
if (hashCode == 0)
|
||||
hashCode = (key != null ? key.hashCode() : 0);
|
||||
return hashCode;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o)
|
||||
{
|
||||
if (o == this)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else if (!(o instanceof Map.Entry<?, ?>))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Map.Entry<?, ?> other = (Map.Entry<?, ?>)o;
|
||||
return Tries.areEqual(key, other.getKey()) && Tries.areEqual(value, other.getValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString()
|
||||
{
|
||||
return key + "=" + value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,83 @@
|
|||
/*
|
||||
* Copyright 2005-2010 Roger Kapsi, Sam Berlin
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils.trie;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
/**
|
||||
* This class is taken from https://github.com/rkapsi/patricia-trie (v0.6), and slightly modified
|
||||
* to correspond to Cassandra code style, as the only Patricia Trie implementation,
|
||||
* which supports pluggable key comparators (e.g. commons-collections PatriciaTrie (which is based
|
||||
* on rkapsi/patricia-trie project) only supports String keys)
|
||||
* but unfortunately is not deployed to the maven central as a downloadable artifact.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A {@link Cursor} can be used to traverse a {@link Trie}, visit each node
|
||||
* step by step and make {@link Decision}s on each step how to continue with
|
||||
* traversing the {@link Trie}.
|
||||
*/
|
||||
public interface Cursor<K, V>
|
||||
{
|
||||
|
||||
/**
|
||||
* The {@link Decision} tells the {@link Cursor} what to do on each step
|
||||
* while traversing the {@link Trie}.
|
||||
*
|
||||
* NOTE: Not all operations that work with a {@link Cursor} support all
|
||||
* {@link Decision} types
|
||||
*/
|
||||
enum Decision
|
||||
{
|
||||
|
||||
/**
|
||||
* Exit the traverse operation
|
||||
*/
|
||||
EXIT,
|
||||
|
||||
/**
|
||||
* Continue with the traverse operation
|
||||
*/
|
||||
CONTINUE,
|
||||
|
||||
/**
|
||||
* Remove the previously returned element
|
||||
* from the {@link Trie} and continue
|
||||
*/
|
||||
REMOVE,
|
||||
|
||||
/**
|
||||
* Remove the previously returned element
|
||||
* from the {@link Trie} and exit from the
|
||||
* traverse operation
|
||||
*/
|
||||
REMOVE_AND_EXIT
|
||||
}
|
||||
|
||||
/**
|
||||
* Called for each {@link Entry} in the {@link Trie}. Return
|
||||
* {@link Decision#EXIT} to finish the {@link Trie} operation,
|
||||
* {@link Decision#CONTINUE} to go to the next {@link Entry},
|
||||
* {@link Decision#REMOVE} to remove the {@link Entry} and
|
||||
* continue iterating or {@link Decision#REMOVE_AND_EXIT} to
|
||||
* remove the {@link Entry} and stop iterating.
|
||||
*
|
||||
* Note: Not all operations support {@link Decision#REMOVE}.
|
||||
*/
|
||||
Decision select(Map.Entry<? extends K, ? extends V> entry);
|
||||
}
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
/*
|
||||
* Copyright 2010 Roger Kapsi
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils.trie;
|
||||
|
||||
import java.util.Comparator;
|
||||
|
||||
/**
|
||||
* This class is taken from https://github.com/rkapsi/patricia-trie (v0.6), and slightly modified
|
||||
* to correspond to Cassandra code style, as the only Patricia Trie implementation,
|
||||
* which supports pluggable key comparators (e.g. commons-collections PatriciaTrie (which is based
|
||||
* on rkapsi/patricia-trie project) only supports String keys)
|
||||
* but unfortunately is not deployed to the maven central as a downloadable artifact.
|
||||
*/
|
||||
|
||||
/**
|
||||
* The {@link KeyAnalyzer} provides bit-level access to keys
|
||||
* for the {@link PatriciaTrie}.
|
||||
*/
|
||||
public interface KeyAnalyzer<K> extends Comparator<K>
|
||||
{
|
||||
/**
|
||||
* Returned by {@link #bitIndex(Object, Object)} if a key's
|
||||
* bits were all zero (0).
|
||||
*/
|
||||
int NULL_BIT_KEY = -1;
|
||||
|
||||
/**
|
||||
* Returned by {@link #bitIndex(Object, Object)} if a the
|
||||
* bits of two keys were all equal.
|
||||
*/
|
||||
int EQUAL_BIT_KEY = -2;
|
||||
|
||||
/**
|
||||
* Returned by {@link #bitIndex(Object, Object)} if a keys
|
||||
* indices are out of bounds.
|
||||
*/
|
||||
int OUT_OF_BOUNDS_BIT_KEY = -3;
|
||||
|
||||
/**
|
||||
* Returns the key's length in bits.
|
||||
*/
|
||||
int lengthInBits(K key);
|
||||
|
||||
/**
|
||||
* Returns {@code true} if a key's bit it set at the given index.
|
||||
*/
|
||||
boolean isBitSet(K key, int bitIndex);
|
||||
|
||||
/**
|
||||
* Returns the index of the first bit that is different in the two keys.
|
||||
*/
|
||||
int bitIndex(K key, K otherKey);
|
||||
|
||||
/**
|
||||
* Returns {@code true} if the second argument is a
|
||||
* prefix of the first argument.
|
||||
*/
|
||||
boolean isPrefix(K key, K prefix);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,152 @@
|
|||
/*
|
||||
* Copyright 2005-2010 Roger Kapsi, Sam Berlin
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils.trie;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.SortedMap;
|
||||
|
||||
import org.apache.cassandra.index.sasi.utils.trie.Cursor.Decision;
|
||||
|
||||
/**
|
||||
* This class is taken from https://github.com/rkapsi/patricia-trie (v0.6), and slightly modified
|
||||
* to correspond to Cassandra code style, as the only Patricia Trie implementation,
|
||||
* which supports pluggable key comparators (e.g. commons-collections PatriciaTrie (which is based
|
||||
* on rkapsi/patricia-trie project) only supports String keys)
|
||||
* but unfortunately is not deployed to the maven central as a downloadable artifact.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Defines the interface for a prefix tree, an ordered tree data structure. For
|
||||
* more information, see <a href="http://en.wikipedia.org/wiki/Trie">Tries</a>.
|
||||
*
|
||||
* @author Roger Kapsi
|
||||
* @author Sam Berlin
|
||||
*/
|
||||
public interface Trie<K, V> extends SortedMap<K, V>
|
||||
{
|
||||
/**
|
||||
* Returns the {@link Map.Entry} whose key is closest in a bitwise XOR
|
||||
* metric to the given key. This is NOT lexicographic closeness.
|
||||
* For example, given the keys:
|
||||
*
|
||||
* <ol>
|
||||
* <li>D = 1000100
|
||||
* <li>H = 1001000
|
||||
* <li>L = 1001100
|
||||
* </ol>
|
||||
*
|
||||
* If the {@link Trie} contained 'H' and 'L', a lookup of 'D' would
|
||||
* return 'L', because the XOR distance between D & L is smaller
|
||||
* than the XOR distance between D & H.
|
||||
*
|
||||
* @return The {@link Map.Entry} whose key is closest in a bitwise XOR metric
|
||||
* to the provided key.
|
||||
*/
|
||||
Map.Entry<K, V> select(K key);
|
||||
|
||||
/**
|
||||
* Returns the key that is closest in a bitwise XOR metric to the
|
||||
* provided key. This is NOT lexicographic closeness!
|
||||
*
|
||||
* For example, given the keys:
|
||||
*
|
||||
* <ol>
|
||||
* <li>D = 1000100
|
||||
* <li>H = 1001000
|
||||
* <li>L = 1001100
|
||||
* </ol>
|
||||
*
|
||||
* If the {@link Trie} contained 'H' and 'L', a lookup of 'D' would
|
||||
* return 'L', because the XOR distance between D & L is smaller
|
||||
* than the XOR distance between D & H.
|
||||
*
|
||||
* @return The key that is closest in a bitwise XOR metric to the provided key.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
K selectKey(K key);
|
||||
|
||||
/**
|
||||
* Returns the value whose key is closest in a bitwise XOR metric to
|
||||
* the provided key. This is NOT lexicographic closeness!
|
||||
*
|
||||
* For example, given the keys:
|
||||
*
|
||||
* <ol>
|
||||
* <li>D = 1000100
|
||||
* <li>H = 1001000
|
||||
* <li>L = 1001100
|
||||
* </ol>
|
||||
*
|
||||
* If the {@link Trie} contained 'H' and 'L', a lookup of 'D' would
|
||||
* return 'L', because the XOR distance between D & L is smaller
|
||||
* than the XOR distance between D & H.
|
||||
*
|
||||
* @return The value whose key is closest in a bitwise XOR metric
|
||||
* to the provided key.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
V selectValue(K key);
|
||||
|
||||
/**
|
||||
* Iterates through the {@link Trie}, starting with the entry whose bitwise
|
||||
* value is closest in an XOR metric to the given key. After the closest
|
||||
* entry is found, the {@link Trie} will call select on that entry and continue
|
||||
* calling select for each entry (traversing in order of XOR closeness,
|
||||
* NOT lexicographically) until the cursor returns {@link Decision#EXIT}.
|
||||
*
|
||||
* <p>The cursor can return {@link Decision#CONTINUE} to continue traversing.
|
||||
*
|
||||
* <p>{@link Decision#REMOVE_AND_EXIT} is used to remove the current element
|
||||
* and stop traversing.
|
||||
*
|
||||
* <p>Note: The {@link Decision#REMOVE} operation is not supported.
|
||||
*
|
||||
* @return The entry the cursor returned {@link Decision#EXIT} on, or null
|
||||
* if it continued till the end.
|
||||
*/
|
||||
Map.Entry<K,V> select(K key, Cursor<? super K, ? super V> cursor);
|
||||
|
||||
/**
|
||||
* Traverses the {@link Trie} in lexicographical order.
|
||||
* {@link Cursor#select(java.util.Map.Entry)} will be called on each entry.
|
||||
*
|
||||
* <p>The traversal will stop when the cursor returns {@link Decision#EXIT},
|
||||
* {@link Decision#CONTINUE} is used to continue traversing and
|
||||
* {@link Decision#REMOVE} is used to remove the element that was selected
|
||||
* and continue traversing.
|
||||
*
|
||||
* <p>{@link Decision#REMOVE_AND_EXIT} is used to remove the current element
|
||||
* and stop traversing.
|
||||
*
|
||||
* @return The entry the cursor returned {@link Decision#EXIT} on, or null
|
||||
* if it continued till the end.
|
||||
*/
|
||||
Map.Entry<K,V> traverse(Cursor<? super K, ? super V> cursor);
|
||||
|
||||
/**
|
||||
* Returns a view of this {@link Trie} of all elements that are prefixed
|
||||
* by the given key.
|
||||
*
|
||||
* <p>In a {@link Trie} with fixed size keys, this is essentially a
|
||||
* {@link #get(Object)} operation.
|
||||
*
|
||||
* <p>For example, if the {@link Trie} contains 'Anna', 'Anael',
|
||||
* 'Analu', 'Andreas', 'Andrea', 'Andres', and 'Anatole', then
|
||||
* a lookup of 'And' would return 'Andreas', 'Andrea', and 'Andres'.
|
||||
*/
|
||||
SortedMap<K, V> prefixMap(K prefix);
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
/*
|
||||
* Copyright 2005-2010 Roger Kapsi
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This class is taken from https://github.com/rkapsi/patricia-trie (v0.6), and slightly modified
|
||||
* to correspond to Cassandra code style, as the only Patricia Trie implementation,
|
||||
* which supports pluggable key comparators (e.g. commons-collections PatriciaTrie (which is based
|
||||
* on rkapsi/patricia-trie project) only supports String keys)
|
||||
* but unfortunately is not deployed to the maven central as a downloadable artifact.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.index.sasi.utils.trie;
|
||||
|
||||
/**
|
||||
* A collection of {@link Trie} utilities
|
||||
*/
|
||||
public class Tries
|
||||
{
|
||||
/**
|
||||
* Returns true if bitIndex is a {@link KeyAnalyzer#OUT_OF_BOUNDS_BIT_KEY}
|
||||
*/
|
||||
static boolean isOutOfBoundsIndex(int bitIndex)
|
||||
{
|
||||
return bitIndex == KeyAnalyzer.OUT_OF_BOUNDS_BIT_KEY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if bitIndex is a {@link KeyAnalyzer#EQUAL_BIT_KEY}
|
||||
*/
|
||||
static boolean isEqualBitKey(int bitIndex)
|
||||
{
|
||||
return bitIndex == KeyAnalyzer.EQUAL_BIT_KEY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if bitIndex is a {@link KeyAnalyzer#NULL_BIT_KEY}
|
||||
*/
|
||||
static boolean isNullBitKey(int bitIndex)
|
||||
{
|
||||
return bitIndex == KeyAnalyzer.NULL_BIT_KEY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given bitIndex is valid. Indices
|
||||
* are considered valid if they're between 0 and
|
||||
* {@link Integer#MAX_VALUE}
|
||||
*/
|
||||
static boolean isValidBitIndex(int bitIndex)
|
||||
{
|
||||
return 0 <= bitIndex;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if both values are either null or equal
|
||||
*/
|
||||
static boolean areEqual(Object a, Object b)
|
||||
{
|
||||
return (a == null ? b == null : a.equals(b));
|
||||
}
|
||||
|
||||
/**
|
||||
* Throws a {@link NullPointerException} with the given message if
|
||||
* the argument is null.
|
||||
*/
|
||||
static <T> T notNull(T o, String message)
|
||||
{
|
||||
if (o == null)
|
||||
throw new NullPointerException(message);
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
/**
|
||||
* A utility method to cast keys. It actually doesn't
|
||||
* cast anything. It's just fooling the compiler!
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
static <K> K cast(Object key)
|
||||
{
|
||||
return (K)key;
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,7 @@ package org.apache.cassandra.io.sstable;
|
|||
|
||||
import java.io.File;
|
||||
import java.util.EnumSet;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.google.common.base.Objects;
|
||||
|
||||
|
|
@ -57,6 +58,8 @@ public class Component
|
|||
SUMMARY("Summary.db"),
|
||||
// table of contents, stores the list of all components for the sstable
|
||||
TOC("TOC.txt"),
|
||||
// built-in secondary index (may be multiple per sstable)
|
||||
SECONDARY_INDEX("SI_.*.db"),
|
||||
// custom component, used by e.g. custom compaction strategy
|
||||
CUSTOM(new String[] { null });
|
||||
|
||||
|
|
@ -74,9 +77,12 @@ public class Component
|
|||
static Type fromRepresentation(String repr)
|
||||
{
|
||||
for (Type type : TYPES)
|
||||
for (String representation : type.repr)
|
||||
if (repr.equals(representation))
|
||||
return type;
|
||||
{
|
||||
if (type.repr == null || type.repr.length == 0 || type.repr[0] == null)
|
||||
continue;
|
||||
if (Pattern.matches(type.repr[0], repr))
|
||||
return type;
|
||||
}
|
||||
return CUSTOM;
|
||||
}
|
||||
}
|
||||
|
|
@ -169,6 +175,7 @@ public class Component
|
|||
case CRC: component = Component.CRC; break;
|
||||
case SUMMARY: component = Component.SUMMARY; break;
|
||||
case TOC: component = Component.TOC; break;
|
||||
case SECONDARY_INDEX: component = new Component(Type.SECONDARY_INDEX, path.right); break;
|
||||
case CUSTOM: component = new Component(Type.CUSTOM, path.right); break;
|
||||
default:
|
||||
throw new IllegalStateException();
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ public class KeyIterator extends AbstractIterator<DecoratedKey> implements Close
|
|||
private final In in;
|
||||
private final IPartitioner partitioner;
|
||||
|
||||
private long keyPosition;
|
||||
|
||||
public KeyIterator(Descriptor desc, CFMetaData metadata)
|
||||
{
|
||||
|
|
@ -99,6 +100,7 @@ public class KeyIterator extends AbstractIterator<DecoratedKey> implements Close
|
|||
if (in.isEOF())
|
||||
return endOfData();
|
||||
|
||||
keyPosition = in.getFilePointer();
|
||||
DecoratedKey key = partitioner.decorateKey(ByteBufferUtil.readWithShortLength(in.get()));
|
||||
RowIndexEntry.Serializer.skip(in.get(), desc.version); // skip remainder of the entry
|
||||
return key;
|
||||
|
|
@ -123,4 +125,9 @@ public class KeyIterator extends AbstractIterator<DecoratedKey> implements Close
|
|||
{
|
||||
return in.length();
|
||||
}
|
||||
|
||||
public long getKeyPosition()
|
||||
{
|
||||
return keyPosition;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
package org.apache.cassandra.io.sstable.format;
|
||||
|
||||
import org.apache.cassandra.db.DecoratedKey;
|
||||
import org.apache.cassandra.db.rows.ColumnData;
|
||||
import org.apache.cassandra.db.rows.Unfiltered;
|
||||
|
||||
/**
|
||||
* Observer for events in the lifecycle of writing out an sstable.
|
||||
|
|
@ -32,7 +32,7 @@ public interface SSTableFlushObserver
|
|||
|
||||
/**
|
||||
* Called when a new partition in being written to the sstable,
|
||||
* but before any cells are processed (see {@link #nextCell(ColumnData)}).
|
||||
* but before any cells are processed (see {@link #nextUnfilteredCluster(Unfiltered)}).
|
||||
*
|
||||
* @param key The key being appended to SSTable.
|
||||
* @param indexPosition The position of the key in the SSTable PRIMARY_INDEX file.
|
||||
|
|
@ -40,13 +40,13 @@ public interface SSTableFlushObserver
|
|||
void startPartition(DecoratedKey key, long indexPosition);
|
||||
|
||||
/**
|
||||
* Called after the cell is written to the sstable.
|
||||
* Called after the unfiltered cluster is written to the sstable.
|
||||
* Will be preceded by a call to {@code startPartition(DecoratedKey, long)},
|
||||
* and the cell should be assumed to belong to that row.
|
||||
* and the cluster should be assumed to belong to that partition.
|
||||
*
|
||||
* @param cell The cell being added to the row.
|
||||
* @param unfilteredCluster The unfiltered cluster being added to SSTable.
|
||||
*/
|
||||
void nextCell(ColumnData cell);
|
||||
void nextUnfilteredCluster(Unfiltered unfilteredCluster);
|
||||
|
||||
/**
|
||||
* Called when all data is written to the file and it's ready to be finished up.
|
||||
|
|
|
|||
|
|
@ -1795,6 +1795,26 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS
|
|||
return sstableMetadata.repairedAt != ActiveRepairService.UNREPAIRED_SSTABLE;
|
||||
}
|
||||
|
||||
public DecoratedKey keyAt(long indexPosition) throws IOException
|
||||
{
|
||||
DecoratedKey key;
|
||||
try (FileDataInput in = ifile.createReader(indexPosition))
|
||||
{
|
||||
if (in.isEOF())
|
||||
return null;
|
||||
|
||||
key = decorateKey(ByteBufferUtil.readWithShortLength(in));
|
||||
|
||||
// hint read path about key location if caching is enabled
|
||||
// this saves index summary lookup and index file iteration which whould be pretty costly
|
||||
// especially in presence of promoted column indexes
|
||||
if (isKeyCacheSetup())
|
||||
cacheKey(key, rowIndexEntrySerializer.deserialize(in));
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: Move someplace reusable
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -277,7 +277,14 @@ public abstract class SSTableWriter extends SSTable implements Transactional
|
|||
|
||||
public final Throwable commit(Throwable accumulate)
|
||||
{
|
||||
return txnProxy.commit(accumulate);
|
||||
try
|
||||
{
|
||||
return txnProxy.commit(accumulate);
|
||||
}
|
||||
finally
|
||||
{
|
||||
observers.forEach(SSTableFlushObserver::complete);
|
||||
}
|
||||
}
|
||||
|
||||
public final Throwable abort(Throwable accumulate)
|
||||
|
|
|
|||
|
|
@ -62,4 +62,9 @@ public class DataOutputBufferFixed extends DataOutputBuffer
|
|||
{
|
||||
throw new BufferOverflowException();
|
||||
}
|
||||
|
||||
public void clear()
|
||||
{
|
||||
buffer.clear();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -169,6 +169,13 @@ public class SequentialWriter extends BufferedDataOutputStreamPlus implements Tr
|
|||
return this;
|
||||
}
|
||||
|
||||
public void skipBytes(int numBytes) throws IOException
|
||||
{
|
||||
flush();
|
||||
fchannel.position(fchannel.position() + numBytes);
|
||||
bufferOffset = fchannel.position();
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronize file contents with disk.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -669,4 +669,45 @@ public class ByteBufferUtil
|
|||
return buf;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check is the given buffer contains a given sub-buffer.
|
||||
*
|
||||
* @param buffer The buffer to search for sequence of bytes in.
|
||||
* @param subBuffer The buffer to match.
|
||||
*
|
||||
* @return true if buffer contains sub-buffer, false otherwise.
|
||||
*/
|
||||
public static boolean contains(ByteBuffer buffer, ByteBuffer subBuffer)
|
||||
{
|
||||
int len = subBuffer.remaining();
|
||||
if (buffer.remaining() - len < 0)
|
||||
return false;
|
||||
|
||||
// adapted form the JDK's String.indexOf()
|
||||
byte first = subBuffer.get(subBuffer.position());
|
||||
int max = buffer.position() + (buffer.remaining() - len);
|
||||
|
||||
for (int i = buffer.position(); i <= max; i++)
|
||||
{
|
||||
/* Look for first character. */
|
||||
if (buffer.get(i) != first)
|
||||
{
|
||||
while (++i <= max && buffer.get(i) != first)
|
||||
{}
|
||||
}
|
||||
|
||||
/* (maybe) Found first character, now look at the rest of v2 */
|
||||
if (i <= max)
|
||||
{
|
||||
int j = i + 1;
|
||||
int end = j + len - 1;
|
||||
for (int k = 1 + subBuffer.position(); j < end && buffer.get(j) == subBuffer.get(k); j++, k++)
|
||||
{}
|
||||
|
||||
if (j == end)
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -844,4 +844,9 @@ public class FBUtilities
|
|||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static long align(long val, int boundary)
|
||||
{
|
||||
return (val + boundary) & ~(boundary - 1);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public abstract class MemoryUtil
|
|||
private static final long UNSAFE_COPY_THRESHOLD = 1024 * 1024L; // copied from java.nio.Bits
|
||||
|
||||
private static final Unsafe unsafe;
|
||||
private static final Class<?> DIRECT_BYTE_BUFFER_CLASS;
|
||||
private static final Class<?> DIRECT_BYTE_BUFFER_CLASS, RO_DIRECT_BYTE_BUFFER_CLASS;
|
||||
private static final long DIRECT_BYTE_BUFFER_ADDRESS_OFFSET;
|
||||
private static final long DIRECT_BYTE_BUFFER_CAPACITY_OFFSET;
|
||||
private static final long DIRECT_BYTE_BUFFER_LIMIT_OFFSET;
|
||||
|
|
@ -65,6 +65,7 @@ public abstract class MemoryUtil
|
|||
DIRECT_BYTE_BUFFER_POSITION_OFFSET = unsafe.objectFieldOffset(Buffer.class.getDeclaredField("position"));
|
||||
DIRECT_BYTE_BUFFER_ATTACHMENT_OFFSET = unsafe.objectFieldOffset(clazz.getDeclaredField("att"));
|
||||
DIRECT_BYTE_BUFFER_CLASS = clazz;
|
||||
RO_DIRECT_BYTE_BUFFER_CLASS = ByteBuffer.allocateDirect(0).asReadOnlyBuffer().getClass();
|
||||
|
||||
clazz = ByteBuffer.allocate(0).getClass();
|
||||
BYTE_BUFFER_OFFSET_OFFSET = unsafe.objectFieldOffset(ByteBuffer.class.getDeclaredField("offset"));
|
||||
|
|
@ -204,7 +205,7 @@ public abstract class MemoryUtil
|
|||
|
||||
public static ByteBuffer duplicateDirectByteBuffer(ByteBuffer source, ByteBuffer hollowBuffer)
|
||||
{
|
||||
assert source.getClass() == DIRECT_BYTE_BUFFER_CLASS;
|
||||
assert source.getClass() == DIRECT_BYTE_BUFFER_CLASS || source.getClass() == RO_DIRECT_BYTE_BUFFER_CLASS;
|
||||
unsafe.putLong(hollowBuffer, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET, unsafe.getLong(source, DIRECT_BYTE_BUFFER_ADDRESS_OFFSET));
|
||||
unsafe.putInt(hollowBuffer, DIRECT_BYTE_BUFFER_POSITION_OFFSET, unsafe.getInt(source, DIRECT_BYTE_BUFFER_POSITION_OFFSET));
|
||||
unsafe.putInt(hollowBuffer, DIRECT_BYTE_BUFFER_LIMIT_OFFSET, unsafe.getInt(source, DIRECT_BYTE_BUFFER_LIMIT_OFFSET));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,163 @@
|
|||
# Stop Words List from http://members.unine.ch/jacques.savoy/clef/index.html
|
||||
ب
|
||||
ا
|
||||
أ
|
||||
،
|
||||
عشر
|
||||
عدد
|
||||
عدة
|
||||
عشرة
|
||||
عدم
|
||||
عام
|
||||
عاما
|
||||
عن
|
||||
عند
|
||||
عندما
|
||||
على
|
||||
عليه
|
||||
عليها
|
||||
زيارة
|
||||
سنة
|
||||
سنوات
|
||||
تم
|
||||
ضد
|
||||
بعد
|
||||
بعض
|
||||
اعادة
|
||||
اعلنت
|
||||
بسبب
|
||||
حتى
|
||||
اذا
|
||||
احد
|
||||
اثر
|
||||
برس
|
||||
باسم
|
||||
غدا
|
||||
شخصا
|
||||
صباح
|
||||
اطار
|
||||
اربعة
|
||||
اخرى
|
||||
بان
|
||||
اجل
|
||||
غير
|
||||
بشكل
|
||||
حاليا
|
||||
بن
|
||||
به
|
||||
ثم
|
||||
اف
|
||||
ان
|
||||
او
|
||||
اي
|
||||
بها
|
||||
صفر
|
||||
حيث
|
||||
اكد
|
||||
الا
|
||||
اما
|
||||
امس
|
||||
السابق
|
||||
التى
|
||||
التي
|
||||
اكثر
|
||||
ايار
|
||||
ايضا
|
||||
ثلاثة
|
||||
الذاتي
|
||||
الاخيرة
|
||||
الثاني
|
||||
الثانية
|
||||
الذى
|
||||
الذي
|
||||
الان
|
||||
امام
|
||||
ايام
|
||||
خلال
|
||||
حوالى
|
||||
الذين
|
||||
الاول
|
||||
الاولى
|
||||
بين
|
||||
ذلك
|
||||
دون
|
||||
حول
|
||||
حين
|
||||
الف
|
||||
الى
|
||||
انه
|
||||
اول
|
||||
ضمن
|
||||
انها
|
||||
جميع
|
||||
الماضي
|
||||
الوقت
|
||||
المقبل
|
||||
اليوم
|
||||
ـ
|
||||
ف
|
||||
و
|
||||
و6
|
||||
قد
|
||||
لا
|
||||
ما
|
||||
مع
|
||||
مساء
|
||||
هذا
|
||||
واحد
|
||||
واضاف
|
||||
واضافت
|
||||
فان
|
||||
قبل
|
||||
قال
|
||||
كان
|
||||
لدى
|
||||
نحو
|
||||
هذه
|
||||
وان
|
||||
واكد
|
||||
كانت
|
||||
واوضح
|
||||
مايو
|
||||
فى
|
||||
في
|
||||
كل
|
||||
لم
|
||||
لن
|
||||
له
|
||||
من
|
||||
هو
|
||||
هي
|
||||
قوة
|
||||
كما
|
||||
لها
|
||||
منذ
|
||||
وقد
|
||||
ولا
|
||||
نفسه
|
||||
لقاء
|
||||
مقابل
|
||||
هناك
|
||||
وقال
|
||||
وكان
|
||||
نهاية
|
||||
وقالت
|
||||
وكانت
|
||||
للامم
|
||||
فيه
|
||||
كلم
|
||||
لكن
|
||||
وفي
|
||||
وقف
|
||||
ولم
|
||||
ومن
|
||||
وهو
|
||||
وهي
|
||||
يوم
|
||||
فيها
|
||||
منها
|
||||
مليار
|
||||
لوكالة
|
||||
يكون
|
||||
يمكن
|
||||
مليون
|
||||
|
|
@ -0,0 +1,260 @@
|
|||
# Stop Words List from http://members.unine.ch/jacques.savoy/clef/index.html
|
||||
а
|
||||
автентичен
|
||||
аз
|
||||
ако
|
||||
ала
|
||||
бе
|
||||
без
|
||||
беше
|
||||
би
|
||||
бивш
|
||||
бивша
|
||||
бившо
|
||||
бил
|
||||
била
|
||||
били
|
||||
било
|
||||
благодаря
|
||||
близо
|
||||
бъдат
|
||||
бъде
|
||||
бяха
|
||||
в
|
||||
вас
|
||||
ваш
|
||||
ваша
|
||||
вероятно
|
||||
вече
|
||||
взема
|
||||
ви
|
||||
вие
|
||||
винаги
|
||||
внимава
|
||||
време
|
||||
все
|
||||
всеки
|
||||
всички
|
||||
всичко
|
||||
всяка
|
||||
във
|
||||
въпреки
|
||||
върху
|
||||
г
|
||||
ги
|
||||
главен
|
||||
главна
|
||||
главно
|
||||
глас
|
||||
го
|
||||
година
|
||||
години
|
||||
годишен
|
||||
д
|
||||
да
|
||||
дали
|
||||
два
|
||||
двама
|
||||
двамата
|
||||
две
|
||||
двете
|
||||
ден
|
||||
днес
|
||||
дни
|
||||
до
|
||||
добра
|
||||
добре
|
||||
добро
|
||||
добър
|
||||
докато
|
||||
докога
|
||||
дори
|
||||
досега
|
||||
доста
|
||||
друг
|
||||
друга
|
||||
други
|
||||
е
|
||||
евтин
|
||||
едва
|
||||
един
|
||||
една
|
||||
еднаква
|
||||
еднакви
|
||||
еднакъв
|
||||
едно
|
||||
екип
|
||||
ето
|
||||
живот
|
||||
за
|
||||
забавям
|
||||
зад
|
||||
заедно
|
||||
заради
|
||||
засега
|
||||
заспал
|
||||
затова
|
||||
защо
|
||||
защото
|
||||
и
|
||||
из
|
||||
или
|
||||
им
|
||||
има
|
||||
имат
|
||||
иска
|
||||
й
|
||||
каза
|
||||
как
|
||||
каква
|
||||
какво
|
||||
както
|
||||
какъв
|
||||
като
|
||||
кога
|
||||
когато
|
||||
което
|
||||
които
|
||||
кой
|
||||
който
|
||||
колко
|
||||
която
|
||||
къде
|
||||
където
|
||||
към
|
||||
лесен
|
||||
лесно
|
||||
ли
|
||||
лош
|
||||
м
|
||||
май
|
||||
малко
|
||||
ме
|
||||
между
|
||||
мек
|
||||
мен
|
||||
месец
|
||||
ми
|
||||
много
|
||||
мнозина
|
||||
мога
|
||||
могат
|
||||
може
|
||||
мокър
|
||||
моля
|
||||
момента
|
||||
му
|
||||
н
|
||||
на
|
||||
над
|
||||
назад
|
||||
най
|
||||
направи
|
||||
напред
|
||||
например
|
||||
нас
|
||||
не
|
||||
него
|
||||
нещо
|
||||
нея
|
||||
ни
|
||||
ние
|
||||
никой
|
||||
нито
|
||||
нищо
|
||||
но
|
||||
нов
|
||||
нова
|
||||
нови
|
||||
новина
|
||||
някои
|
||||
някой
|
||||
няколко
|
||||
няма
|
||||
обаче
|
||||
около
|
||||
освен
|
||||
особено
|
||||
от
|
||||
отгоре
|
||||
отново
|
||||
още
|
||||
пак
|
||||
по
|
||||
повече
|
||||
повечето
|
||||
под
|
||||
поне
|
||||
поради
|
||||
после
|
||||
почти
|
||||
прави
|
||||
пред
|
||||
преди
|
||||
през
|
||||
при
|
||||
пък
|
||||
първата
|
||||
първи
|
||||
първо
|
||||
пъти
|
||||
равен
|
||||
равна
|
||||
с
|
||||
са
|
||||
сам
|
||||
само
|
||||
се
|
||||
сега
|
||||
си
|
||||
син
|
||||
скоро
|
||||
след
|
||||
следващ
|
||||
сме
|
||||
смях
|
||||
според
|
||||
сред
|
||||
срещу
|
||||
сте
|
||||
съм
|
||||
със
|
||||
също
|
||||
т
|
||||
тази
|
||||
така
|
||||
такива
|
||||
такъв
|
||||
там
|
||||
твой
|
||||
те
|
||||
тези
|
||||
ти
|
||||
т.н.
|
||||
то
|
||||
това
|
||||
тогава
|
||||
този
|
||||
той
|
||||
толкова
|
||||
точно
|
||||
три
|
||||
трябва
|
||||
тук
|
||||
тъй
|
||||
тя
|
||||
тях
|
||||
у
|
||||
утре
|
||||
харесва
|
||||
хиляди
|
||||
ч
|
||||
часа
|
||||
че
|
||||
често
|
||||
чрез
|
||||
ще
|
||||
щом
|
||||
юмрук
|
||||
я
|
||||
як
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue