SASI tokenizer for simple delimiter based entries

Patch by Mick Semb Wever; reviewed by Michael Kjellman for CASSANDRA-14247

CHANGES.txt

@@ -1,4 +1,5 @@
 3.11.3
+ * SASI tokenizer for simple delimiter based entries (CASSANDRA-14247)
  * Fix Loss of digits when doing CAST from varint/bigint to decimal (CASSANDRA-14170)
  * RateBasedBackPressure unnecessarily invokes a lock on the Guava RateLimiter (CASSANDRA-14163)
  * Fix wildcard GROUP BY queries (CASSANDRA-14209)

src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzer.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sasi.analyzer;
+
+import java.nio.CharBuffer;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.google.common.annotations.Beta;
+import com.google.common.base.Preconditions;
+
+import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.AsciiType;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.utils.AbstractIterator;
+
+@Beta
+public class DelimiterAnalyzer extends AbstractAnalyzer
+{
+
+    private static final Map<AbstractType<?>,Charset> VALID_ANALYZABLE_TYPES = new HashMap<AbstractType<?>,Charset>()
+    {{
+            put(UTF8Type.instance, StandardCharsets.UTF_8);
+            put(AsciiType.instance, StandardCharsets.US_ASCII);
+    }};
+
+    private char delimiter;
+    private Charset charset;
+    private Iterator<ByteBuffer> iter;
+
+    public DelimiterAnalyzer()
+    {
+    }
+
+    public ByteBuffer next()
+    {
+        return iter.next();
+    }
+
+    public void init(Map<String, String> options, AbstractType validator)
+    {
+        DelimiterTokenizingOptions tokenizingOptions = DelimiterTokenizingOptions.buildFromMap(options);
+        delimiter = tokenizingOptions.getDelimiter();
+
+        if (!VALID_ANALYZABLE_TYPES.containsKey(validator))
+            throw new IllegalArgumentException(String.format("Only text types supported, got %s", validator));
+
+        charset = VALID_ANALYZABLE_TYPES.get(validator);
+    }
+
+    public boolean hasNext()
+    {
+        return iter.hasNext();
+    }
+
+    public void reset(ByteBuffer input)
+    {
+        Preconditions.checkNotNull(input);
+        final CharBuffer cb = charset.decode(input);
+
+        this.iter = new AbstractIterator<ByteBuffer>() {
+            protected ByteBuffer computeNext() {
+
+                if (!cb.hasRemaining())
+                    return endOfData();
+
+                CharBuffer readahead = cb.duplicate();
+                // loop until we see the next delimiter character, or reach end of data
+                while (readahead.hasRemaining() && readahead.get() != delimiter);
+
+                char[] chars = new char[readahead.position() - cb.position() - (readahead.hasRemaining() ? 1 : 0)];
+                cb.get(chars);
+                Preconditions.checkState(!cb.hasRemaining() || cb.get() == delimiter);
+                return charset.encode(CharBuffer.wrap(chars));
+            }
+        };
+    }
+
+
+    public boolean isTokenizing()
+    {
+        return true;
+    }
+}

src/java/org/apache/cassandra/index/sasi/analyzer/DelimiterTokenizingOptions.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sasi.analyzer;
+
+import java.util.Map;
+
+/** Simple tokenizer based on a specified delimiter (rather than whitespace).
+ */
+public class DelimiterTokenizingOptions
+{
+    public static final String DELIMITER = "delimiter";
+
+    private final char delimiter;
+
+    private DelimiterTokenizingOptions(char delimiter)
+    {
+        this.delimiter = delimiter;
+    }
+
+    char getDelimiter()
+    {
+        return delimiter;
+    }
+
+    private static class OptionsBuilder
+    {
+        private char delimiter = ',';
+
+        public DelimiterTokenizingOptions build()
+        {
+            return new DelimiterTokenizingOptions(delimiter);
+        }
+    }
+
+    static DelimiterTokenizingOptions buildFromMap(Map<String, String> optionsMap)
+    {
+        OptionsBuilder optionsBuilder = new OptionsBuilder();
+
+        for (Map.Entry<String, String> entry : optionsMap.entrySet())
+        {
+            switch (entry.getKey())
+            {
+                case DELIMITER:
+                {
+                    String value = entry.getValue();
+                    if (1 != value.length())
+                        throw new IllegalArgumentException(String.format("Only single character delimiters supported, was %s", value));
+
+                    optionsBuilder.delimiter = entry.getValue().charAt(0);
+                    break;
+                }
+            }
+        }
+        return optionsBuilder.build();
+    }
+}

test/unit/org/apache/cassandra/index/sasi/analyzer/DelimiterAnalyzerTest.java

@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.index.sasi.analyzer;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.UTF8Type;
+import org.apache.cassandra.utils.ByteBufferUtil;
+import org.apache.commons.io.IOUtils;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class DelimiterAnalyzerTest
+{
+
+    @Test
+    public void caseSensitiveAnalizer() throws Exception
+    {
+        DelimiterAnalyzer analyzer = new DelimiterAnalyzer();
+
+        analyzer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        String testString = "Nip it in the bud";
+        ByteBuffer toAnalyze = ByteBuffer.wrap(testString.getBytes());
+        analyzer.reset(toAnalyze);
+        StringBuilder output = new StringBuilder();
+        while (analyzer.hasNext())
+            output.append(ByteBufferUtil.string(analyzer.next()) + (analyzer.hasNext() ? ' ' : ""));
+
+        Assert.assertTrue(testString.equals(output.toString()));
+        Assert.assertFalse(testString.toLowerCase().equals(output.toString()));
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void ensureIncompatibleInputSkipped() throws Exception
+    {
+        new DelimiterAnalyzer().init(new HashMap(), Int32Type.instance);
+    }
+
+    @Test
+    public void testTokenizationLoremIpsum() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/lorem_ipsum.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        tokenizer.reset(bb);
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(69, tokens.size());
+
+    }
+
+    @Test
+    public void testTokenizationJaJp1() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ja_jp_1.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, "。");
+                }},
+            UTF8Type.instance);
+
+        tokenizer.reset(bb);
+        List<ByteBuffer> tokens = new ArrayList<>();
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(4, tokens.size());
+    }
+
+    @Test
+    public void testTokenizationJaJp2() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ja_jp_2.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, "。");
+                }},
+            UTF8Type.instance);
+
+        tokenizer.reset(bb);
+        List<ByteBuffer> tokens = new ArrayList<>();
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(2, tokens.size());
+    }
+
+    @Test
+    public void testTokenizationRuRu1() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/ru_ru_1.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        tokenizer.reset(bb);
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(447, tokens.size());
+    }
+
+    @Test
+    public void testTokenizationZnTw1() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/zn_tw_1.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        tokenizer.reset(bb);
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(403, tokens.size());
+    }
+
+    @Test
+    public void testTokenizationAdventuresOfHuckFinn() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/adventures_of_huckleberry_finn_mark_twain.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        tokenizer.reset(bb);
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(104594, tokens.size());
+    }
+
+    @Test
+    public void testWorldCities() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/world_cities_a.csv")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, ",");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        tokenizer.reset(bb);
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(122265, tokens.size());
+    }
+
+    @Test
+    public void tokenizeDomainNamesAndUrls() throws Exception
+    {
+        ByteBuffer bb = ByteBuffer.wrap(IOUtils.toByteArray(
+                DelimiterAnalyzerTest.class.getClassLoader().getResourceAsStream("tokenization/top_visited_domains.txt")));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        tokenizer.reset(bb);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        while (tokenizer.hasNext())
+            tokens.add(tokenizer.next());
+
+        assertEquals(12, tokens.size());
+    }
+
+    @Test
+    public void testReuseAndResetTokenizerInstance() throws Exception
+    {
+        List<ByteBuffer> bbToTokenize = new ArrayList<>();
+        bbToTokenize.add(ByteBuffer.wrap("Nip it in the bud".getBytes()));
+        bbToTokenize.add(ByteBuffer.wrap("I couldn’t care less".getBytes()));
+        bbToTokenize.add(ByteBuffer.wrap("One and the same".getBytes()));
+        bbToTokenize.add(ByteBuffer.wrap("The squeaky wheel gets the grease.".getBytes()));
+        bbToTokenize.add(ByteBuffer.wrap("The pen is mightier than the sword.".getBytes()));
+
+        DelimiterAnalyzer tokenizer = new DelimiterAnalyzer();
+
+        tokenizer.init(
+            new HashMap()
+                {{
+                    put(DelimiterTokenizingOptions.DELIMITER, " ");
+                }},
+            UTF8Type.instance);
+
+        List<ByteBuffer> tokens = new ArrayList<>();
+        for (ByteBuffer bb : bbToTokenize)
+        {
+            tokenizer.reset(bb);
+            while (tokenizer.hasNext())
+                tokens.add(tokenizer.next());
+        }
+        assertEquals(26, tokens.size());
+    }
+
+}