From 2aa2b59d37899fbe114bdd75e09c10699729c32a Mon Sep 17 00:00:00 2001 From: Lorina Poland Date: Mon, 8 Jan 2024 12:46:02 -0800 Subject: [PATCH] Minor adds and fixes in the documentation patch by Lorina Poland; reviewed by Michael Semb Wever for CASSANDRA-19249, CASSANDRA-18990, CASSANDRA-15719 Co-authored-by: Annette Dennis Co-authored-by: Mohsin Medmood Co-authored-by: Josh Wong --- NEWS.txt | 2 +- doc/cql3/CQL.textile | 4 +- .../CQL/sai/create-vector-index-cycling.cql | 4 +- .../examples/CQL/sai/create-vector-index.cql | 4 +- .../CQL/sai/cyclist_semi_pro_sai_indices.cql | 16 +-- .../CQL/sai/index-sai-case-sensitive.cql | 4 +- ...ai-composite-partition-key-index-match.cql | 8 +- .../CQL/sai/index-sai-from-list-match.cql | 4 +- .../CQL/sai/index-sai-from-set-match.cql | 4 +- ...ndex-sai-multiple-index-match-with-and.cql | 12 +- .../CQL/sai/index-sai-similarity-function.cql | 4 +- .../CQL/sai/index-sai-single-index-match.cql | 4 +- .../examples/CQL/sai/index-sai-udt-match.cql | 4 +- .../cassandra/examples/CQL/sai/sai-error.cql | 4 +- .../sai/cyclist_semi_pro_describe.result | 8 +- .../pages/developing/cql/cql_singlefile.adoc | 4 +- .../developing/cql/create-custom-index.adoc | 14 +- .../cql/indexing/sai/_sai-create.adoc | 2 +- .../developing/cql/indexing/sai/sai-faq.adoc | 18 +-- .../pages/managing/operating/repair.adoc | 11 +- .../cql-commands/create-custom-index.adoc | 18 +-- .../reference/cql-commands/create-index.adoc | 120 +++++++++++++++--- .../pages/reference/vector-data-type.adoc | 2 + .../pages/vector-search/concepts.adoc | 1 + .../pages/vector-search/data-modeling.adoc | 35 ++++- .../cassandra/partials/table-properties.adoc | 2 +- pylib/cqlshlib/cqlshmain.py | 2 +- 27 files changed, 214 insertions(+), 101 deletions(-) diff --git a/NEWS.txt b/NEWS.txt index ab2c3e3fb1..aab5ef9b72 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -598,7 +598,7 @@ Upgrading internode_socket_receive_buffer_size_in_bytes. To support upgrades pre-4.0, we add backward compatibility and currently both old and new names should work. Cassandra 4.0.0 and Cassandra 4.0.1 work ONLY with the new names (They weren't updated in cassandra.yaml though). - - DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESRIBE|DESC will not work in cqlsh 6.0.0 + - DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESCRIBE|DESC will not work in cqlsh 6.0.0 being connected to earlier major Cassandra versions where DESCRIBE does not exist server side. - cqlsh shell startup script now prefers 'python3' before 'python' when identifying a runtime. - As part of the Internode Messaging improvement work in CASSANDRA-15066, matching response verbs for every request diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index 88918f87d9..959533f771 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -1626,7 +1626,7 @@ __Syntax:__ bc(syntax).. ::= GRANT ( ALL ( PERMISSIONS )? | ( PERMISSION )? (, PERMISSION)* ) ON TO - ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE | UNMASK | SELECT_MASKED + ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | EXECUTE | UNMASK | SELECT_MASKED ::= ALL KEYSPACES | KEYSPACE @@ -1683,7 +1683,7 @@ __Syntax:__ bc(syntax).. ::= REVOKE ( ALL ( PERMISSIONS )? | ( PERMISSION )? (, PERMISSION)* ) ON FROM - ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | EXECUTE | UNMASK | SELECT_MASKED + ::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | EXECUTE | UNMASK | SELECT_MASKED ::= ALL KEYSPACES | KEYSPACE diff --git a/doc/modules/cassandra/examples/CQL/sai/create-vector-index-cycling.cql b/doc/modules/cassandra/examples/CQL/sai/create-vector-index-cycling.cql index f5379f1d2a..2883f9f9cf 100644 --- a/doc/modules/cassandra/examples/CQL/sai/create-vector-index-cycling.cql +++ b/doc/modules/cassandra/examples/CQL/sai/create-vector-index-cycling.cql @@ -1,2 +1,2 @@ -CREATE CUSTOM INDEX IF NOT EXISTS ann_index - ON cycling.comments_vs(comment_vector) USING 'StorageAttachedIndex'; \ No newline at end of file +CREATE INDEX IF NOT EXISTS ann_index + ON cycling.comments_vs(comment_vector) USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/create-vector-index.cql b/doc/modules/cassandra/examples/CQL/sai/create-vector-index.cql index e379365c62..580baf7f70 100644 --- a/doc/modules/cassandra/examples/CQL/sai/create-vector-index.cql +++ b/doc/modules/cassandra/examples/CQL/sai/create-vector-index.cql @@ -1,2 +1,2 @@ -CREATE CUSTOM INDEX IF NOT EXISTS ann_index - ON vsearch.products(item_vector) USING 'StorageAttachedIndex'; \ No newline at end of file +CREATE INDEX IF NOT EXISTS ann_index ON vsearch.products(item_vector) + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/cyclist_semi_pro_sai_indices.cql b/doc/modules/cassandra/examples/CQL/sai/cyclist_semi_pro_sai_indices.cql index c239f176c7..e12d65d0d8 100644 --- a/doc/modules/cassandra/examples/CQL/sai/cyclist_semi_pro_sai_indices.cql +++ b/doc/modules/cassandra/examples/CQL/sai/cyclist_semi_pro_sai_indices.cql @@ -6,17 +6,17 @@ DROP INDEX IF EXISTS cycling.country_sai_idx; DROP INDEX IF EXISTS cycling.registration_sai_idx; // tag::createQuickStartIndices[] -CREATE CUSTOM INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) -USING 'StorageAttachedIndex' +CREATE INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) +USING 'sai' WITH OPTIONS = {'case_sensitive': 'false', 'normalize': 'true', 'ascii': 'true'}; -CREATE CUSTOM INDEX age_sai_idx ON cycling.cyclist_semi_pro (age) -USING 'StorageAttachedIndex'; +CREATE INDEX age_sai_idx ON cycling.cyclist_semi_pro (age) +USING 'sai'; -CREATE CUSTOM INDEX country_sai_idx ON cycling.cyclist_semi_pro (country) -USING 'StorageAttachedIndex' +CREATE INDEX country_sai_idx ON cycling.cyclist_semi_pro (country) +USING 'sai' WITH OPTIONS = {'case_sensitive': 'false', 'normalize': 'true', 'ascii': 'true'}; -CREATE CUSTOM INDEX registration_sai_idx ON cycling.cyclist_semi_pro (registration) -USING 'StorageAttachedIndex'; +CREATE INDEX registration_sai_idx ON cycling.cyclist_semi_pro (registration) +USING 'sai'; // end::createQuickStartIndices[] diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-case-sensitive.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-case-sensitive.cql index 71a1d2b413..8bb3686ecc 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-case-sensitive.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-case-sensitive.cql @@ -1,3 +1,3 @@ -CREATE CUSTOM INDEX commenter_cs_idx ON cycling.comments_vs (commenter) -USING 'StorageAttachedIndex' +CREATE INDEX commenter_cs_idx ON cycling.comments_vs (commenter) +USING 'sai' WITH OPTIONS = {'case_sensitive': 'true', 'normalize': 'true', 'ascii': 'true'}; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-composite-partition-key-index-match.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-composite-partition-key-index-match.cql index e8f4e3a89c..8c7023250c 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-composite-partition-key-index-match.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-composite-partition-key-index-match.cql @@ -1,6 +1,6 @@ -CREATE CUSTOM INDEX race_name_idx +CREATE INDEX race_name_idx ON cycling.rank_by_year_and_name (race_name) - USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX race_year_idx + USING 'sai'; +CREATE INDEX race_year_idx ON cycling.rank_by_year_and_name (race_year) - USING 'StorageAttachedIndex'; \ No newline at end of file + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-from-list-match.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-from-list-match.cql index 7019b9d6e4..7f8cd21205 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-from-list-match.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-from-list-match.cql @@ -1,3 +1,3 @@ -CREATE CUSTOM INDEX events_idx +CREATE INDEX events_idx ON cycling.upcoming_calendar (events) - USING 'StorageAttachedIndex'; \ No newline at end of file + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-from-set-match.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-from-set-match.cql index ba2bfd9930..b024a824ab 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-from-set-match.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-from-set-match.cql @@ -1,3 +1,3 @@ -CREATE CUSTOM INDEX teams_idx +CREATE INDEX teams_idx ON cycling.cyclist_career_teams (teams) - USING 'StorageAttachedIndex'; \ No newline at end of file + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-multiple-index-match-with-and.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-multiple-index-match-with-and.cql index 8c0f9b8ad0..1a0533006c 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-multiple-index-match-with-and.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-multiple-index-match-with-and.cql @@ -1,10 +1,10 @@ -CREATE CUSTOM INDEX commenter_idx +CREATE INDEX commenter_idx ON cycling.comments_vs (commenter) - USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX created_at_idx + USING 'sai'; +CREATE INDEX created_at_idx ON cycling.comments_vs (created_at) - USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX ann_index + USING 'sai'; +CREATE INDEX ann_index ON cycling.comments_vs (comment_vector) - USING 'StorageAttachedIndex'; + USING 'sai'; diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-similarity-function.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-similarity-function.cql index 76508d9a22..3ed937fb38 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-similarity-function.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-similarity-function.cql @@ -1,4 +1,4 @@ -CREATE CUSTOM INDEX sim_comments_idx +CREATE INDEX sim_comments_idx ON cycling.comments_vs (comment_vector) - USING 'StorageAttachedIndex' + USING 'sai' WITH OPTIONS = { 'similarity_function': 'DOT_PRODUCT'}; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-single-index-match.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-single-index-match.cql index 8a9208faf5..157c383967 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-single-index-match.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-single-index-match.cql @@ -1,3 +1,3 @@ -CREATE CUSTOM INDEX commenter_idx +CREATE INDEX commenter_idx ON cycling.comments_vs (commenter) - USING 'StorageAttachedIndex'; \ No newline at end of file + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/index-sai-udt-match.cql b/doc/modules/cassandra/examples/CQL/sai/index-sai-udt-match.cql index 30a86c770e..3aff24eb72 100644 --- a/doc/modules/cassandra/examples/CQL/sai/index-sai-udt-match.cql +++ b/doc/modules/cassandra/examples/CQL/sai/index-sai-udt-match.cql @@ -1,3 +1,3 @@ -CREATE CUSTOM INDEX races_idx +CREATE INDEX races_idx ON cycling.cyclist_races (races) - USING 'StorageAttachedIndex'; \ No newline at end of file + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/CQL/sai/sai-error.cql b/doc/modules/cassandra/examples/CQL/sai/sai-error.cql index bcb79d6a22..50b4d2aecd 100644 --- a/doc/modules/cassandra/examples/CQL/sai/sai-error.cql +++ b/doc/modules/cassandra/examples/CQL/sai/sai-error.cql @@ -1,2 +1,2 @@ -CREATE CUSTOM INDEX ON demo2.person_id_name_primarykey (id) - USING 'StorageAttachedIndex'; \ No newline at end of file +CREATE INDEX ON demo2.person_id_name_primarykey (id) + USING 'sai'; \ No newline at end of file diff --git a/doc/modules/cassandra/examples/RESULTS/sai/cyclist_semi_pro_describe.result b/doc/modules/cassandra/examples/RESULTS/sai/cyclist_semi_pro_describe.result index 84e9ed0355..8d737cc49f 100644 --- a/doc/modules/cassandra/examples/RESULTS/sai/cyclist_semi_pro_describe.result +++ b/doc/modules/cassandra/examples/RESULTS/sai/cyclist_semi_pro_describe.result @@ -22,8 +22,8 @@ CREATE TABLE cycling.cyclist_semi_pro ( AND nodesync = {'enabled': 'true', 'incremental': 'true'} AND read_repair = 'BLOCKING' AND speculative_retry = '99PERCENTILE'; -CREATE CUSTOM INDEX registration_sai_idx ON cycling.cyclist_semi_pro (registration) USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX country_sai_idx ON cycling.cyclist_semi_pro (country) USING 'StorageAttachedIndex' WITH OPTIONS = {'normalize': 'true', 'case_sensitive': 'false', 'ascii': 'true'}; -CREATE CUSTOM INDEX age_sai_idx ON cycling.cyclist_semi_pro (age) USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) USING 'StorageAttachedIndex' WITH OPTIONS = {'normalize': 'true', 'case_sensitive': 'false', 'ascii': 'true'}; +CREATE INDEX registration_sai_idx ON cycling.cyclist_semi_pro (registration) USING 'sai'; +CREATE INDEX country_sai_idx ON cycling.cyclist_semi_pro (country) USING 'sai' WITH OPTIONS = {'normalize': 'true', 'case_sensitive': 'false', 'ascii': 'true'}; +CREATE INDEX age_sai_idx ON cycling.cyclist_semi_pro (age) USING 'sai'; +CREATE INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) USING 'sai' WITH OPTIONS = {'normalize': 'true', 'case_sensitive': 'false', 'ascii': 'true'}; // end::cyclistSemiProDescribeResults[] diff --git a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc index 7976fd0af7..18dd52e13d 100644 --- a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc +++ b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc @@ -2519,7 +2519,7 @@ _Syntax:_ bc(syntax).. + ::= GRANT ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON TO -::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | UNMASK | SELECT_MASKED +::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | UNMASK | SELECT_MASKED EXECUTE ::= ALL KEYSPACES + @@ -2591,7 +2591,7 @@ _Syntax:_ bc(syntax).. + ::= REVOKE ( ALL ( PERMISSIONS )? | ( PERMISSION )? ) ON FROM -::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESRIBE | UNMASK | SELECT_MASKED +::= CREATE | ALTER | DROP | SELECT | MODIFY | AUTHORIZE | DESCRIBE | UNMASK | SELECT_MASKED EXECUTE ::= ALL KEYSPACES + diff --git a/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc b/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc index 329e373966..29fcfbc1bd 100644 --- a/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc +++ b/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc @@ -28,12 +28,12 @@ See the xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[SAI section // tag::syntax[] ---- -CREATE CUSTOM INDEX [ IF NOT EXISTS ] [ ] +CREATE [CUSTOM] INDEX [ IF NOT EXISTS ] [ ] ON [ .] () | [ (KEYS()) ] | [ (VALUES()) ] | [ (ENTRIES()) ] - USING 'StorageAttachedIndex' + USING 'sai' [ WITH OPTIONS = { } ] ; ---- // end::syntax[] @@ -156,9 +156,9 @@ Creating multiple SAI indexes with different map types *on the same column* requ [source,language-cql] ---- -CREATE CUSTOM INDEX ON audit (KEYS(text_map)) USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX ON audit (VALUES(text_map)) USING 'StorageAttachedIndex'; -CREATE CUSTOM INDEX ON audit (ENTRIES(text_map)) USING 'StorageAttachedIndex'; +CREATE INDEX ON audit (KEYS(text_map)) USING 'sai'; +CREATE INDEX ON audit (VALUES(text_map)) USING 'sai'; +CREATE INDEX ON audit (ENTRIES(text_map)) USING 'sai'; ---- Insert some data: @@ -321,7 +321,7 @@ Create an SAI index using the collection's `years` column. [source,language-cql] ---- -CREATE CUSTOM INDEX ON calendar(years) USING 'StorageAttachedIndex'; +CREATE INDEX ON calendar(years) USING 'sai'; ---- Insert some random `int` list data for `years`, just for demo purposes. @@ -381,7 +381,7 @@ Create an SAI index using the collection's `years` column -- this time for the ` [source,language-cql] ---- -CREATE CUSTOM INDEX ON calendar2(years) USING 'StorageAttachedIndex'; +CREATE INDEX ON calendar2(years) USING 'sai'; ---- Insert some random `int` set data for `years`, again just for demo purposes. diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc index aaa279e0fe..d7a5c708f8 100644 --- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc +++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc @@ -10,7 +10,7 @@ To create a simple SAI index: include::cassandra:example$CQL/sai/cyclist_semi_pro_sai_indices.cql[tag=createQuickStartIndices] ---- -For most SAI indexes, the column name is defined in the `CREATE CUSTOM INDEX` statement that also uses `USING 'StorageAttachedIndex'`. +For most SAI indexes, the column name is defined in the `CREATE INDEX` statement that also uses `USING 'sai'`. The SAI index options are defined in the `WITH OPTIONS` clause. The `case_sensitive` option is set to `false` to allow case-insensitive searches. The `normalize` option is set to `true` to allow searches to be normalized for Unicode characters. diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc index 07a23e832f..95f6783a25 100644 --- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc +++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc @@ -15,11 +15,11 @@ SAI combines: include::cassandra:partial$sai/support-databases.adoc[] -After creating your database, a keyspace, and one or more tables, use `+CREATE CUSTOM INDEX ... -USING 'StorageAttachedIndex'+` to define one or more SAI indexes on the table. +After creating your database, a keyspace, and one or more tables, use `+CREATE INDEX ... +USING 'sai'+` to define one or more SAI indexes on the table. For Cassandra databases, use `cqlsh`. -The same `+CREATE CUSTOM INDEX ... -USING 'StorageAttachedIndex'+` command is available for both. +The same `+CREATE INDEX ... +USING 'sai'+` command is available for both. See xref:cassandra:getting-started/sai-quickstart.adoc[SAI quickstart]. == What configuration settings should I use with SAI? @@ -89,8 +89,8 @@ The features, by design, are intentionally simple and easy to use. At a high level, SAI indexes are: -* Created and dropped per column via CQL `+CREATE CUSTOM INDEX ... -USING 'StorageAttachedIndex'+` commands and `DROP INDEX` commands. +* Created and dropped per column via CQL `+CREATE INDEX ... +USING 'sai'+` commands and `DROP INDEX` commands. Start in xref:cassandra:getting-started/sai-quickstart.adoc[SAI quickstart]. * Rebuilt and backed up via nodetool. @@ -203,15 +203,15 @@ SELECT * FROM audit WHERE text_map CONTAINS KEY 'Giovani'; For query examples with `CONTAINS` clauses that take advantage of SAI collection maps, lists, and sets, be sure to see xref:developing/cql/indexing/sai/collections.adoc[SAI collection map examples with keys, values, and entries]. ==== -== On the `CREATE CUSTOM INDEX` command for SAI, what options are available? +== On the `CREATE INDEX` command for SAI, what options are available? Use the `WITH OPTIONS` clause to indicate how SAI should handle case sensitivity and special characters in the index. For example, given a string column `lastname`: [source,language-cql] ---- -CREATE CUSTOM INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) - USING 'StorageAttachedIndex' WITH OPTIONS = +CREATE INDEX lastname_sai_idx ON cycling.cyclist_semi_pro (lastname) + USING 'sai' WITH OPTIONS = {'case_sensitive': 'false', 'normalize': 'true', 'ascii': 'true'}; ---- diff --git a/doc/modules/cassandra/pages/managing/operating/repair.adoc b/doc/modules/cassandra/pages/managing/operating/repair.adoc index 3c6b20d3a9..1823a6d4ef 100644 --- a/doc/modules/cassandra/pages/managing/operating/repair.adoc +++ b/doc/modules/cassandra/pages/managing/operating/repair.adoc @@ -62,13 +62,10 @@ Or even on specific tables: nodetool repair [options] ---- -The repair command only repairs token ranges on the node being repaired, -it doesn't repair the whole cluster. By default, repair will operate on -all token ranges replicated by the node you're running repair on, which -will cause duplicate work if you run it on every node. The `-pr` flag -will only repair the "primary" ranges on a node, so you can repair your -entire cluster by running `nodetool repair -pr` on each node in a single -datacenter. + +The repair command repairs token ranges only on the node being repaired; it does not repair the whole cluster. +By default, repair operates on all token ranges replicated by the node on which repair is run, causing duplicate work when running it on every node. Avoid duplicate work by using the `-pr` flag to repair only the "primary" ranges on a node. +Do a full cluster repair by running the `nodetool repair -pr` command on each node in each datacenter in the cluster, until all of the nodes and datacenters are repaired. The specific frequency of repair that's right for your cluster, of course, depends on several factors. However, if you're just starting out diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc index c89521192e..b4dc08566f 100644 --- a/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc +++ b/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc @@ -8,7 +8,6 @@ You can create multiple secondary indexes on the same database table, with each All column date types except the following are supported for SAI indexes: * `counter` -* geospatial types: `PointType`, `LineStringType`, `PolygonType` * non-frozen user-defined type (UDT) .One exception @@ -22,7 +21,7 @@ If you need to query based on one of those columns, an SAI index is a helpful op Defining one or more SAI indexes based on any column in a database table (with the rules noted above) subsequently gives you the ability to run performant queries that use the indexed column to filter results. -See the xref:cassandra:developing/indexing/sai/sai-overview.adoc[SAI section]. +See the xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[SAI section]. == Syntax @@ -53,7 +52,7 @@ include::cassandra:partial$cql-syntax-legend.adoc[] index_name:: Optional identifier for index. -If no name is specified, the default is used, `__idx`. +If no name is specified, the default used is `\_\_idx`. Enclose in quotes to use special characters or to preserve capitalization. column_name:: @@ -63,7 +62,7 @@ SAI returns `InvalidRequestException` if you try to define an index on a column map_name:: // LLP FIX -Used with xref:cassandra:developing/collections/collection-create.adoc[collections], identifier of the `map_name` specified in `CREATE TABLE` ... +Used with xref:cassandra:developing/cql/collections/collection-create.adoc[collections], identifier of the `map_name` specified in `CREATE TABLE` ... `map()`. The regular column syntax applies for collection types `list` and `set`. @@ -119,14 +118,14 @@ include::cassandra:example$CQL/sai/cyclist_semi_pro_sai_indices.cql[tag=createQu ---- For sample queries that find data in `cycling.cyclist_semi_pro` via these sample SAI indexes, see xref:cassandra:getting-started/sai-quickstart.adoc#saiQuickStart__saiQuickStartSubmitQueries[Submit CQL queries]. -Also refer xref:cassandra:developing/indexing/sai/sai-query.adoc[Examine SAI column index and query rules]. +Also refer xref:cassandra:developing/cql/indexing/sai/sai-query.adoc[Querying with SAI]. [[saiCollectionsExamples]] === SAI collection map examples with keys, values, and entries // LLP FIX The following examples demonstrate using collection maps of multiple types (`keys`, `values`, `entries`) in SAI indexes. -For related information, see xref:cassandra:developing/collections/collection-create.adoc[Creating collections] and xref:cassandra:developing/collections/map.adoc[Using map type]. +For related information, see xref:cassandra:developing/cql/collections/collection-create.adoc[Creating collections] and xref:cassandra:developing/cql/collections/map.adoc[Using map type]. Also refer to the SAI collection examples of type xref:#saiCreateCustomIndexCollectionsListAndSetExamples[list and set] in this topic. @@ -293,9 +292,10 @@ Remember that in CQL queries using SAI indexes, the `CONTAINS` clauses are suppo These examples demonstrate using collections with the `list` and `set` types in SAI indexes. For related information, see: // LLP FIX -* xref:cassandra:developing/collections/collection-create.adoc[Creating collections] -* xref:cassandra:developing/collections/list.adoc[Using list type] -* xref:cassandra:developing/collections/set.adoc[Using set type] + +* xref:cassandra:developing/cql/collections/collection-create.adoc[Creating collections] +* xref:cassandra:developing/cql/collections/list.adoc[Using list type] +* xref:cassandra:developing/cql/collections/set.adoc[Using set type] If you have not already, create the keyspace. diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc index 48bcd218ab..71994d86e1 100644 --- a/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc +++ b/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc @@ -1,9 +1,21 @@ = CREATE INDEX -:description: Defines a new index for a single column of a table. +:description: Defines a new secondary index (2i) or storage-attached index (SAI) for a single column of a table. {description} -{product} supports creating an index on most columns, including the partition and cluster columns of a `PRIMARY KEY`, collections, and static columns. For maps, you can index using the key, value, or entries (a key:value pair). +{product} supports creating a secondary index or storage-attached index on most columns, including the partition and cluster columns of a `PRIMARY KEY`, collections, and static columns. +For maps, you can index using the key, value, or entries (a key:value pair). + +All column date types except the following are supported for SAI indexes: + +* `counter` +* non-frozen user-defined type (UDT) + +.One exception +**** +You cannot define an SAI index based on the partition key when it's comprised of only one column. +If you attempt to create an SAI index in this case, SAI issues an error message. +**** *See also:* xref:cassandra:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] for Storage-Attached Indexes (SAI), xref:cassandra:reference/cql-commands/drop-index.adoc[DROP INDEX] @@ -19,14 +31,13 @@ include::example$BNF/index_name.bnf[] // tag::syntax[] ---- CREATE INDEX [ IF NOT EXISTS ] - ON [.] - ([ ( KEYS | FULL ) ] ) - (ENTRIES ) + ON [.] + ( [ ( KEYS | FULL | ENTRIES ) ] ) // | [ (KEYS()) ] // | [ (VALUES()) ] // | [ (ENTRIES()) ] - [USING 'sai' - [ WITH OPTIONS = { } ]]; + [USING 'sai'] + [ WITH OPTIONS = { } ]; ---- // end::syntax[] @@ -38,36 +49,105 @@ include::cassandra:partial$cql-syntax-legend.adoc[] == Required parameters -*table_name*:: -Name of the table to index. +[cols="1,3"] +|=== -*column_name*:: -Name of the column to index. +| Parameter | Description + +| table_name +| Name of the table to index. + +| column_name +| Name of the column to index. +SAI allows only alphanumeric characters and underscores in names. +SAI returns `InvalidRequestException` if you try to define an index on a column name that contains other characters, and does not create the index. +|=== == Optional parameters -*index_name*:: -Name of the index. -Enclose in quotes to use special characters or preserve capitalization. -If no name is specified, {product} names the index: `__idx`. +[cols="1,3"] +|=== -*keyspace_name*:: -Name of the keyspace that contains the table to index. +| Parameter | Description + +| index_name +| Name of the index. +Enclose in quotes to use special characters or preserve capitalization. +If no name is specified, {product} names the index as `\_\_idx`. + +| keyspace_name +| Name of the keyspace that contains the table to index. If no name is specified, the current keyspace is used. +| map_name +| Used with xref:cassandra:developing/cql/collections/collection-create.adoc[collections], identifier of the `map_name` specified in `CREATE TABLE` ... +`map()`. +The regular column syntax applies for collection types `list` and `set`. +|=== + +[cols=",,"] +|============================ +|option_map 2+<|Define options in JSON simple format. +.4+||`case_sensitive` |Ignore case in matching string values. Default: `true` +|`normalize`|When set to `true`, perform https://unicode.org/faq/normalization.html[Unicode normalization] on indexed strings. +SAI supports Normalization Form C (NFC) Unicode. +When set to `true`, SAI normalizes the different versions of a given Unicode character to a single version, retaining all the marks and symbols in the index. +For example, SAI would change the character Å (U+212B) to Å (U+00C5). + +When implementations keep strings in a normalized form, equivalent strings have a unique binary representation. +See https://unicode.org/reports/tr15/[Unicode Standard Annex #15, Unicode Normalization Forms]. + +Default: `false`. +|`ascii` |When set to `true`, SAI converts alphabetic, numeric, and symbolic characters that are not in the Basic Latin Unicode block (the first 127 ASCII characters) to the ASCII equivalent, if one exists. +For example, this option changes à to a. +Default: `false`. +|`similarity_function` |Vector search relies on computing the similarity or distance between vectors to identify relevant matches. +The similarity function is used to compute the similarity between two vectors. +Valid options are: `EUCLIDEAN`, `DOT_PRODUCT`, `COSINE` +Default: `COSINE` +|============================ + == Usage notes If the column already contains data, it is indexed during the execution of this statement. After an index has been created, it is automatically updated when data in the column changes. Indexing with the `CREATE INDEX` command can impact performance. -Before creating an index, be aware of when and xref:cassandra:developing/indexing/2i/2i-when-to-use.adoc#when-no-index[when not to create an index]. +Before creating an index, be aware of when and xref:cassandra:developing/cql/indexing/2i/2i-when-to-use.adoc#when-no-index[when not to create an index]. *Restriction:* Indexing counter columns is not supported. +=== SAI indexes + +You can define an SAI index on one of the columns in a table's composite partition key, i.e., a partition key comprised of multiple columns. +If you need to query based on one of those columns, an SAI index is a helpful option. +In fact, you can define an SAI index on each column in a composite partition key, if needed. + +Defining one or more SAI indexes based on any column in a database table allows queries to use the indexed column to filter results. + +=== SAI query operators + +SAI supports the following query operators for tables with SAI indexes: + +include::cassandra:partial$sai/supported-query-operators-list.adoc[] + +SAI does not support the following query operators for tables with SAI indexes: + +include::cassandra:partial$sai/notSupportedOperators.adoc[] + +See the xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[SAI section]. + == Examples -=== Creating an index on a clustering column +// LLP +// Need all of these for both 2i, SASI, and SAI: +// Create index on non-primary key column +// Create index on primary key column - partition key? +// on a composite partition key +// Create index on primary key column - clustering key +// set, list, map - key, value, entries, full + +=== Creating a SAI index on a clustering column Define a table having a xref:cassandra:reference/cql-commands/create-table.adoc#cqlPKcomposite[composite partition key], and then create an index on a clustering column. @@ -158,7 +238,7 @@ include::cassandra:example$RESULTS/sai/select_all_from_cyclist_career_teams-team [[CreatIdxCollKey]] === Creating an index on map keys -You can create an index on xref:cassandra:developing/indexing/2i/2i-create-on-collection.adoc[map collection keys]. +You can create an index on xref:cassandra:developing/cql/indexing/2i/_2i-create-on-collection.adoc[map collection keys]. If an index of the map values of the collection exists, drop that index before creating an index on the map collection keys. Assume a cyclist table contains this map data where `nation is the map key and `Canada` is the map value`: diff --git a/doc/modules/cassandra/pages/reference/vector-data-type.adoc b/doc/modules/cassandra/pages/reference/vector-data-type.adoc index 10d26f4d58..7fa19d407b 100644 --- a/doc/modules/cassandra/pages/reference/vector-data-type.adoc +++ b/doc/modules/cassandra/pages/reference/vector-data-type.adoc @@ -8,6 +8,8 @@ The new type `VECTOR` has the following properties: * elements may not be null * flatten array (aka multi-cell = false) +Although the data type specified for a vector used in vector search is a floating point number, the vector data type can be used to create a vector of any data type. For example, a vector of `int` or `bigint` can be created, but cannot be used with vector search. + [NOTE] ==== Vectors are limited to a maximum dimension of 8K (2^13) items. diff --git a/doc/modules/cassandra/pages/vector-search/concepts.adoc b/doc/modules/cassandra/pages/vector-search/concepts.adoc index 3a6196eaba..05c97e4c1d 100644 --- a/doc/modules/cassandra/pages/vector-search/concepts.adoc +++ b/doc/modules/cassandra/pages/vector-search/concepts.adoc @@ -29,6 +29,7 @@ Embeddings capture the semantic meaning of the text, which in turn, allow querie Large Language Models (LLMs) generate contextual embeddings for the data, and optimize embeddings for queries. Trained embeddings like those produced by LLMs can be used in Natural Language Processing (NLP) tasks such as text classification, sentiment analysis, and machine translation. +You can embed almost any kind of data and retrieve good results with vector search. As models continue to improve, the quality of results will also continue to improve. == Storage Attached Indexing (SAI) diff --git a/doc/modules/cassandra/pages/vector-search/data-modeling.adoc b/doc/modules/cassandra/pages/vector-search/data-modeling.adoc index 0150fd4bf3..232873ea19 100644 --- a/doc/modules/cassandra/pages/vector-search/data-modeling.adoc +++ b/doc/modules/cassandra/pages/vector-search/data-modeling.adoc @@ -27,6 +27,12 @@ This means that the embeddings should follow the same principles and rules to en Using the same embedding library guarantees this compatibility because the library consistently transforms data into vectors in a specific, defined way. For example, comparing Word2Vec embeddings with BERT (an LLM) embeddings could be problematic because these models have different architectures and create embeddings in fundamentally different ways. +Thus, the vector data type is a fixed-length vector of floating-point numbers. +The dimension value is defined by the embedding model you use. +Some machine learning libraries will tell you the dimension value, but you must define it with the embedding model. +Selecting an embedding model for your dataset that creates good structure by ensuring related objects are nearby each other in the embedding space is important. +You may need to test out different embedding models to determine which one works best for your dataset. + == Preprocessing embeddings vectors Normalizing is about scaling the data so it has a length of one. @@ -129,4 +135,31 @@ Your {cassandra} database using Vector Search efficiently distributes data and a Continuously evaluate and iterate the data to refine search results against known truth and user feedback. This also helps identify areas for improvement. -Iteratively refining the vector representations, similarity metrics, indexing techniques, or preprocessing steps can lead to better search performance and user satisfaction. \ No newline at end of file +Iteratively refining the vector representations, similarity metrics, indexing techniques, or preprocessing steps can lead to better search performance and user satisfaction. + +== Use cases + +Vector databases with well-optimized embeddings allow for new ways to search and associate data, generating results which previously would not have been possible with traditional databases. + +Examples: + +* Search for items that are similar to a given item, without needing to know the exact item name or IDs +* Retrieve documents based on similarity of context and content rather than exact string or keyword matches +* Expand search results across dissimilar items, such as searching for a product and retrieving contextually similar products from a different category +* Execute word similarity searches, and suggest to users ways to rephrase queries or passages +* Encode text, images, audio, or video as queries and retrieve media that are conceptually, visually, audibly, or contextually similar to the input +* Reduce time spent on metadata and curation by automatically generating associations for data +* Improve data quality by automatically identifying and removing duplicates + +== Best practices + +* Store relevant metadata about a vector in other columns in your table. For example, if your vector is an image, store the original image in the same table. +* Select a pre-trained model based on the queries you will need to make to your database. + +== Limitations + +While the vector embeddings can replace or augment some functions of a traditional database, vector embeddings are not a replacement for other data types. Embeddings are best applied as a supplement to existing data because of the limitations: + +* Vector embeddings are not human-readable. Embeddings are not recommended when seeking to +directly retrieve data from a table. +* The model might not be able to capture all relevant information from the data, leading to incorrect or incomplete results. \ No newline at end of file diff --git a/doc/modules/cassandra/partials/table-properties.adoc b/doc/modules/cassandra/partials/table-properties.adoc index 2c49a7a495..d15ad30eb9 100644 --- a/doc/modules/cassandra/partials/table-properties.adoc +++ b/doc/modules/cassandra/partials/table-properties.adoc @@ -55,7 +55,7 @@ Valid values: * `TRUE`- create CDC log * `FALSE`- do not create CDC log -*comments = 'some text that describes the table'* :: +*comment = 'some text that describes the table'* :: Provide documentation on the table. [TIP] diff --git a/pylib/cqlshlib/cqlshmain.py b/pylib/cqlshlib/cqlshmain.py index f4f4146c96..c4b4ace2ef 100755 --- a/pylib/cqlshlib/cqlshmain.py +++ b/pylib/cqlshlib/cqlshmain.py @@ -1284,7 +1284,7 @@ class Shell(cmd.Cmd): future = self.session.execute_async(stmt) if self.connection_versions['build'][0] < '4': - print('\nWARN: DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESRIBE|DESC ' + print('\nWARN: DESCRIBE|DESC was moved to server side in Cassandra 4.0. As a consequence DESCRIBE|DESC ' 'will not work in cqlsh %r connected to Cassandra %r, the version that you are connected to. ' 'DESCRIBE does not exist server side prior Cassandra 4.0.' % (version, self.connection_versions['build']))