diff --git a/doc/antora.yml b/doc/antora.yml index 364caec91f..e50d13daa2 100644 --- a/doc/antora.yml +++ b/doc/antora.yml @@ -5,11 +5,10 @@ prerelease: true asciidoc: attributes: cass_url: 'http://cassandra.apache.org/' - product: Cassandra - cassandra: Apache Cassandra - 30_version: '3.0' - 22_version: '2.2' - 21_version: '2.1' + cass-50: 'Cassandra 5.0' + cassandra: 'Cassandra' + product: 'Apache Cassandra' + nav: - modules/ROOT/nav.adoc - modules/cassandra/nav.adoc \ No newline at end of file diff --git a/doc/modules/cassandra/examples/BNF/native_type.bnf b/doc/modules/cassandra/examples/BNF/native_type.bnf index c4e9c268db..0aaa46acbe 100644 --- a/doc/modules/cassandra/examples/BNF/native_type.bnf +++ b/doc/modules/cassandra/examples/BNF/native_type.bnf @@ -1,4 +1,4 @@ native_type::= ASCII | BIGINT | BLOB | BOOLEAN | COUNTER | DATE | DECIMAL | DOUBLE | DURATION | FLOAT | INET | INT | SMALLINT | TEXT | TIME | TIMESTAMP | TIMEUUID | TINYINT | -UUID | VARCHAR | VARINT +UUID | VARCHAR | VARINT | VECTOR diff --git a/doc/modules/cassandra/examples/CQL/vector-search/vector-search-cycling.cql b/doc/modules/cassandra/examples/CQL/vector-search/vector-search-cycling.cql new file mode 100644 index 0000000000..3bc4162303 --- /dev/null +++ b/doc/modules/cassandra/examples/CQL/vector-search/vector-search-cycling.cql @@ -0,0 +1,119 @@ +// tag::create-vs-keyspace-cycling[] +CREATE KEYSPACE IF NOT EXISTS cycling + WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : '1' }; +// end::create-vs-keyspace-cycling[] + +// tag::use-vs-keyspace-cycling[] +USE cycling; +// end::use-vs-keyspace-cycling[] + +// tag::create-vs-table[] +CREATE TABLE IF NOT EXISTS cycling.comments_vs ( + record_id timeuuid, + id uuid, + commenter text, + comment text, + comment_vector VECTOR ; + created_at timestamp, + PRIMARY KEY (id, created_at) +) +WITH CLUSTERING ORDER BY (created_at DESC); +// end::create-vs-table[] + +// tag::alter-vs-table[] +ALTER TABLE cycling.comments_vs + ADD comment_vector VECTOR ; <1> +// end::alter-vs-table[] + +// tag::create-vs-index[] +CREATE INDEX IF NOT EXISTS ann_index + ON cycling.comments_vs(comment_vector) USING 'sai'; +// end::create-vs-index[] + +// tag::create-vs-index-with-option[] +CREATE INDEX IF NOT EXISTS ann_index + ON vsearch.com(item_vector) USING 'sai' + WITH OPTIONS = { 'similarity_function': 'DOT_PRODUCT' }; +// end::create-vs-index-with-option[] + +// tag::insert-vector-data-cycling[] +INSERT INTO cycling.comments_vs (record_id, id, commenter, comment, created_at, comment_vector) + VALUES ( + now(), + e7ae5cf3-d358-4d99-b900-85902fda9bb0, + '2017-02-14 12:43:20-0800', + 'Raining too hard should have postponed', + 'Alex', + [0.45, 0.09, 0.01, 0.2, 0.11] +); +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + e7ae5cf3-d358-4d99-b900-85902fda9bb0, + '2017-03-21 13:11:09.999-0800', + 'Second rest stop was out of water', + 'Alex', + [0.99, 0.5, 0.99, 0.1, 0.34] +); +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + e7ae5cf3-d358-4d99-b900-85902fda9bb0, + '2017-04-01 06:33:02.16-0800', + 'LATE RIDERS SHOULD NOT DELAY THE START', + 'Alex', + [0.9, 0.54, 0.12, 0.1, 0.95] +); + +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + c7fceba0-c141-4207-9494-a29f9809de6f, + totimestamp(now()), + 'The gift certificate for winning was the best', + 'Amy', + [0.13, 0.8, 0.35, 0.17, 0.03] +); + +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + c7fceba0-c141-4207-9494-a29f9809de6f, + '2017-02-17 12:43:20.234+0400', + 'Glad you ran the race in the rain', + 'Amy', + [0.3, 0.34, 0.2, 0.78, 0.25] +); + +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + c7fceba0-c141-4207-9494-a29f9809de6f, + '2017-03-22 5:16:59.001+0400', + 'Great snacks at all reststops', + 'Amy', + [0.1, 0.4, 0.1, 0.52, 0.09] +); +INSERT INTO cycling.comments_vs (record_id, id, created_at, comment, commenter, comment_vector) + VALUES ( + now(), + c7fceba0-c141-4207-9494-a29f9809de6f, + '2017-04-01 17:43:08.030+0400', + 'Last climb was a killer', + 'Amy', + [0.3, 0.75, 0.2, 0.2, 0.5] +); +// end::insert-vector-data-cycling[] + +// tag::select-vector-data-cycling[] +SELECT * FROM cycling.comments_vs + ORDER BY comment_vector ANN OF [0.15, 0.1, 0.1, 0.35, 0.55] + LIMIT 3; +// end::select-vector-data-cycling[] + +// tag::select-vector-data-similarity-cycling[] +SELECT description, similarity_cosine(item_vector, [0.2, 0.15, 0.3, 0.2, 0.05]) + FROM vsearch.cycling + ORDER BY comment_vector ANN OF [0.1, 0.15, 0.3, 0.12, 0.05] + LIMIT 1; +// end::select-vector-data-similarity-cycling[] \ No newline at end of file diff --git a/doc/modules/cassandra/examples/RESULTS/select_all_from_popular_count.result b/doc/modules/cassandra/examples/RESULTS/select_all_from_popular_count.result new file mode 100644 index 0000000000..06df096b67 Binary files /dev/null and b/doc/modules/cassandra/examples/RESULTS/select_all_from_popular_count.result differ diff --git a/doc/modules/cassandra/examples/TEXT/normalized-embedding-vectors.txt b/doc/modules/cassandra/examples/TEXT/normalized-embedding-vectors.txt new file mode 100644 index 0000000000..dc2311c814 --- /dev/null +++ b/doc/modules/cassandra/examples/TEXT/normalized-embedding-vectors.txt @@ -0,0 +1,3 @@ +[0.27, 0.40, 0.80, 0.32, 0.13] +[0.88, 0.18, 0.02, 0.39, 0.21] +[0.15, 0.07, 0.12, 0.44, 0.88] \ No newline at end of file diff --git a/doc/modules/cassandra/examples/TEXT/original-embedding-vectors.txt b/doc/modules/cassandra/examples/TEXT/original-embedding-vectors.txt new file mode 100644 index 0000000000..37f76a5104 --- /dev/null +++ b/doc/modules/cassandra/examples/TEXT/original-embedding-vectors.txt @@ -0,0 +1,3 @@ +[0.1, 0.15, 0.3, 0.12, 0.05] +[0.45, 0.09, 0.01, 0.2, 0.11] +[0.1, 0.05, 0.08, 0.3, 0.6] \ No newline at end of file diff --git a/doc/modules/cassandra/nav.adoc b/doc/modules/cassandra/nav.adoc index bacf5073db..53a1d83af6 100644 --- a/doc/modules/cassandra/nav.adoc +++ b/doc/modules/cassandra/nav.adoc @@ -33,7 +33,7 @@ *** xref:cassandra:developing/data-modeling/data-modeling_refining.adoc[] *** xref:cassandra:developing/data-modeling/data-modeling_schema.adoc[] *** xref:cassandra:developing/data-modeling/data-modeling_tools.adoc[] - +// CQL ** xref:cassandra:developing/cql/index.adoc[Cassandra Query Language (CQL)] *** xref:cassandra:developing/cql/definitions.adoc[Definitions] *** xref:cassandra:developing/cql/types.adoc[Data types] @@ -62,42 +62,50 @@ *** xref:cassandra:developing/cql/changes.adoc[Changes] *** xref:cassandra:developing/cql/SASI.adoc[SASI] *** xref:cassandra:developing/cql/cql_singlefile.adoc[Single file of CQL information] -.Managing -** xref:cassandra:managing/configuration/index.adoc[Configuration] -*** xref:cassandra:managing/configuration/cass_yaml_file.adoc[cassandra.yaml] -*** xref:cassandra:managing/configuration/cass_rackdc_file.adoc[cassandra-rackdc.properties] -*** xref:cassandra:managing/configuration/cass_env_sh_file.adoc[cassandra-env.sh] -*** xref:cassandra:managing/configuration/cass_topo_file.adoc[cassandra-topologies.properties] -*** xref:cassandra:managing/configuration/cass_cl_archive_file.adoc[commitlog-archiving.properties] -*** xref:cassandra:managing/configuration/cass_logback_xml_file.adoc[logback.xml] -*** xref:cassandra:managing/configuration/cass_jvm_options_file.adoc[jvm-* files] -*** xref:cassandra:managing/configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units] -** xref:cassandra:managing/operating/index.adoc[Operating] -*** xref:cassandra:managing/operating/snitch.adoc[Snitches] -*** xref:cassandra:managing/operating/topo_changes.adoc[Topology changes] -*** xref:cassandra:managing/operating/repair.adoc[Repair] -*** xref:cassandra:managing/operating/read_repair.adoc[Read repair] -*** xref:cassandra:managing/operating/hints.adoc[Hints] -*** xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters] -*** xref:cassandra:managing/operating/compression.adoc[Compression] -*** xref:cassandra:managing/operating/cdc.adoc[Change Data Capture (CDC)] -*** xref:cassandra:managing/operating/backups.adoc[Backups] -*** xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading] -*** xref:cassandra:managing/operating/metrics.adoc[Metrics] -*** xref:cassandra:managing/operating/security.adoc[Security] -*** xref:cassandra:managing/operating/hardware.adoc[Hardware] -*** xref:cassandra:managing/operating/compaction/index.adoc[Compaction] -*** xref:cassandra:managing/operating/virtualtables.adoc[Virtual tables] -*** xref:cassandra:managing/operating/auditlogging.adoc[Audit logging] -*** xref:cassandra:managing/operating/audit_logging.adoc[Audit logging 2] -*** xref:cassandra:managing/operating/fqllogging.adoc[Full query logging] -*** xref:cassandra:managing/operating/transientreplication.adoc[Transient replication] +// Vector Search +** xref:cassandra:vector-search/overview.adoc[] +*** xref:cassandra:vector-search/concepts.adoc[] +**** xref:cassandra:vector-search/data-modeling.adoc[] +*** xref:cassandra:vector-search/quickstarts.adoc[] +**** xref:cassandra:getting-started/vector-search-quickstart.adoc[] +// **** xref:getting-started/sai-quickstart.adoc[] -** xref:cassandra:managing/tools/index.adoc[Tools] -*** xref:cassandra:managing/tools/cqlsh.adoc[cqlsh: the CQL shell] -*** xref:cassandra:managing/tools/nodetool/nodetool.adoc[nodetool] -*** xref:cassandra:managing/tools/sstable/index.adoc[SSTable tools] -*** xref:cassandra:managing/tools/cassandra_stress.adoc[cassandra-stress] +** xref:cassandra:managing/index.adoc[] +*** xref:cassandra:managing/configuration/index.adoc[] +**** xref:cassandra:managing/configuration/cass_yaml_file.adoc[cassandra.yaml] +**** xref:cassandra:managing/configuration/cass_rackdc_file.adoc[cassandra-rackdc.properties] +**** xref:cassandra:managing/configuration/cass_env_sh_file.adoc[cassandra-env.sh] +**** xref:cassandra:managing/configuration/cass_topo_file.adoc[cassandra-topologies.properties] +**** xref:cassandra:managing/configuration/cass_cl_archive_file.adoc[commitlog-archiving.properties] +**** xref:cassandra:managing/configuration/cass_logback_xml_file.adoc[logback.xml] +**** xref:cassandra:managing/configuration/cass_jvm_options_file.adoc[jvm-* files] +**** xref:cassandra:managing/configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units] +*** xref:cassandra:managing/operating/index.adoc[] +**** xref:cassandra:managing/operating/snitch.adoc[Snitches] +**** xref:cassandra:managing/operating/topo_changes.adoc[Topology changes] +**** xref:cassandra:managing/operating/repair.adoc[Repair] +**** xref:cassandra:managing/operating/read_repair.adoc[Read repair] +**** xref:cassandra:managing/operating/hints.adoc[Hints] +**** xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters] +**** xref:cassandra:managing/operating/compression.adoc[Compression] +**** xref:cassandra:managing/operating/cdc.adoc[Change Data Capture (CDC)] +**** xref:cassandra:managing/operating/backups.adoc[Backups] +**** xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading] +**** xref:cassandra:managing/operating/metrics.adoc[Metrics] +**** xref:cassandra:managing/operating/security.adoc[Security] +**** xref:cassandra:managing/operating/hardware.adoc[Hardware] +**** xref:cassandra:managing/operating/compaction/index.adoc[Compaction] +**** xref:cassandra:managing/operating/virtualtables.adoc[Virtual tables] +**** xref:cassandra:managing/operating/auditlogging.adoc[Audit logging] +**** xref:cassandra:managing/operating/audit_logging.adoc[Audit logging 2] +**** xref:cassandra:managing/operating/fqllogging.adoc[Full query logging] +**** xref:cassandra:managing/operating/transientreplication.adoc[Transient replication] + +*** xref:cassandra:managing/tools/index.adoc[Tools] +**** xref:cassandra:managing/tools/cqlsh.adoc[cqlsh: the CQL shell] +**** xref:cassandra:managing/tools/nodetool/nodetool.adoc[nodetool] +**** xref:cassandra:managing/tools/sstable/index.adoc[SSTable tools] +**** xref:cassandra:managing/tools/cassandra_stress.adoc[cassandra-stress] ** xref:cassandra:troubleshooting/index.adoc[Troubleshooting] *** xref:cassandra:troubleshooting/finding_nodes.adoc[Finding misbehaving nodes] @@ -105,10 +113,12 @@ *** xref:cassandra:troubleshooting/use_nodetool.adoc[Using nodetool] *** xref:cassandra:troubleshooting/use_tools.adoc[Using external tools to deep-dive] -** xref:cassandra:reference/index.adoc[Reference] -*** xref:cassandra:reference/cql-commands/commands-toc.adoc[] -**** xref:cassandra:reference/cql-commands/create-custom-index.adoc[] -**** xref:cassandra:reference/cql-commands/create-index.adoc[] -**** xref:cassandra:reference/cql-commands/drop-index.adoc[] +** xref:reference/index.adoc[] +*** xref:reference/cql-commands/alter-table.adoc[] +*** xref:reference/cql-commands/create-index.adoc[] +*** xref:reference/cql-commands/create-custom-index.adoc[] +*** xref:reference/cql-commands/create-table.adoc[] +*** xref:reference/cql-commands/drop-index.adoc[] +*** xref:reference/cql-commands/drop-table.adoc[] -** xref:cassandra:integrating/plugins/index.adoc[] +** xref:integrating/plugins/index.adoc[] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc b/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc new file mode 100644 index 0000000000..9388725356 --- /dev/null +++ b/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc @@ -0,0 +1,112 @@ += Good use of BATCH statement +:description: How to use a BATCH statement. + +Batch operations can be beneficial, as shown in the following examples. +The examples use the table `cyclist_expenses`: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=create_table] +---- + +Note that `balance` is `STATIC`. + +include::cassandra:partial$single-partition-batch-note.adoc[] + +== Single partition batch + +* The first INSERT in the `BATCH` statement sets the `balance` to zero. +The next two statements insert an `expense_id` and change the `balance` value. +All the `INSERT` and `UPDATE` statements in this batch write to the same partition, keeping the latency of the write operation low. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=batch_Vera] +---- ++ +This batching example includes conditional updates combined with using xref:reference:static.adoc[static columns]. +Recall that single partition batches are not logged. ++ +[NOTE] +==== + It would be reasonable to expect that an UPDATE to the balance could be included in this BATCH statement: ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=update_Vera] +---- ++ +However, it is important to understand that all the statements processed in a `BATCH` statement timestamp the records with the same value. +The operations may not perform in the order listed in the `BATCH` statement. +The UPDATE might be processed BEFORE the first INSERT that sets the balance value to zero, allowing the conditional to be met. +==== ++ +An acknowledgement of a batch statement is returned if the batch operation is successful. ++ +[source,results] +---- +include::cassandra:example$RESULTS/true.result[] +---- ++ +The resulting table will only have one record so far. ++ +[source,results] +---- +include::cassandra:example$RESULTS/select_initial_from_cyclist_expenses.result[] +---- + +* The balance can be adjusted separately with an UPDATE statement. +Now the `balance` will reflect that breakfast was unpaid. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=update_Vera] +---- ++ +[source,results] +---- +include::cassandra:example$RESULTS/select_after_update_from_cyclist_expenses.result[] +---- + +* The table `cyclist_expenses` stores records about each purchase by a cyclist and includes the running balance of all the cyclist's purchases. +Because the balance is static, all purchase records for a cyclist have the same running balance. +This `BATCH` statement inserts expenses for two more meals changes the balance to reflect that breakfast and dinner were unpaid. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=batch_food] +---- ++ +[source,results] +---- +include::cassandra:example$RESULTS/select_after_batch_food_from_cyclist_expenses.result[] +---- + +* Finally, the cyclist pays off all outstanding bills and the `balance` of the account goes to zero. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=batch_paid] +---- ++ +[source,results] +---- +include::cassandra:example$RESULTS/select_after_batch_paid_from_cyclist_expenses.result[] +---- ++ +Because the column is static, you can provide only the partition key when updating the data. +To update a non-static column, you would also have to provide a clustering key. +Using batched conditional updates, you can maintain a running balance. +If the balance were stored in a separate table, maintaining a running balance would not be possible because a batch having conditional updates cannot span multiple partitions. + +== Multiple partition logged batch + +* Another example is using `BATCH` to perform a multiple partition insert that involves writing the same data to two related tables that must be synchronized. +The following example modifies multiple partitions, which in general is to be avoided, but the batch only contains two statements: ++ +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=batch_multiple_partitions] +---- ++ +Another common use for this type of batch operation is updating usernames and passwords. diff --git a/doc/modules/cassandra/pages/developing/cql/counter-column.adoc b/doc/modules/cassandra/pages/developing/cql/counter-column.adoc new file mode 100644 index 0000000000..3841073793 --- /dev/null +++ b/doc/modules/cassandra/pages/developing/cql/counter-column.adoc @@ -0,0 +1,54 @@ += Using a counter +:description: A counter is a special column for storing a number that is updated by increments or decrements. + +{description} + +To load data into a counter column, or to increase or decrease the value of the counter, use the UPDATE command. +{product} rejects USING TIMESTAMP or USING TTL in the command to update a counter column. + +== Procedure + +. Create a table for the counter column. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/popular_count-table.cql[tag=create] +---- + +. Loading data into a counter column is different than other tables. +The data is updated rather than inserted. ++ +[source,language-cql] +---- +include::cassandra:example$CQL/popular_count-table.cql[tag=batch] +---- ++ +[source,language-cql] +---- +include::cassandra:example$CQL/popular_count-table.cql[tag=count] +---- + +. The `popularity` column has a value of 64. ++ +[tabs] +==== +CQL:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/popular_count-table.cql[tag=select] +---- +-- + +Result:: ++ +-- +[source,results] +---- +include::cassandra:example$RESULTS/select_all_from_popular_count.result[] +---- +-- +==== + +Additional increments or decrements changes the value of the counter column. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/developing/cql/drop-index.adoc b/doc/modules/cassandra/pages/developing/cql/drop-index.adoc index cbb18053d7..9d9986183c 100644 --- a/doc/modules/cassandra/pages/developing/cql/drop-index.adoc +++ b/doc/modules/cassandra/pages/developing/cql/drop-index.adoc @@ -1,5 +1,4 @@ = DROP INDEX -:page-aliases: cql_reference:cql_commands/cqlDropIndex.adoc :description: Removes an index from a table. Removes an existing index. diff --git a/doc/modules/cassandra/pages/developing/cql/index.adoc b/doc/modules/cassandra/pages/developing/cql/index.adoc index 73056d6dc8..a070df39a3 100644 --- a/doc/modules/cassandra/pages/developing/cql/index.adoc +++ b/doc/modules/cassandra/pages/developing/cql/index.adoc @@ -14,7 +14,7 @@ For that reason, when used in this document, these terms (tables, rows and colum * xref:developing/cql/ddl.adoc[Data definition language] * xref:developing/cql/dml.adoc[Data manipulation language] * xref:developing/cql/operators.adoc[Operators] -* xref:developing/cql/indexing/indexing-concepts.adoc[] +* xref:cql/indexing/indexing-concepts.adoc[Indexing] * xref:developing/cql/mvs.adoc[Materialized views] * xref:developing/cql/functions.adoc[Functions] * xref:developing/cql/json.adoc[JSON] diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/indexing-concepts.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/indexing-concepts.adoc index 40d4e3902d..86862035a1 100644 --- a/doc/modules/cassandra/pages/developing/cql/indexing/indexing-concepts.adoc +++ b/doc/modules/cassandra/pages/developing/cql/indexing/indexing-concepts.adoc @@ -1,5 +1,4 @@ = Indexing concepts -:page-aliases: developing:cql/indexes.adoc :description: How to query data from tables using indexes. :y: ✓ :n: 𐄂 diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc index e815caac81..12a213600c 100644 --- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc +++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc @@ -99,10 +99,11 @@ See xref:cassandra:managing/tools/nodetool/toolsNodetool.html[nodetool]. * Monitored via a combination of nodetool, CQL virtual tables, system metrics, and JMX. See xref:cassandra:developing/cql/indexing/sai/monitoring.adoc[Monitor SAI indexes]. -[[saiLimitsFaq]] -== Are there limits on the number of SAI indexes per table, and total per cluster? +// LLP: There are currently no limits placed on SAI indexes in Apache C*. +// [[saiLimitsFaq]] +// == Are there limits on the number of SAI indexes per table, and total per cluster? -include::cassandra:partial$sai/config-limits.adoc[] +// include::cassandra:partial$sai/config-limits.adoc[] == On which column in a database table can I base an SAI index? diff --git a/doc/modules/cassandra/pages/developing/cql/types.adoc b/doc/modules/cassandra/pages/developing/cql/types.adoc index e4db5c64d2..db46d4c6ac 100644 --- a/doc/modules/cassandra/pages/developing/cql/types.adoc +++ b/doc/modules/cassandra/pages/developing/cql/types.adoc @@ -51,6 +51,7 @@ generally used as a “conflict-free” timestamp. Also see `timeuuid-functions` | `uuid` | `uuid` | A https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] (of any version) | `varchar` | `string` | UTF8 encoded string | `varint` | `integer` | Arbitrary-precision integer +| `vector` | `float` | A fixed length non-null, flattened array of float values https://issues.apache.org/jira/browse/CASSANDRA-18504[CASSANDRA-18504] added this data type to {cass-50}. |=== === Counters diff --git a/doc/modules/cassandra/pages/getting-started/vector-search-quickstart.adoc b/doc/modules/cassandra/pages/getting-started/vector-search-quickstart.adoc new file mode 100644 index 0000000000..d0b6c6b0bf --- /dev/null +++ b/doc/modules/cassandra/pages/getting-started/vector-search-quickstart.adoc @@ -0,0 +1,32 @@ += Vector Search Quickstart + +To enable your machine learning model, Vector Search uses data to be compared by similarity within a database, even if it is not explicitly defined by a connection. A vector is an array of floating point type that represents a specific object or entity. + +The foundation of Vector Search lies within the embeddings, which are compact representations of text as vectors of floating-point numbers. These embeddings are generated by feeding the text through an API, which uses a neural network to transform the input into a fixed-length vector. Embeddings capture the semantic meaning of the text, providing a more nuanced understanding than traditional term-based approaches. The vector representation allows for input that is substantially similar to produce output vectors that are geometrically close; inputs that are not similar are geometrically further apart. + +To enable Vector Search, a new `vector` data type is available in your {cassandra} database with Vector Search. + +== Prerequisites + +There are no prerequisite tasks. + +In general, to use Vector Search with Apache Cassandra, you'll follow these instructions: + +The embeddings were randomly generated in this quickstart. +Generally, you would run both your source documents/contents through an embeddings generator, as well as the query you were asking to match. +This example is simply to show the mechanics of how to use CQL to create vector search data objects. + +include::../vector-search/_create-vector-keyspace-cycling.adoc[leveloffset=+1] + +include::../vector-search/_use-vector-keyspace-cycling.adoc[leveloffset=+1] + +include::../vector-search/_create-vector-table-cycling.adoc[leveloffset=+1] + +include::../vector-search/_create-vector-index-cycling.adoc[leveloffset=+1] + +include::../vector-search/_load-vector-data-cycling.adoc[leveloffset=+1] + +include::../vector-search/_query-vector-data-cycling.adoc[leveloffset=+1] + +With the code examples, you have a working example of our Vector Search. +Load your own data and use the search function. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/managing/configuration/index.adoc b/doc/modules/cassandra/pages/managing/configuration/index.adoc index 336ff5dfe9..6c40a8ce86 100644 --- a/doc/modules/cassandra/pages/managing/configuration/index.adoc +++ b/doc/modules/cassandra/pages/managing/configuration/index.adoc @@ -1,4 +1,5 @@ = Configuring Cassandra +:navtitle: Configuring This section describes how to configure Apache Cassandra. diff --git a/doc/modules/cassandra/pages/managing/operating/index.adoc b/doc/modules/cassandra/pages/managing/operating/index.adoc index c22b33f731..b7491cf652 100644 --- a/doc/modules/cassandra/pages/managing/operating/index.adoc +++ b/doc/modules/cassandra/pages/managing/operating/index.adoc @@ -1,4 +1,5 @@ = Operating Cassandra +:navtitle: Operating * xref:cassandra:managing/operating/hardware.adoc[Hardware] * xref:cassandra:managing/operating/security.adoc[Security] diff --git a/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc b/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc index b4ea261219..807d1ab7f4 100644 --- a/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc +++ b/doc/modules/cassandra/pages/managing/operating/virtualtables.adoc @@ -248,7 +248,6 @@ counters | 26214400 | 0 | 0 | NaN | (4 rows) ---- -======= === CIDR filtering metrics Virtual Tables The `cidr_filtering_metrics_counts` virtual table lists counts metrics specific to CIDR filtering. A query on `cidr_filtering_metrics_counts` virtual table lists metrics similar to below. diff --git a/doc/modules/cassandra/pages/overview/terminology.adoc b/doc/modules/cassandra/pages/overview/terminology.adoc index 8200056e18..a39ba0f3fb 100644 --- a/doc/modules/cassandra/pages/overview/terminology.adoc +++ b/doc/modules/cassandra/pages/overview/terminology.adoc @@ -1,23 +1,5 @@ = Terminology -a | b | xref:#c[c] | d | e | f - -[[c]] -cluster:: -A ring of nodes that holds a database. - -node:: -A machine that holds Cassandra replicas. -Each node holds a portion of the whole database. - -replica:: -A copy of a portion of the whole database. Each node holds some replicas. - -replication:: -The process of creating replicas across nodes in a cluster. - -replication factor (RF):: -A scalar value that sets the number of replicas of each partition in a cluster. -For example, and RF=3 means that three nodes hold a replica of each partition. +See the https://cassandra.apache.org/_/glossary.html[glossary]. diff --git a/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc b/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc new file mode 100644 index 0000000000..2c6071c529 --- /dev/null +++ b/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc @@ -0,0 +1,353 @@ += ALTER TABLE +:description: Modifies the columns and properties of a table. + +{description} + +Add new columns, drop existing columns, renames columns, and modify table properties. +The command returns no results. + +[NOTE] +==== +`ALTER COLUMNFAMILY` is deprecated. +==== + +*See also:* xref:cassandra:reference/cql-commands/create-table.adoc[CREATE TABLE], +xref:cassandra:reference/cql-commands/drop-table.adoc[DROP TABLE], +xref:cassandra:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] for Storage-Attached Indexes (SAI), xref:cassandra:reference/cql-commands/create-index.adoc[CREATE INDEX] for secondary indexes (2i) + +== Syntax + +BNF definition: + +[source,bnf] +---- +include::cassandra:example$BNF/alter_table.bnf[] +---- + +// tag::syntax[] +---- +ALTER TABLE [.] + [ ADD ( | ) [ , ... ] ] + [ DROP [ , ... ] ] + [ [ RENAME TO ] ] + [ WITH [ , ... ] ]; +---- +// end::syntax[] + +.Syntax legend +[%collapsible] +==== +include::cassandra:partial$cql-syntax-legend.adoc[] +==== + +== Required parameters + +*table_name*:: +Name of the table to alter. + +*column_name*:: +Name of the column to alter, drop, or add. + +// Column Definitions + +include::cassandra:partial$table-column-definitions.adoc[] + +== Optional parameters + +*keyspace_name*:: +Name of the keyspace that contains the table to alter. +If no name is specified, the current keyspace is used. + +*ADD ( | )* :: +Add one or more columns and set the column data types. +Specify the column names followed by the data types. +The column value is automatically set to null. +To add multiple columns, use a comma separated list of columns placed inside parentheses. ++ +---- + [ , ] +[ [ , ... ] +---- +*Restriction:* Adding columns to a primary key is not supported after a table has been created. + +*DROP ( | )* :: +Drop one or more columns. +The values contained in the row are also dropped and not recoverable. +To drop multiple columns, use a comma separated list of columns placed inside parentheses. + +*RENAME TO * :: +Changes the name of a primary key column and preserves the existing values. ++ +*Restriction:* Not supported on materialized view base-tables, or tables with secondary indexes. + +// Table Options + +include::cassandra:partial$table-properties.adoc[] + +*table_properties* :: +You can modify an existing table's properties. +Some properties are single options that are set to a value: ++ +[source,no-highlight] +---- + = [ AND ... ] +---- ++ +For example, `speculative_retry = '10ms'`. +Enclose the value for a string property in single quotation marks. ++ +Other table properties are set using a JSON map: `+option_name = { : [ , ... ] }+` ++ +See xref:reference:cql-commands/create-table.adoc#table_options[table_options] for more details. + +== Usage notes + +*Restrictions:* + +* Can only rename clustering columns in the primary key. +* Cannot change the data type of a column. +* For a table that has a materialized view, cannot drop a column from the table even if the column is not used in the materialized view. +* Cannot rename or drop columns that have dependent secondary indexes. +* Do not add a column with the same name as an existing column but with a different data type. +It will prevent commit log replays and corrupt existing SSTables with old data. + +== Examples + +This section uses the xref:cassandra:cyclist_races-table.adoc[cyclist_races] table. + +=== Adding a column + +To add a column, use the ADD instruction: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_races-table.cql[tag=add] +---- + +To add a column of a collection type: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_races-table.cql[tag=list] +---- + +This operation does not validate the existing data. + +*Restriction:* You cannot use the `ADD` instruction to add: + +* A column with the same name as an existing column +* A static column if the table has no clustering columns. + +=== Dropping a column + +To remove a column from the table, use the DROP instruction: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_races-table.cql[tag=drop] +---- + +`DROP` removes the column from the table definition. +The column becomes unavailable for queries immediately after it is dropped. +The database drops the column data during the next compaction. + +*Restriction:* + +* If you drop a column then re-add it, {product} +does not restore the values written before the column was dropped. +* Do not re-add a dropped column that contained timestamps generated by a client; +you can re-add columns with timestamps generated by the xref:developing/querying/use-write-time.adoc[write time] facility. + +=== Renaming a column + +To rename a column in the xref:cassandra:race_times-table.adoc[race_times] table: + +[source,language-cql] +---- +include::cassandra:example$CQL/race_times-table.cql[tag=rename] +---- + +*Restriction:* The following restrictions apply to `RENAME`: + +* You can only rename clustering columns, which are part of the primary key. +* You cannot rename the partition key because the partition key determines the data storage location on a node. +If a different partition name is required, the table must be recreated and the data migrated. ++ +[NOTE] +==== +There are many restrictions when using `RENAME` because SSTables are immutable. +To change the state of the data on disk, everything must be rewritten. +==== + + +* You can index a renamed column. +* You cannot rename a column if an index has been created on it. +* You cannot rename a static column. + +=== Modifying table properties + +To change an existing table's properties, use `ALTER TABLE` and `WITH`. +You can specify a: + +* Single property name and value. +* Property map to set the names and values, as shown in the xref:cql-commands/alter-table.adoc#alter-compression[next section on compression and compaction]. + +For example, to add a comment to the xref:cassandra:cyclist_base-table.adoc[cyclist_base] table using WITH: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_base-table.cql[tag=alt] +---- + +Enclose a text property value in single quotation marks. + +[[alter-compression]] +=== Modifying compression and compaction + +Use a property map to alter the xref:cassandra:comments-table.adoc[comments] table's compression or compaction setting: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_base-table.cql[tag=alt] +---- + +Enclose the name of each key in single quotes. +If the value is a string, enclose the string in quotes as well. + +[CAUTION] +==== +If you change the compaction strategy of a table with existing data, the database rewrites all existing SSTables using the new strategy. +This can take hours, which can be a major problem for a production system. +For strategies to minimize this disruption, see http://blog.alteroot.org/articles/2015-04-20/change-cassandra-compaction-strategy-on-production-cluster.html[How to change the compaction strategy on a production cluster] and https://groups.google.com/forum/#!topic/nosql-databases/iYPoy-06Qvs[Impact of Changing Compaction Strategy]. +==== + +=== Changing caching + +Set the number of rows per partition to store in the row cache for the xref:cassandra:comments-table.adoc[comments] table to 10 rows: + +[source,language-cql] +---- +include::cassandra:example$CQL/comments-table.cql[tag=cache] +---- + +=== Change the speculative retries + +Modify the `cyclist_base` table to 95th percentile for speculative retry: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_base-table.cql[tag=specr] +---- + +Modify the `cyclist_base` table to use 10 milliseconds for speculative retry: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_base-table.cql[tag=milli] +---- + +=== Enabling and disabling background compaction + +The following example sets the `enabled` property to `false` to disable background compaction: + +[source,language-cql] +---- +include::cassandra:example$CQL/comments-table.cql[tag=nocompact] +---- + +[WARNING] +==== +Disabling background compaction can be harmful: without it, the database does not regain disk space, and could allow https://docs.datastax.com/en/glossary/doc/glossary/gloss_zombie.html[zombies] to propagate. +Although compaction uses I/O, it is better to leave it enabled in most cases. +==== + +=== Reading extended compaction logs + +Set the `log_all` subproperty to `true` to collect in-depth information about compaction activity on a node in a dedicated log file. + +[IMPORTANT] +==== +If you enable extended compaction logging for any table on any node, it is enabled for all tables on all nodes in the cluster. +==== + +When extended compaction is enabled, the database creates a file named compaction-%d.log (where `%d` is a sequential number) in home/logs. + +The compaction logging service logs detailed information about the following types of compaction events: + +* `type:enable` ++ +Lists SSTables that have been flushed previously. ++ +[source,no-highlight] +---- +{"type":"enable","keyspace":"test","table":"t","time":1470071098866,"strategies": + [ {"strategyId":"0","type":"LeveledCompactionStrategy","tables":[],"repaired":true,"folders": + ["/home/carl/oss/cassandra/bin/../data/data"]}, + {"strategyId":"1","type":"LeveledCompactionStrategy","tables":[],"repaired":false,"folders": + ["/home/carl/oss/cassandra/bin/../data/data"] + } + ] +} +---- + +* `type: flush` ++ +Logs a flush event from a memtable to an SSTable on disk, including the CompactionStrategy for each table. ++ +---- +{"type":"flush","keyspace":"test","table":"t","time":1470083335639,"tables": + [ {"strategyId":"1","table": + {"generation":1,"version":"mb","size":106846362,"details": + {"level":0,"min_token":"-9221834874718566760","max_token":"9221396997139245178"} + } + } + ] +} +---- + +* `type: compaction` ++ +Logs a compaction event. ++ +---- +{"type":"compaction","keyspace":"test","table":"t","time":1470083660267, + "start":"1470083660188","end":"1470083660267","input": + [ {"strategyId":"1","table": + {"generation":1372,"version":"mb","size":1064979,"details": + {"level":1,"min_token":"7199305267944662291","max_token":"7323434447996777057"} + } + } + ],"output": + [ {"strategyId":"1","table": + {"generation":1404,"version":"mb","size":1064306,"details": + {"level":2,"min_token":"7199305267944662291","max_token":"7323434447996777057"} + } + } + ] +} +---- + +* `type: pending` ++ +Lists the number of pending tasks for a compaction strategy. ++ +---- +{"type":"pending","keyspace":"test","table":"t", + "time":1470083447967,"strategyId":"1","pending":200} +---- + +=== Reviewing the table definition + +Use `DESCRIBE` or `DESC` to view the table definition. + +[source,language-cql] +---- +include::cassandra:example$CQLSH/comments-desc_table.cqlsh[] +---- + +The table details including the column names are returned. + +[source,results] +---- +include::cassandra:example$RESULTS/comments-desc_table.result[] +---- diff --git a/doc/modules/cassandra/pages/reference/index.adoc b/doc/modules/cassandra/pages/reference/index.adoc index 28e17467a8..8be1146452 100644 --- a/doc/modules/cassandra/pages/reference/index.adoc +++ b/doc/modules/cassandra/pages/reference/index.adoc @@ -1,4 +1,7 @@ = Reference -* xref:cassandra:reference/cql-commands/commands-toc.adoc[CQL commands] -* xref:cassandra:reference/java17.adoc[Java 17 usage] \ No newline at end of file +* xref:reference/cql-commands/commands-toc.adoc[CQL commands] +* xref:reference/java17.adoc[Java 17] +* xref:reference/counter-type.adoc[Counter type] +* xref:reference/static.adoc[Static columns] +* xref:reference/sai-virtual-table-indexes.adoc[SAI virtual table] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/reference/static.adoc b/doc/modules/cassandra/pages/reference/static.adoc new file mode 100644 index 0000000000..afa193cdbb --- /dev/null +++ b/doc/modules/cassandra/pages/reference/static.adoc @@ -0,0 +1,55 @@ += Creating columns with a single value (static column) +:description: In a table that uses clustering columns, non-clustering columns can be declared static in the table definition. + +Static column values are shared among the rows in the partition. +In a table that uses https://cassandra.apache.org/_/glossary.html#clustering-column[clustering columns], non-clustering columns can be declared static in the table definition. +https://cassandra.apache.org/_/glossary.html#static-column[Static columns] are only static within a given partition. + +In the following example, the `flag` column is static: + +[source,language-cql] +---- +include::cassandra:example$CQL/country_flag-table.cql[tag=staticColumn] +---- + +[source,language-cql] +---- +include::cassandra:example$CQL/country_flag-table.cql[tag=insert] +---- + +[tabs] +==== +CQL:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/country_flag-table.cql[tag=select] +---- +-- + +Result:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$RESULTS/select_initial_from_country_flag.result[] +---- +-- +==== + + +[IMPORTANT] +==== +The following restrictions apply: + +* A table that does not define any clustering columns cannot have a static column. +The table that does not have clustering columns has a one-row partition in which every column is inherently static. +* A column designated to be the partition key cannot be static. +==== + +You can do xref:developing/batch/batch-good-example.adoc[batch conditional updates to a static column]. + +Use the `DISTINCT` keyword to select static columns. +In this case, the database retrieves only the beginning (static column) of the partition. + diff --git a/doc/modules/cassandra/pages/reference/vector-data-type.adoc b/doc/modules/cassandra/pages/reference/vector-data-type.adoc new file mode 100644 index 0000000000..4e360fb547 --- /dev/null +++ b/doc/modules/cassandra/pages/reference/vector-data-type.adoc @@ -0,0 +1,53 @@ += Vector CQL data type + +A new CQL data type, a `VECTOR` is required for Vector Search. +https://issues.apache.org/jira/browse/CASSANDRA-18504 added this data type to {cass-50}. +The new type `VECTOR` has the following properties: + +* fixed length array +* elements may not be null +* flatten array (aka multi-cell = false) + + +Create a keyspace: + +[source,cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-keyspace-cycling] +---- + +Use the keyspace: + +[source,cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=use-vs-keyspace-cycling] +---- + +This data type can be used in a `CREATE TABLE` statement: + +[source, cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-table] +---- +<1> Create a 5-dimensional embedding + +Index the vector column: + +[source, cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-index] +---- + +Insert data: + +[source, cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=insert-vector-data-cycling] +---- + +Select data: + +[source, cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=select-vector-data-cycling] +---- \ No newline at end of file diff --git a/doc/modules/cassandra/pages/tooling/cassandra-stress.adoc b/doc/modules/cassandra/pages/tooling/cassandra-stress.adoc index c367a74dbf..e363c65bbd 100644 --- a/doc/modules/cassandra/pages/tooling/cassandra-stress.adoc +++ b/doc/modules/cassandra/pages/tooling/cassandra-stress.adoc @@ -1,5 +1,4 @@ = Cassandra Stress -:page-aliases: cassandra-stress.adoc [IMPORTANT] ==== diff --git a/doc/modules/cassandra/pages/tooling/hash-password.adoc b/doc/modules/cassandra/pages/tooling/hash-password.adoc index ad9db02591..54b3a3c86b 100644 --- a/doc/modules/cassandra/pages/tooling/hash-password.adoc +++ b/doc/modules/cassandra/pages/tooling/hash-password.adoc @@ -1,6 +1,4 @@ = Hash password tool -:page-aliases: hash_password.adoc - The `hash_password` tool is used to get the jBcrypt hash of a password. This hash can be used in CREATE/ALTER ROLE/USER statements for improved security. diff --git a/doc/modules/cassandra/pages/vector-search/.sai-quickstart.adoc b/doc/modules/cassandra/pages/vector-search/.sai-quickstart.adoc new file mode 100644 index 0000000000..f89aeee6b1 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/.sai-quickstart.adoc @@ -0,0 +1,198 @@ += SAI Quickstart +:description: Follow the steps to get started quickly with Storage-Attached Indexing (SAI). + +To get started with Storage-Attached Indexing (SAI), we'll do the following steps: + +* Create a keyspace. +* Create a table. +* Create an xref:reference:cql-commands/create-custom-index.adoc[index using SAI]. +* Add data. +* Create and run a query using SAI. + +The examples in this quickstart topic show SAI indexes with non-partition key columns. +// See the next topic, xref:developing/cql/indexing/sai/sai-query.adoc[Examine SAI column index and query rules], for examples that include indexing based on a single column from a composite partition key. + +== Create a keyspace + +In `cqlsh`, define the `cycling` xref:reference/cql-commands/create-keyspace.adoc[keyspace] to try the commands in a test environment: + +[source,language-cql] +---- +include::cassandra:example$CQL/0_create_keyspace.cql[tag=cyclingks] +---- + +== Create a database table + +Using `cqlsh` or the CQL Console, create a `cyclist_semi_pro` database xref:reference:cql-commands/create-table.adoc[table] in the `cycling` keyspace or the keyspace name of your choice: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro-table.cql[tag=createCyclistSemiProTable] +---- + +== Create SAI indexes on the database table + +To test a non-trivial query, you'll need some SAI indexes. +Use xref:reference:cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] commands to create SAI indexes on a few non-primary-key columns in the `cyclist_semi_pro` table: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_sai_indices.cql[tag=createQuickStartIndices] +---- + +Let's take a look at the description of the table and its indexes: +[tabs] +==== +Query:: ++ +-- +[source,language-cql] +---- +DESCRIBE TABLE cycling.cyclist_semi_pro; +---- +-- + +Result:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_describe.results[tag=cyclistSemiProDescribeResults] +---- +-- +==== + +== Add data to your table + +Use CQLSH `INSERT` commands to add some data to the `cyclist_semi_pro` database table: + +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_inserts.cql[tag=createQuickStartIndices] +---- + +Adding data in this quickstart topic shows simple `INSERT` commands. +To load databases with many rows, consider using https://docs.datastax.com/en/dsbulk/doc/index.html[DataStax Bulk Loader for Apache Cassandra]. + +== Try out CQL queries + +Use the CQLSH `SELECT` command to submit queries. + +[TIP] +==== +The supported query operators for tables with SAI indexes: +include::cassandra:partial$sai/supported-query-operators-list.adoc[] +include::cassandra:partial$sai/notSupportedOperators.adoc[] +==== + +Find a specific semi-pro cyclist: + +[tabs] +==== +Query:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_specific_rider.cql[tag=cyclistSelectSpecificRider] +---- +-- + +Result:: ++ +-- +[source, console] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_specific_rider.results[tag=cyclistSelectSpecificRiderResults] +---- +-- +==== + +Find semi-pro cyclists whose age is less than or equal to 23: + +[tabs] +==== +Query:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_age.cql[tag=cyclistSelectAge] +---- +-- + +Result:: ++ +-- +[source,console] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_age.results[tag=cyclistSelectAgeResults] +---- +-- +==== + +Find semi-pro cyclists from Great Britain: + +[tabs] +==== +Query:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_country.cql[tag=cyclistSelectCountry] +---- +-- + +Result:: ++ +-- +[source,console] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_country.results[tag=cyclistSelectCountryResults] +---- +-- +==== + +Find semi-pro cyclists who registered between a given date range: + +[tabs] +==== +Query:: ++ +-- +[source,language-cql] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_date_range.cql[tag=cyclistSelectDateRange] +---- +-- + +Result:: ++ +-- +[source,results] +---- +include::cassandra:example$CQL/cyclist_semi_pro_select_date_range.results[tag=cyclistSelectDateRangeResults] +---- +-- +==== + +[TIP] +==== +For query examples with `CONTAINS` clauses that take advantage of SAI collection maps, lists, and sets, be sure to see xref:developing/cql/indexing/sai/collections.adoc#saiUsingCollectionsMapExamples[SAI collection map examples with keys, values, and entries] and xref:developing/cql/indexing/sai/collections.adoc#saiUsingCollectionsListAndSetExamples[SAI collection examples with list and set types]. +==== + +== Removing an SAI index + +To remove an SAI index, use `DROP INDEX`. + +Example: + +[source,language-cql] +---- +DROP INDEX IF EXISTS cycling.age_sai_idx; +---- + +== What's next? + +xref:developing/cql/indexing/sai/sai-query.adoc[Using SAI] examines column index and query rules, plus collection maps. diff --git a/doc/modules/cassandra/pages/vector-search/_create-vector-index-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_create-vector-index-cycling.adoc new file mode 100644 index 0000000000..e475b3c49b --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_create-vector-index-cycling.adoc @@ -0,0 +1,22 @@ += Create vector index + +Create the custom index with Storage Attached Indexing (SAI): + +[source,cql] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-index] +---- + +For more about SAI, see the xref:../developing/cql/indexing/sai/sai-overview.adoc[Storage Attached Indexing] documentation. + +[IMPORTANT] +==== +The index can be created with options that define the similarity function: + +[source] +---- +include::cassandra:example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-index-with-option] +---- + +Valid values for the `similarity_function` are `DOT_PRODUCT`, `COSINE`, or `EUCLIDEAN`. +==== \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/_create-vector-keyspace-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_create-vector-keyspace-cycling.adoc new file mode 100644 index 0000000000..df2fd77138 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_create-vector-keyspace-cycling.adoc @@ -0,0 +1,9 @@ += Create vector keyspace + +Create the keyspace you want to use for your Vector Search table. +This example uses `cycling` as the `keyspace name`: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-keyspace-cycling] +---- \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/_create-vector-table-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_create-vector-table-cycling.adoc new file mode 100644 index 0000000000..d061251690 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_create-vector-table-cycling.adoc @@ -0,0 +1,15 @@ += Create vector table + +Create a new table in your keyspace, including the `comments_vector` column for vector. The code below creates a vector with five values: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=create-vs-table] +---- + +Optionally, you can alter an existing table to add a vector column: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=alter-vs-table] +---- \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/_load-vector-data-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_load-vector-data-cycling.adoc new file mode 100644 index 0000000000..e94fc3303a --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_load-vector-data-cycling.adoc @@ -0,0 +1,8 @@ += Load vector data into your database + +Insert data into the table using the new type: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=insert-vector-data-cycling] +---- \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/_query-vector-data-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_query-vector-data-cycling.adoc new file mode 100644 index 0000000000..e6892e2c12 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_query-vector-data-cycling.adoc @@ -0,0 +1,33 @@ += Query vector data with CQL + +To query data using Vector Search, use a `SELECT` query: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=select-vector-data-cycling] +---- + +To obtain the distance calculation of the best scoring node closest to the query data as part of the results, use a `SELECT` query: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=select-vector-data-similarity-cycling] +---- + +The supported functions for this type of query are: + +* similarity_dot_product +* similarity_cosine +* similarity_euclidean + +with the parameters of (, ). +Both parameters represent vectors. + +[NOTE] +==== +* The limit must be 1,000 or fewer. +* Vector Search utilizes Approximate Nearest Neighbor (ANN) that in most cases yields results almost as good as the exact match. The scaling is superior to Exact Nearest Neighbor (KNN). + +* Least-similar searches are not supported. +* Vector Search works optimally on tables with no overwrites or deletions of the `item_vector` column. For an `item_vector` column with changes, expect slower search results. +==== \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/_use-vector-keyspace-cycling.adoc b/doc/modules/cassandra/pages/vector-search/_use-vector-keyspace-cycling.adoc new file mode 100644 index 0000000000..a6b4aa31d0 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/_use-vector-keyspace-cycling.adoc @@ -0,0 +1,8 @@ += Use vector keyspace + +Select the keyspace you want to use for your Vector Search table. This example uses `cycling` as the `keyspace name`: + +[source,cql] +---- +include::example$CQL/vector-search/vector-search-cycling.cql[tag=use-vs-keyspace-cycling] +---- \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/concepts.adoc b/doc/modules/cassandra/pages/vector-search/concepts.adoc new file mode 100644 index 0000000000..3a6196eaba --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/concepts.adoc @@ -0,0 +1,53 @@ += Vector search concepts +:navtitle: Concepts +:description: Vector Search concepts - How it works, how to use it, and more. + +Vector Search is a new feature added to {cass-50}. +It is a powerful technique for finding relevant content within large datasets and is particularly useful for AI applications. +Vector Search also makes use of xref:cassandra:developing/cql/indexing/sai/overview.adoc[Storage-Attached Indexes(SAI)], leveraging the new modularity of the latter feature. +Vector Search is the first instance of validating the extensibility of SAI. + +Data stored in a database is useful, but the context of that data is critical to applications. +https://cassandra.apache.org/_/glossary.html#machine-learning-ml[Machine learning] in applications allows users to get product recommendations, match similar images, and a host of other capabilities. +A machine learning model is a program that can find patterns or make decisions from a previously unseen dataset. +To power a machine learning model in an application, Vector Search does similarity comparison of stored database data to discover connections in data that may not be explicitly defined. + +One key to doing similarity comparisons in a machine learning model is the ability to store https://cassandra.apache.org/_/glossary.html#embeddings[embeddings] vectors, arrays of floating-point numbers that represent the similarity of specific objects or entities. +Vector Search brings that functionality to the high availability {product} database. + +.Want to get started quickly? Here's how! +**** +xref:cassandra:getting-started/vector-search-quickstart.adoc[Vector Search Quickstart] +**** + +The foundation of Vector Search lies within the embeddings, compact representations of text or images as high-dimensional vectors of floating-point numbers. +For text processing, embeddings are generated by feeding the text to a machine learning model. +These models generally use a neural network to transform the input into a fixed-length vector. +When words are represented as high-dimensional vectors, the aim is to arrange the vectors so that similar words end up closer together in the vector space and dissimilar word end up further apart. +Creating the vectors in this manner is referred to as preserving semantic or structural similarity. +Embeddings capture the semantic meaning of the text, which in turn, allow queries to rely on a more nuanced understanding of the text as opposed to traditional term-based approaches. + +Large Language Models (LLMs) generate contextual embeddings for the data, and optimize embeddings for queries. +Trained embeddings like those produced by LLMs can be used in Natural Language Processing (NLP) tasks such as text classification, sentiment analysis, and machine translation. + +== Storage Attached Indexing (SAI) + +SAI is a required feature providing unparalleled I/O throughput for databases to use Vector Search as well as other search indexing. +SAI is a highly-scalable and globally-distributed index that adds column-level indexes to any vector data type column. + +SAI provides the most indexing functionality available - indexing both queries and content (large inputs include such items as documents, words, and images) to capture semantics. + + +For more about SAI, see the xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[Storage Attached Index] documentation. + +[NOTE] +==== +You cannot change index settings without dropping and rebuilding the index. +==== + +It is better to create the index and then load the data. +This method avoids the concurrent building of the index as data loads. + +== New Vector CQL data type + +A new xref:developing/cql/types.adoc[vector data type] is added to CQL to support Vector Search. It is designed to save and retrieve embeddings vectors. diff --git a/doc/modules/cassandra/pages/vector-search/data-modeling.adoc b/doc/modules/cassandra/pages/vector-search/data-modeling.adoc new file mode 100644 index 0000000000..0150fd4bf3 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/data-modeling.adoc @@ -0,0 +1,132 @@ += Data Modeling + +As you develop AI and Machine Learning (ML) applications using Vector Search, here are some data modeling considerations. +These factors help effectively leverage vector search to produce accurate and efficient search responses within your application. + +== Data representation + +Vector search relies on representing data points as high-dimensional vectors. +The choice of vector representation depends on the nature of the data. + +For data that consists of text documents, techniques like word embeddings (e.g., https://towardsdatascience.com/word2vec-explained-49c52b4ccb71[Word2Vec]) or document embeddings (e.g., https://en.wikipedia.org/wiki/Word2vec#doc2vec[Doc2Vec]) can be used to convert text into vectors. +More complex models can also be used to generate embeddings using Large Language Models (LLMs) like https://openai.com/gpt-4[OpenAI GPT-4] or https://ai.meta.com/llama/[Meta LLaMA 2]. +Word2Vec is a relatively simple model that uses a shallow neural network to learn embeddings for words based on their context. +The key concept is that Word2Vec generates a single fixed vector for each word, regardless of the context in which the word is used. +LLMs are much more complex models that use deep neural networks, specifically transformer architectures, to learn embeddings for words based on their context. +Unlike Word2Vec, these models generate contextual embeddings, meaning the same word can have different embeddings depending on the context in which it is used. + +Images can be represented using deep learning techniques like https://medium.com/@singole/an-extensive-guide-to-convolution-neural-network-2023-84872b16bd78[convolutional neural networks (CNNs)] or pre-trained models such as https://openai.com/research/clip[Contrastive Language Image Pre-training (CLIP)]. +Select a vector representation that captures the essential features of the data. + +== Dataset dimensions + +A vector search only works when the vectors have the same dimensions, because vector operations like https://www.mathsisfun.com/algebra/vectors-dot-product.html[dot product] and https://www.learndatasci.com/glossary/cosine-similarity/[cosine similarity] require vectors to have the same number of dimensions. + +For vector search, it is crucial that all embeddings are created in the same vector space. +This means that the embeddings should follow the same principles and rules to enable proper comparison and analysis. +Using the same embedding library guarantees this compatibility because the library consistently transforms data into vectors in a specific, defined way. +For example, comparing Word2Vec embeddings with BERT (an LLM) embeddings could be problematic because these models have different architectures and create embeddings in fundamentally different ways. + +== Preprocessing embeddings vectors + +Normalizing is about scaling the data so it has a length of one. +This is typically done by dividing each element in a vector by the vector's length. + +Standardizing is about shifting (subtracting the mean) and scaling (dividing by the standard deviation) the data so it has a mean of zero and a standard deviation of one. + +It is important to note that standardizing and normalizing in the context of embedding vectors are not the same. +The correct preprocessing method (standardizing, normalizing, or even something else) depends on the specific characteristics of your data and what you are trying to achieve with your machine learning model. +Preprocessing steps may involve cleaning and tokenizing text, resizing and normalizing images, or handling missing values. + +=== Normalizing embedding vectors + +Normalizing embedding vectors is a process that ensures every embedding vector in your vector space has a length (or norm) of one. +This is done by dividing each element of the vector by the vector's length (also known as its `Euclidean` norm or `L2` norm). + +For example, look at the embedding vectors from the xref:getting-started/vector-search-quickstart.adoc[Vector Search Quickstart] and their normalized counterparts, where a consistent length has been used for all the vectors: + +[tabs] +==== +Original:: ++ +-- +[source,console] +---- +include::cassandra:example$TEXT/original-embedding-vectors.txt[] +---- +-- + +Normalized:: ++ +-- +[source, console] +---- +include::cassandra:example$TEXT/normalized-embedding-vectors.txt[] +---- +-- +==== + +The primary reason you would normalize vectors when working with embeddings is that it makes comparisons between vectors more meaningful. +By normalizing, you ensure that comparisons are not affected by the scale of the vectors, and are solely based on their direction. +This is particularly useful to calculate the cosine similarity between vectors, where the focus is on the angle between vectors (directional relationship), not their magnitude. + +Normalizing embedding vectors is a way of standardizing your high-dimensional data so that comparisons between different vectors are more meaningful and less affected by the scale of the original vectors. +Since `dot product` and cosine are equivalent for normalized vectors, but the `dot product` algorithm is 50% faster, it is recommended that developers use `dot product` for the similarity function. + +However, if embeddings are NOT normalized, then `dot product` silently returns meaningless query results. +Therefore, `dot product` is not set as the default similarity function in Vector Search. + +When you use OpenAI, PaLM, or Simsce to generate your embeddings, they are normalized by default. +If you use a different library, you want to normalize your vectors and set the similarity function to `dot product`. +See how to set the similarity function in the xref:getting-started/vector-search-quickstart.adoc#_create_vector_index[Vector Search quickstart]. + +Normalization is not required for all vector search examples. + +=== Standardizing embedding vectors + +Standardizing embedding vectors typically refers to a process similar to that used in statistics where data is standardized to have a mean of zero and a standard deviation of one. +The goal of standardizing is to transform the embedding vectors so they have properties of a standard normal Gaussian distribution. + +If you are using a machine learning model that uses distances between points (like nearest neighbors or any model that uses Euclidean distance or cosine similarity), standardizing can ensure that all features contribute equally to the distance calculations. +Without standardization, features on larger scales can dominate the distance calculations. + +In the context of neural networks, for example, having input values that are on a similar scale can help the network learn more effectively, because it ensures that no particular feature dominates the learning process simply because of its scale. + +== Indexing and Storage + +SAI indexing and storage mechanisms are tailored for large datasets like vector search. +Currently, SAI uses https://github.com/jbellis/jvector[JVector], an algorithm for Approximate Nearest Neighbor (ANN) search and close cousin to Hierarchical Navigable Small World (HNSW). + +The goal of ANN search algorithms like JVector is to find the data points in a dataset that are closest (or most similar) to a given query point. However, finding the exact nearest neighbors can be computationally expensive, particularly when dealing with high-dimensional data. +Therefore, ANN algorithms aim to find the nearest neighbors approximately, prioritizing speed and efficiency over exact accuracy. + +JVector achieves this goal by creating a hierarchy of graphs, where each level of the hierarchy corresponds to a `small world` graph that is navigable. +It is inspired by DiskANN, a disk-backed ANN library, to store the graphs on disk. +For any given node (data point) in the graph, it is easy to find a path to any other node. +The higher levels of the hierarchy have fewer nodes and are used for coarse navigation, while the lower levels have more nodes and are used for fine navigation. +Such indexing structures enable fast retrieval by narrowing down the search space to potential matches. + +JVector also uses the Panama SIMD API to accelerate index build and queries. + +== Similarity Metric + +Vector search relies on computing the similarity or distance between vectors to identify relevant matches. +Choosing an appropriate similarity metric is crucial, as different metrics may be more suitable for specific types of data. +Common similarity metrics include cosine similarity, Euclidean distance, or Jaccard similarity. +The choice of metric should align with the characteristics of the data and the desired search behavior. + +Vector Search supports three similarity metrics: `cosine similarity`, `dot product`, and `Euclidean distance`. +The default similarity algorithm for the Vector Search indexes is `cosine similarity`. +The recommendation is to use the `dot product` on normalized embeddings for most applications, because `dot product` is 50% faster than `cosine similarity`. + +== Scalability and Performance + +Scalability is a critical consideration as your dataset expands. +Vector search algorithms should be designed to handle large-scale datasets efficiently. +Your {cassandra} database using Vector Search efficiently distributes data and accesses that data with parallel processing to enhance performance. + +== Evaluation and iteration + +Continuously evaluate and iterate the data to refine search results against known truth and user feedback. +This also helps identify areas for improvement. +Iteratively refining the vector representations, similarity metrics, indexing techniques, or preprocessing steps can lead to better search performance and user satisfaction. \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/overview.adoc b/doc/modules/cassandra/pages/vector-search/overview.adoc new file mode 100644 index 0000000000..3448d65d06 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/overview.adoc @@ -0,0 +1,30 @@ += Overview of Vector Search +:navtitle: Vector Search overview +:description: Vector Search overview +:keywords: CEP-30 + +Vector Search is a new feature added to {cass-50}. +It is a powerful technique for finding relevant content within large document collections and is particularly useful for AI applications. + +xref:cassandra:vector-search/concepts.adoc[Vector Search concepts]:: +Use Vector Search to create powerful relevant content search for any CQL table. + +xref:cassandra:getting-started/vector-search-quickstart.adoc[Vector Search quickstart]:: +Follow the steps to get started quickly with Vector Search. + +// xref:cassandra:vector-search/faq.adoc[Vector Search FAQs]:: +// Frequently asked questions about Vector Search. + +xref:cassandra:vector-search/vector-search-working-with.adoc[Working with Vector Search]:: +Create, check, alter, drop, and query Vector Search. + +// xref:cassandra:vector-search/vector-search-operations.adoc[Vector Search operations]:: +// Configuring and monitoring. + +xref:cassandra:vector-search/vector-search-query.adoc[Querying with Vector Search]:: +Understand how to run Vector Search queries. + +xref:cassandra:vector-search/data-modeling.adoc[Data modeling with Vector Search]:: +The ins and outs of data modeling with Vector Search. + +*Reference:* xref:cassandra:reference/cql-commands/create-table.adoc[CREATE TABLE], xref:cassandra:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] \ No newline at end of file diff --git a/doc/modules/cassandra/pages/vector-search/quickstarts.adoc b/doc/modules/cassandra/pages/vector-search/quickstarts.adoc new file mode 100644 index 0000000000..20c24447f2 --- /dev/null +++ b/doc/modules/cassandra/pages/vector-search/quickstarts.adoc @@ -0,0 +1,5 @@ += Quickstarts + +// * xref:vector-search/sai-quickstart.adoc[] +// * xref:vector-search/cql.adoc[] +* xref:cassandra:getting-started/vector-search-quickstart.adoc[] diff --git a/doc/modules/cassandra/partials/cql-syntax-legend.adoc b/doc/modules/cassandra/partials/cql-syntax-legend.adoc index 1b4d8e9006..0064826f66 100644 --- a/doc/modules/cassandra/partials/cql-syntax-legend.adoc +++ b/doc/modules/cassandra/partials/cql-syntax-legend.adoc @@ -60,5 +60,4 @@ This syntax is useful when arguments might be mistaken for command line options. | `@=''` | Search CQL only: Identify the entity and literal value to overwrite the XML element in the schema and solrConfig files. -|=== - +|=== \ No newline at end of file diff --git a/doc/modules/cassandra/partials/sai/support-databases.adoc b/doc/modules/cassandra/partials/sai/support-databases.adoc index 028d55cdab..6a3f082989 100644 --- a/doc/modules/cassandra/partials/sai/support-databases.adoc +++ b/doc/modules/cassandra/partials/sai/support-databases.adoc @@ -1,4 +1,2 @@ -Supported databases: - -* {cassandra} 5.0 +{cass-50} is the only supported database currently. diff --git a/doc/scripts/gen-nodetool-docs.py b/doc/scripts/gen-nodetool-docs.py index 6cee6fe9e7..835f3f40a2 100644 --- a/doc/scripts/gen-nodetool-docs.py +++ b/doc/scripts/gen-nodetool-docs.py @@ -89,7 +89,7 @@ with open(outdir + "/nodetool.adoc", "w+") as output: with open(helpfilename, "r+") as helpfile: output.write("= Nodetool\n\n== Usage\n\n") for commandLine in helpfile: - command = command_re.sub(r'\nxref:tools/nodetool/\2.adoc[\2] - ',commandLine) + command = command_re.sub(r'\nxref:modules/cassandra/pages/managing/tools/nodetool/\2.adoc[\2] - ',commandLine) output.write(command) # create the command usage pages