From 830b0d8c544111c4f4318e7c2dab67176b261b55 Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Mon, 7 Mar 2011 16:09:15 +0000 Subject: [PATCH 1/2] combine similar short tests since setup/teardown is so expensive git-svn-id: https://svn.apache.org/repos/asf/cassandra/branches/cassandra-0.7@1078826 13f79535-47bb-0310-9956-ffa450edef68 --- test/system/test_thrift_server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/system/test_thrift_server.py b/test/system/test_thrift_server.py index 5cb9ebb79c..e2388ca9b4 100644 --- a/test/system/test_thrift_server.py +++ b/test/system/test_thrift_server.py @@ -548,7 +548,8 @@ class TestMutations(ThriftTester): _assert_columnpath_exists(key, ColumnPath('Super1', super_column='sc1', column=c)) _assert_columnpath_exists(key, ColumnPath('Super2', super_column='sc1', column=c)) - def test_batch_mutate_does_not_accept_cosc_and_deletion_in_same_mutation(self): + def test_bad_batch_calls(self): + # mutate_does_not_accept_cosc_and_deletion_in_same_mutation def too_full(): _set_keyspace('Keyspace1') col = ColumnOrSuperColumn(column=Column("foo", 'bar', 0)) @@ -557,7 +558,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(too_full, InvalidRequestException) - def test_batch_mutate_does_not_yet_accept_slice_ranges(self): + # test_batch_mutate_does_not_yet_accept_slice_ranges def send_range(): _set_keyspace('Keyspace1') sp = SlicePredicate(slice_range=SliceRange(start='0', finish="", count=10)) @@ -566,7 +567,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(send_range, InvalidRequestException) - def test_batch_mutate_does_not_accept_cosc_on_undefined_cf(self): + # test_batch_mutate_does_not_accept_cosc_on_undefined_cf: def bad_cf(): _set_keyspace('Keyspace1') col = ColumnOrSuperColumn(column=Column("foo", 'bar', 0)) @@ -574,7 +575,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(bad_cf, InvalidRequestException) - def test_batch_mutate_does_not_accept_deletion_on_undefined_cf(self): + # test_batch_mutate_does_not_accept_deletion_on_undefined_cf def bad_cf(): _set_keyspace('Keyspace1') d = Deletion(2, predicate=SlicePredicate(column_names=['baz'])) From cb71747c88efc7ad3e7dda67316855d6571aadfc Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Mon, 7 Mar 2011 17:30:07 +0000 Subject: [PATCH 2/2] add incremental_backups option patch by Yang Yang and jbellis for CASSANDRA-1872 git-svn-id: https://svn.apache.org/repos/asf/cassandra/branches/cassandra-0.7@1078858 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 2 ++ conf/cassandra.yaml | 6 ++++++ src/java/org/apache/cassandra/config/Config.java | 2 ++ .../cassandra/config/DatabaseDescriptor.java | 6 +++++- .../apache/cassandra/db/ColumnFamilyStore.java | 15 +++++++++++++++ .../org/apache/cassandra/io/sstable/SSTable.java | 2 +- test/conf/cassandra.yaml | 1 + .../cassandra/db/ColumnFamilyStoreTest.java | 16 ++++++++++++++++ 8 files changed, 48 insertions(+), 2 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 76dcd202cb..bfe3bbc436 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -8,6 +8,8 @@ * avoid exporting an un-requested row in sstable2json, when exporting a key that does not exist (CASSANDRA-2168) * track and migrate cached pages during compaction (CASSANDRA-1902) + * add incremental_backups option (CASSANDRA-1872) + 0.7.3 * Keep endpoint state until aVeryLongTime (CASSANDRA-2115) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 24868b89b4..f2c8097690 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -212,6 +212,12 @@ thrift_framed_transport_size_in_mb: 15 # internal thrift overhead. thrift_max_message_length_in_mb: 16 +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# Keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + # Whether or not to take a snapshot before each compaction. Be # careful using this option, since Cassandra won't clean up the # snapshots for you. Mostly useful if you're paranoid when there diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 36abea6cc2..520bab4d79 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -115,6 +115,8 @@ public class Config // TODO: remove in 0.8 public Boolean enable_page_cache_migration = false; + public boolean incremental_backups = false; + public static enum CommitLogSync { periodic, batch diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 8094c5e094..6c2da41c1c 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -37,7 +37,6 @@ import org.apache.cassandra.config.Config.RequestSchedulerId; import org.apache.cassandra.db.ColumnFamilyType; import org.apache.cassandra.db.DefsTable; import org.apache.cassandra.db.Table; -import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.migration.Migration; import org.apache.cassandra.dht.IPartitioner; @@ -1218,4 +1217,9 @@ public class DatabaseDescriptor { return conf.enable_page_cache_migration; } + + public static boolean incrementalBackupsEnabled() + { + return conf.incremental_backups; + } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 3c23dfd6b9..7270b0678b 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -919,6 +919,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { assert sstable.getColumnFamilyName().equals(columnFamily); ssTables.add(Arrays.asList(sstable)); + if (DatabaseDescriptor.incrementalBackupsEnabled()) + { + File keyspaceDir = new File(sstable.getFilename()).getParentFile(); + File backupsDir = new File(keyspaceDir, "backups"); + try + { + if (!backupsDir.exists() && !backupsDir.mkdirs()) + throw new IOException("Unable to create " + backupsDir); + sstable.createLinks(backupsDir.getCanonicalPath()); + } + catch (IOException e) + { + throw new IOError(e); + } + } CompactionManager.instance.submitMinorIfNeeded(this); } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index 41c6f46f98..05d6212f0a 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -183,7 +183,7 @@ public abstract class SSTable /** * Discovers existing components for the descriptor. Slow: only intended for use outside the critical path. */ - static Set componentsFor(final Descriptor desc) throws IOException + static Set componentsFor(final Descriptor desc) { final Set components = new HashSet(); desc.directory.list(new FilenameFilter() diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 7c401a623a..d5bb13d56e 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -24,6 +24,7 @@ endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch dynamic_snitch: true request_scheduler: org.apache.cassandra.scheduler.RoundRobinScheduler request_scheduler_id: keyspace +incremental_backups: true keyspaces: - name: Keyspace1 replica_placement_strategy: org.apache.cassandra.locator.SimpleStrategy diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 752ba530a8..9794d8d230 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -18,6 +18,7 @@ */ package org.apache.cassandra.db; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -39,6 +40,7 @@ import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.Util; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.ConfigurationException; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.filter.*; import org.apache.cassandra.dht.IPartitioner; @@ -52,6 +54,7 @@ import org.apache.cassandra.thrift.IndexType; import org.apache.cassandra.utils.WrappedRunnable; import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; import static org.apache.cassandra.Util.column; import static org.apache.cassandra.Util.getBytes; import static org.junit.Assert.assertNull; @@ -515,4 +518,17 @@ public class ColumnFamilyStoreTest extends CleanupHelper rms.add(rm); return Util.writeColumnFamily(rms); } + + @Test + public void testBackupAfterFlush() throws Throwable + { + insertKey1Key2(); + + File backupDir = new File(DatabaseDescriptor.getDataFileLocationForTable("Keyspace2", 0), "backups"); + for (String f : Arrays.asList("Standard1-f-1-Data.db", "Standard1-f-1-Index.db", "Standard1-f-2-Data.db", "Standard1-f-2-Index.db", + "Standard1-f-1-Filter.db", "Standard1-f-1-Statistics.db", "Standard1-f-2-Filter.db", "Standard1-f-2-Statistics.db")) + { + assertTrue("can not find backedup file:" + f, new File(backupDir, f).exists()); + } + } }