diff --git a/CHANGES.txt b/CHANGES.txt index 59b3bf68b7..2eb9705030 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -21,6 +21,8 @@ * avoid exporting an un-requested row in sstable2json, when exporting a key that does not exist (CASSANDRA-2168) * track and migrate cached pages during compaction (CASSANDRA-1902) + * add incremental_backups option (CASSANDRA-1872) + 0.7.3 * Keep endpoint state until aVeryLongTime (CASSANDRA-2115) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 60d3b6f1d5..25f46bb337 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -217,6 +217,12 @@ thrift_framed_transport_size_in_mb: 15 # internal thrift overhead. thrift_max_message_length_in_mb: 16 +# Set to true to have Cassandra create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# Keyspace data. Removing these links is the operator's +# responsibility. +incremental_backups: false + # Whether or not to take a snapshot before each compaction. Be # careful using this option, since Cassandra won't clean up the # snapshots for you. Mostly useful if you're paranoid when there diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index 89ba3d0e04..b9a123dfad 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -118,6 +118,8 @@ public class Config // TODO: remove in 0.8 public Boolean enable_page_cache_migration = false; + public boolean incremental_backups = false; + public static enum CommitLogSync { periodic, batch diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 04dd94be54..e1105b5e56 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -37,7 +37,6 @@ import org.apache.cassandra.config.Config.RequestSchedulerId; import org.apache.cassandra.db.ColumnFamilyType; import org.apache.cassandra.db.DefsTable; import org.apache.cassandra.db.Table; -import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.migration.Migration; import org.apache.cassandra.dht.IPartitioner; @@ -1240,4 +1239,9 @@ public class DatabaseDescriptor { return conf.enable_page_cache_migration; } + + public static boolean incrementalBackupsEnabled() + { + return conf.incremental_backups; + } } diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index e3f19c1ce9..08c3755ae9 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -962,6 +962,21 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean { assert sstable.getColumnFamilyName().equals(columnFamily); ssTables.add(Arrays.asList(sstable)); + if (DatabaseDescriptor.incrementalBackupsEnabled()) + { + File keyspaceDir = new File(sstable.getFilename()).getParentFile(); + File backupsDir = new File(keyspaceDir, "backups"); + try + { + if (!backupsDir.exists() && !backupsDir.mkdirs()) + throw new IOException("Unable to create " + backupsDir); + sstable.createLinks(backupsDir.getCanonicalPath()); + } + catch (IOException e) + { + throw new IOError(e); + } + } CompactionManager.instance.submitMinorIfNeeded(this); } diff --git a/src/java/org/apache/cassandra/io/sstable/SSTable.java b/src/java/org/apache/cassandra/io/sstable/SSTable.java index 41c6f46f98..05d6212f0a 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTable.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTable.java @@ -183,7 +183,7 @@ public abstract class SSTable /** * Discovers existing components for the descriptor. Slow: only intended for use outside the critical path. */ - static Set componentsFor(final Descriptor desc) throws IOException + static Set componentsFor(final Descriptor desc) { final Set components = new HashSet(); desc.directory.list(new FilenameFilter() diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index f774903496..9738cd5e6d 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -32,6 +32,7 @@ encryption_options: keystore_password: cassandra truststore: conf/.truststore truststore_password: cassandra +incremental_backups: true keyspaces: - name: Keyspace1 replica_placement_strategy: org.apache.cassandra.locator.SimpleStrategy diff --git a/test/system/test_thrift_server.py b/test/system/test_thrift_server.py index ab82d77300..cf7a94be3f 100644 --- a/test/system/test_thrift_server.py +++ b/test/system/test_thrift_server.py @@ -548,7 +548,8 @@ class TestMutations(ThriftTester): _assert_columnpath_exists(key, ColumnPath('Super1', super_column='sc1', column=c)) _assert_columnpath_exists(key, ColumnPath('Super2', super_column='sc1', column=c)) - def test_batch_mutate_does_not_accept_cosc_and_deletion_in_same_mutation(self): + def test_bad_batch_calls(self): + # mutate_does_not_accept_cosc_and_deletion_in_same_mutation def too_full(): _set_keyspace('Keyspace1') col = ColumnOrSuperColumn(column=Column("foo", 'bar', 0)) @@ -557,7 +558,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(too_full, InvalidRequestException) - def test_batch_mutate_does_not_yet_accept_slice_ranges(self): + # test_batch_mutate_does_not_yet_accept_slice_ranges def send_range(): _set_keyspace('Keyspace1') sp = SlicePredicate(slice_range=SliceRange(start='0', finish="", count=10)) @@ -566,7 +567,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(send_range, InvalidRequestException) - def test_batch_mutate_does_not_accept_cosc_on_undefined_cf(self): + # test_batch_mutate_does_not_accept_cosc_on_undefined_cf: def bad_cf(): _set_keyspace('Keyspace1') col = ColumnOrSuperColumn(column=Column("foo", 'bar', 0)) @@ -574,7 +575,7 @@ class TestMutations(ThriftTester): ConsistencyLevel.ONE) _expect_exception(bad_cf, InvalidRequestException) - def test_batch_mutate_does_not_accept_deletion_on_undefined_cf(self): + # test_batch_mutate_does_not_accept_deletion_on_undefined_cf def bad_cf(): _set_keyspace('Keyspace1') d = Deletion(2, predicate=SlicePredicate(column_names=['baz'])) diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 752ba530a8..9794d8d230 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -18,6 +18,7 @@ */ package org.apache.cassandra.db; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -39,6 +40,7 @@ import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.Util; import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.ConfigurationException; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.columniterator.IdentityQueryFilter; import org.apache.cassandra.db.filter.*; import org.apache.cassandra.dht.IPartitioner; @@ -52,6 +54,7 @@ import org.apache.cassandra.thrift.IndexType; import org.apache.cassandra.utils.WrappedRunnable; import static junit.framework.Assert.assertEquals; +import static junit.framework.Assert.assertTrue; import static org.apache.cassandra.Util.column; import static org.apache.cassandra.Util.getBytes; import static org.junit.Assert.assertNull; @@ -515,4 +518,17 @@ public class ColumnFamilyStoreTest extends CleanupHelper rms.add(rm); return Util.writeColumnFamily(rms); } + + @Test + public void testBackupAfterFlush() throws Throwable + { + insertKey1Key2(); + + File backupDir = new File(DatabaseDescriptor.getDataFileLocationForTable("Keyspace2", 0), "backups"); + for (String f : Arrays.asList("Standard1-f-1-Data.db", "Standard1-f-1-Index.db", "Standard1-f-2-Data.db", "Standard1-f-2-Index.db", + "Standard1-f-1-Filter.db", "Standard1-f-1-Statistics.db", "Standard1-f-2-Filter.db", "Standard1-f-2-Statistics.db")) + { + assertTrue("can not find backedup file:" + f, new File(backupDir, f).exists()); + } + } }