natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert flush directory (CASSANDRA-6357) 

Patch by Jonathan Ellis; reviewed by Benedict Elliott Smith
This commit is contained in:
belliottsmith 2014-05-29 22:37:50 +01:00
parent 5420b7a229
commit ee950b4103
15 changed files with 48 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
2.1.0-rc1
* Revert flush directory (CASSANDRA-6357)
* More efficient executor service for fast operations (CASSANDRA-4718)
* Move less common tools into a new cassandra-tools package (CASSANDRA-7160)
* Support more concurrent requests in native protocol (CASSANDRA-7231)

10
conf/cassandra.yaml
View File

@ -100,12 +100,6 @@ data_file_directories:
# separate spindle than the data directories.
commitlog_directory: /var/lib/cassandra/commitlog
# location to write flushing sstables to. Ideally, this will also be
# a separate spindle in HDD deployments. If you only have two spindles,
# have it share with the data spindle. By default, this will point at the data
# directory.
# flush_directory: /var/lib/cassandra/flush
# policy for data disk failures:
# stop_paranoid: shut down gossip and Thrift even for single-sstable errors.
# stop: shut down gossip and Thrift, leaving the node effectively dead, but
@ -299,8 +293,8 @@ memtable_allocation_type: heap_buffers
# This sets the amount of memtable flush writer threads. These will
# be blocked by disk io, and each one will hold a memtable in memory
# while blocked. If your flush directory is backed by SSD, you may
# want to increase this; by default it will be set to 2.
#memtable_flush_writers: 2
# want to increase this.
memtable_flush_writers: 2
# A fixed memory pool size in MB for for SSTable index summaries. If left
# empty, this will default to 5% of the heap size. If the memory usage of

1
src/java/org/apache/cassandra/config/Config.java
View File

@ -141,7 +141,6 @@ public class Config
public volatile Integer inter_dc_stream_throughput_outbound_megabits_per_sec = 0;
public String[] data_file_directories;
public String flush_directory;
public String saved_caches_directory;

10
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -461,9 +461,6 @@ public class DatabaseDescriptor
/* data file and commit log directories. they get created later, when they're needed. */
if (conf.commitlog_directory != null && conf.data_file_directories != null && conf.saved_caches_directory != null)
{
if (conf.flush_directory == null)
conf.flush_directory = conf.data_file_directories[0];
for (String datadir : conf.data_file_directories)
{
if (datadir.equals(conf.commitlog_directory))
@ -670,8 +667,6 @@ public class DatabaseDescriptor
throw new ConfigurationException("saved_caches_directory must be specified");
FileUtils.createDirectory(conf.saved_caches_directory);
FileUtils.createDirectory(conf.flush_directory);
}
catch (ConfigurationException e)
{
@ -1523,9 +1518,4 @@ public class DatabaseDescriptor
String arch = System.getProperty("os.arch");
return arch.contains("64") || arch.contains("sparcv9");
}
public static String getFlushLocation()
{
return conf.flush_directory;
}
}

81
src/java/org/apache/cassandra/db/Directories.java
View File

@ -17,6 +17,8 @@
*/
package org.apache.cassandra.db;
import static com.google.common.collect.Sets.newHashSet;
import java.io.File;
import java.io.FileFilter;
import java.io.IOError;
@ -39,26 +41,20 @@ import com.google.common.collect.ImmutableSet.Builder;
import com.google.common.collect.Iterables;
import com.google.common.primitives.Longs;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.config.Config;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.*;
import org.apache.cassandra.io.FSError;
import org.apache.cassandra.io.FSWriteError;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTable;
import org.apache.cassandra.io.sstable.SSTableDeletingTask;
import org.apache.cassandra.io.util.FileUtils;
import org.apache.cassandra.io.sstable.*;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import static com.google.common.collect.Sets.newHashSet;
/**
* Encapsulate handling of paths to the data files.
*
@ -92,14 +88,12 @@ public class Directories
public static final String SECONDARY_INDEX_NAME_SEPARATOR = ".";
public static final DataDirectory[] dataDirectories;
public static final DataDirectory flushDirectory;
static
{
String[] locations = DatabaseDescriptor.getAllDataFileLocations();
dataDirectories = new DataDirectory[locations.length];
for (int i = 0; i < locations.length; ++i)
dataDirectories[i] = new DataDirectory(new File(locations[i]));
flushDirectory = new DataDirectory(new File(DatabaseDescriptor.getFlushLocation()));
}
@ -178,7 +172,6 @@ public class Directories
private final CFMetaData metadata;
private final File[] dataPaths;
private final File flushPath;
/**
* Create Directories of given ColumnFamily.
@ -192,7 +185,6 @@ public class Directories
if (StorageService.instance.isClientMode())
{
dataPaths = null;
flushPath = null;
return;
}
@ -222,9 +214,7 @@ public class Directories
dataPaths[i] = new File(dataDirectories[i].location, join(metadata.ksName, directoryName));
}
flushPath = new File(flushDirectory.location, join(metadata.ksName, directoryName));
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
{
try
{
@ -239,26 +229,6 @@ public class Directories
}
}
/**
* @return an iterable of all possible sstable paths, including flush and post-compaction locations.
* Guaranteed to only return one copy of each path, even if there is no dedicated flush location and
* it shares with the others.
*/
private Iterable<File> allSSTablePaths()
{
return ImmutableSet.<File>builder().add(dataPaths).add(flushPath).build();
}
/**
* @return an iterable of all possible sstable directories, including flush and post-compaction locations.
* Guaranteed to only return one copy of each directories, even if there is no dedicated flush location and
* it shares with the others.
*/
private static Iterable<DataDirectory> allSSTableDirectories()
{
return ImmutableSet.<DataDirectory>builder().add(dataDirectories).add(flushDirectory).build();
}
/**
* Returns SSTable location which is inside given data directory.
*
@ -267,7 +237,7 @@ public class Directories
*/
public File getLocationForDisk(DataDirectory dataDirectory)
{
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
{
if (dir.getAbsolutePath().startsWith(dataDirectory.location.getAbsolutePath()))
return dir;
@ -277,7 +247,7 @@ public class Directories
public Descriptor find(String filename)
{
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
{
if (new File(dir, filename).exists())
return Descriptor.fromFilename(dir, filename).left;
@ -285,9 +255,9 @@ public class Directories
return null;
}
public File getDirectoryForCompactedSSTables()
public File getDirectoryForNewSSTables()
{
File path = getCompactionLocationAsFile();
File path = getWriteableLocationAsFile();
// Requesting GC has a chance to free space only if we're using mmap and a non SUN jvm
if (path == null
@ -300,15 +270,15 @@ public class Directories
// Note: GCInspector will do this already, but only sun JVM supports GCInspector so far
SSTableDeletingTask.rescheduleFailedTasks();
Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS);
path = getCompactionLocationAsFile();
path = getWriteableLocationAsFile();
}
return path;
}
public File getCompactionLocationAsFile()
public File getWriteableLocationAsFile()
{
return getLocationForDisk(getCompactionLocation());
return getLocationForDisk(getWriteableLocation());
}
/**
@ -316,7 +286,7 @@ public class Directories
*
* @throws IOError if all directories are blacklisted.
*/
public DataDirectory getCompactionLocation()
public DataDirectory getWriteableLocation()
{
List<DataDirectory> candidates = new ArrayList<>();
@ -346,13 +316,6 @@ public class Directories
return candidates.get(0);
}
public DataDirectory getFlushLocation()
{
return BlacklistedDirectories.isUnwritable(flushPath)
? getCompactionLocation()
: flushDirectory;
}
public static File getSnapshotDirectory(Descriptor desc, String snapshotName)
{
return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
@ -360,7 +323,7 @@ public class Directories
public File getSnapshotManifestFile(String snapshotName)
{
return new File(getDirectoryForCompactedSSTables(), join(SNAPSHOT_SUBDIR, snapshotName, "manifest.json"));
return new File(getDirectoryForNewSSTables(), join(SNAPSHOT_SUBDIR, snapshotName, "manifest.json"));
}
public static File getBackupsDirectory(Descriptor desc)
@ -469,7 +432,7 @@ public class Directories
if (filtered)
return;
for (File location : allSSTablePaths())
for (File location : dataPaths)
{
if (BlacklistedDirectories.isUnreadable(location))
continue;
@ -531,7 +494,7 @@ public class Directories
public Map<String, Pair<Long, Long>> getSnapshotDetails()
{
final Map<String, Pair<Long, Long>> snapshotSpaceMap = new HashMap<>();
for (final File dir : allSSTablePaths())
for (final File dir : dataPaths)
{
final File snapshotDir = new File(dir,SNAPSHOT_SUBDIR);
if (snapshotDir.exists() && snapshotDir.isDirectory())
@ -561,7 +524,7 @@ public class Directories
}
public boolean snapshotExists(String snapshotName)
{
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
{
File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName));
if (snapshotDir.exists())
@ -589,7 +552,7 @@ public class Directories
// The snapshot must exist
public long snapshotCreationTime(String snapshotName)
{
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
{
File snapshotDir = new File(dir, join(SNAPSHOT_SUBDIR, snapshotName));
if (snapshotDir.exists())
@ -601,7 +564,7 @@ public class Directories
public long trueSnapshotsSize()
{
long result = 0L;
for (File dir : allSSTablePaths())
for (File dir : dataPaths)
result += getTrueAllocatedSizeIn(new File(dir, join(SNAPSHOT_SUBDIR)));
return result;
}
@ -633,7 +596,7 @@ public class Directories
public static List<File> getKSChildDirectories(String ksName)
{
List<File> result = new ArrayList<>();
for (DataDirectory dataDirectory : allSSTableDirectories())
for (DataDirectory dataDirectory : dataDirectories)
{
File ksDir = new File(dataDirectory.location, ksName);
File[] cfDirs = ksDir.listFiles();
@ -651,7 +614,7 @@ public class Directories
public List<File> getCFDirectories()
{
List<File> result = new ArrayList<>();
for (File dataDirectory : allSSTablePaths())
for (File dataDirectory : dataPaths)
{
if (dataDirectory.isDirectory())
result.add(dataDirectory);

5
src/java/org/apache/cassandra/db/Memtable.java
View File

@ -302,11 +302,6 @@ public class Memtable
* 1.2); // bloom filter and row index overhead
}
protected Directories.DataDirectory getWriteableLocation()
{
return cfs.directories.getFlushLocation();
}
public long getExpectedWriteSize()
{
return estimatedSize;

5
src/java/org/apache/cassandra/db/compaction/AbstractCompactionTask.java
View File

@ -49,11 +49,6 @@ public abstract class AbstractCompactionTask extends DiskAwareRunnable
assert compacting.contains(sstable) : sstable.getFilename() + " is not correctly marked compacting";
}
protected Directories.DataDirectory getWriteableLocation()
{
return cfs.directories.getCompactionLocation();
}
/**
* executes the task and unmarks sstables compacting
*/

4
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -674,7 +674,7 @@ public class CompactionManager implements CompactionManagerMBean
logger.info("Cleaning up {}", sstable);
File compactionFileLocation = cfs.directories.getDirectoryForCompactedSSTables();
File compactionFileLocation = cfs.directories.getDirectoryForNewSSTables();
if (compactionFileLocation == null)
throw new IOException("disk full");
@ -953,7 +953,7 @@ public class CompactionManager implements CompactionManagerMBean
Set<SSTableReader> sstableAsSet = new HashSet<>();
sstableAsSet.add(sstable);
File destination = cfs.directories.getDirectoryForCompactedSSTables();
File destination = cfs.directories.getDirectoryForNewSSTables();
SSTableRewriter repairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);
SSTableRewriter unRepairedSSTableWriter = new SSTableRewriter(cfs, sstableAsSet, sstable.maxDataAge, OperationType.ANTICOMPACTION, false);

2
src/java/org/apache/cassandra/db/compaction/Scrubber.java
View File

@ -80,7 +80,7 @@ public class Scrubber implements Closeable
this.isOffline = isOffline;
// Calculate the expected compacted filesize
this.destination = cfs.directories.getDirectoryForCompactedSSTables();
this.destination = cfs.directories.getDirectoryForNewSSTables();
if (destination == null)
throw new IOException("disk full");

4
src/java/org/apache/cassandra/io/util/DiskAwareRunnable.java
View File

@ -34,7 +34,7 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
while (true)
{
writeSize = getExpectedWriteSize();
directory = getWriteableLocation();
directory = getDirectories().getWriteableLocation();
if (directory != null || !reduceScopeForLimitedSpace())
break;
}
@ -54,8 +54,6 @@ public abstract class DiskAwareRunnable extends WrappedRunnable
}
}
protected abstract Directories.DataDirectory getWriteableLocation();
/**
* Get sstable directories for the CF.
* @return Directories instance for the CF.

3
src/java/org/apache/cassandra/streaming/StreamReader.java
View File

@ -40,6 +40,7 @@ import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.sstable.Component;
import org.apache.cassandra.io.sstable.Descriptor;
import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.streaming.messages.FileMessageHeader;
import org.apache.cassandra.utils.ByteBufferUtil;
@ -110,7 +111,7 @@ public class StreamReader
protected SSTableWriter createWriter(ColumnFamilyStore cfs, long totalSize, long repairedAt) throws IOException
{
Directories.DataDirectory localDir = cfs.directories.getCompactionLocation();
Directories.DataDirectory localDir = cfs.directories.getWriteableLocation();
if (localDir == null)
throw new IOException("Insufficient disk space to store " + totalSize + " bytes");
desc = Descriptor.fromFilename(cfs.getTempSSTablePath(cfs.directories.getLocationForDisk(localDir)));

3
src/java/org/apache/cassandra/streaming/StreamReceiveTask.java
View File

@ -27,6 +27,7 @@ import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.service.ActiveRepairService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.Pair;
@ -97,7 +98,7 @@ public class StreamReceiveTask extends StreamTask
Pair<String, String> kscf = Schema.instance.getCF(task.cfId);
ColumnFamilyStore cfs = Keyspace.open(kscf.left).getColumnFamilyStore(kscf.right);
StreamLockfile lockfile = new StreamLockfile(cfs.directories.getCompactionLocationAsFile(), UUID.randomUUID());
StreamLockfile lockfile = new StreamLockfile(cfs.directories.getWriteableLocationAsFile(), UUID.randomUUID());
lockfile.create(task.sstables);
List<SSTableReader> readers = new ArrayList<>();
for (SSTableWriter writer : task.sstables)

14
test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java
View File

@ -954,7 +954,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
for (int version = 1; version <= 2; ++version)
{
Descriptor existing = new Descriptor(cfs.directories.getDirectoryForCompactedSSTables(), "Keyspace2", "Standard1", version, Descriptor.Type.FINAL);
Descriptor existing = new Descriptor(cfs.directories.getDirectoryForNewSSTables(), "Keyspace2", "Standard1", version, Descriptor.Type.FINAL);
Descriptor desc = new Descriptor(Directories.getBackupsDirectory(existing), "Keyspace2", "Standard1", version, Descriptor.Type.FINAL);
for (Component c : new Component[]{ Component.DATA, Component.PRIMARY_INDEX, Component.FILTER, Component.STATS })
assertTrue("can not find backedup file:" + desc.filenameFor(c), new File(desc.filenameFor(c)).exists());
@ -1662,7 +1662,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
ByteBuffer key = bytes("key");
// 1st sstable
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForCompactedSSTables(), cfmeta, StorageService.getPartitioner());
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), cfmeta, StorageService.getPartitioner());
writer.newRow(key);
writer.addColumn(bytes("col"), bytes("val"), 1);
writer.close();
@ -1674,7 +1674,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
final SSTableReader sstable1 = SSTableReader.open(sstableToOpen.getKey());
// simulate incomplete compaction
writer = new SSTableSimpleWriter(dir.getDirectoryForCompactedSSTables(),
writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(),
cfmeta, StorageService.getPartitioner())
{
protected SSTableWriter getWriter()
@ -1729,7 +1729,7 @@ public class ColumnFamilyStoreTest extends SchemaLoader
// Write SSTable generation 3 that has ancestors 1 and 2
final Set<Integer> ancestors = Sets.newHashSet(1, 2);
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForCompactedSSTables(),
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(),
cfmeta, StorageService.getPartitioner())
{
protected SSTableWriter getWriter()
@ -1796,13 +1796,13 @@ public class ColumnFamilyStoreTest extends SchemaLoader
ByteBuffer key = bytes("key");
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForCompactedSSTables(),
cfmeta, StorageService.getPartitioner());
SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(),
cfmeta, StorageService.getPartitioner());
writer.newRow(key);
writer.addColumn(bytes("col"), bytes("val"), 1);
writer.close();
writer = new SSTableSimpleWriter(dir.getDirectoryForCompactedSSTables(),
writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(),
cfmeta, StorageService.getPartitioner());
writer.newRow(key);
writer.addColumn(bytes("col"), bytes("val"), 1);

16
test/unit/org/apache/cassandra/db/DirectoriesTest.java
View File

@ -30,7 +30,6 @@ import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.google.common.collect.Lists;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@ -125,7 +124,7 @@ public class DirectoriesTest
for (CFMetaData cfm : CFM)
{
Directories directories = new Directories(cfm);
assertEquals(cfDir(cfm), directories.getDirectoryForCompactedSSTables());
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
Descriptor desc = new Descriptor(cfDir(cfm), KS, cfm.cfName, 1, Descriptor.Type.FINAL);
File snapshotDir = new File(cfDir(cfm), File.separator + Directories.SNAPSHOT_SUBDIR + File.separator + "42");
@ -187,13 +186,12 @@ public class DirectoriesTest
public void testDiskFailurePolicy_best_effort()
{
DiskFailurePolicy origPolicy = DatabaseDescriptor.getDiskFailurePolicy();
List<DataDirectory> directories = Lists.asList(Directories.flushDirectory, Directories.dataDirectories);
try
try
{
DatabaseDescriptor.setDiskFailurePolicy(DiskFailurePolicy.best_effort);
for (DataDirectory dd : directories)
for (DataDirectory dd : Directories.dataDirectories)
{
dd.location.setExecutable(false);
dd.location.setWritable(false);
@ -210,7 +208,7 @@ public class DirectoriesTest
}
finally
{
for (DataDirectory dd : directories)
for (DataDirectory dd : Directories.dataDirectories)
{
dd.location.setExecutable(true);
dd.location.setWritable(true);
@ -226,7 +224,7 @@ public class DirectoriesTest
for (final CFMetaData cfm : CFM)
{
final Directories directories = new Directories(cfm);
assertEquals(cfDir(cfm), directories.getDirectoryForCompactedSSTables());
assertEquals(cfDir(cfm), directories.getDirectoryForNewSSTables());
final String n = Long.toString(System.nanoTime());
Callable<File> directoryGetter = new Callable<File>() {
public File call() throws Exception {

2
test/unit/org/apache/cassandra/io/sstable/SSTableSimpleWriterTest.java
View File

@ -44,7 +44,7 @@ public class SSTableSimpleWriterTest extends SchemaLoader
String cfname = "StandardInteger1";
Keyspace t = Keyspace.open(keyspaceName); // make sure we create the directory
File dir = new Directories(Schema.instance.getCFMetaData(keyspaceName, cfname)).getDirectoryForCompactedSSTables();
File dir = new Directories(Schema.instance.getCFMetaData(keyspaceName, cfname)).getDirectoryForNewSSTables();
assert dir.exists();
IPartitioner partitioner = StorageService.getPartitioner();