natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix high memory use due to tracking reads on incrementally opened sstable readers 

Patch by Jake Luciani; reviewed by Jason Brown for (CASSANDRA-8066)
This commit is contained in:
T Jake Luciani 2014-10-14 08:55:50 -04:00
parent 63cb95e012
commit c5cbd58a3b
5 changed files with 31 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
2.1.1
* Fix high memory use due to tracking reads on incrementally opened sstable readers (CASSANDRA-8066)
* Fix EXECUTE request with skipMetadata=false returning no metadata
(CASSANDRA-8054)
* Allow concurrent use of CQLBulkOutputFormat (CASSANDRA-7776)

2
src/java/org/apache/cassandra/db/ColumnFamilyStore.java
View File

@ -2149,7 +2149,7 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean
{
for (SSTableReader ssTable : currentView.sstables)
{
if (ssTable.isOpenEarly || (predicate != null && !predicate.apply(ssTable)))
if (ssTable.openReason == SSTableReader.OpenReason.EARLY || (predicate != null && !predicate.apply(ssTable)))
{
continue;
}

2
src/java/org/apache/cassandra/io/sstable/SSTableDeletingTask.java
View File

@ -51,7 +51,7 @@ public class SSTableDeletingTask implements Runnable
public SSTableDeletingTask(SSTableReader referent)
{
this.referent = referent;
if (referent.isOpenEarly)
if (referent.openReason == SSTableReader.OpenReason.EARLY)
{
this.desc = referent.descriptor.asType(Descriptor.Type.TEMPLINK);
this.components = Sets.newHashSet(Component.DATA, Component.PRIMARY_INDEX);

44
src/java/org/apache/cassandra/io/sstable/SSTableReader.java
View File

@ -159,7 +159,15 @@ public class SSTableReader extends SSTable
* The age is in milliseconds since epoc and is local to this host.
*/
public final long maxDataAge;
public final boolean isOpenEarly;
public enum OpenReason
{
NORMAL,
EARLY,
METADATA_CHANGE
}
public final OpenReason openReason;
// indexfile and datafile: might be null before a call to load()
private SegmentedFile ifile;
@ -338,7 +346,7 @@ public class SSTableReader extends SSTable
partitioner,
System.currentTimeMillis(),
statsMetadata,
false);
OpenReason.NORMAL);
// special implementation of load to use non-pooled SegmentedFile builders
SegmentedFile.Builder ibuilder = new BufferedSegmentedFile.Builder();
@ -387,7 +395,7 @@ public class SSTableReader extends SSTable
partitioner,
System.currentTimeMillis(),
statsMetadata,
false);
OpenReason.NORMAL);
// load index and filter
long start = System.nanoTime();
@ -467,7 +475,7 @@ public class SSTableReader extends SSTable
IFilter bf,
long maxDataAge,
StatsMetadata sstableMetadata,
boolean isOpenEarly)
OpenReason openReason)
{
assert desc != null && partitioner != null && ifile != null && dfile != null && isummary != null && bf != null && sstableMetadata != null;
return new SSTableReader(desc,
@ -479,7 +487,7 @@ public class SSTableReader extends SSTable
bf,
maxDataAge,
sstableMetadata,
isOpenEarly);
openReason);
}
@ -489,18 +497,19 @@ public class SSTableReader extends SSTable
IPartitioner partitioner,
long maxDataAge,
StatsMetadata sstableMetadata,
boolean isOpenEarly)
OpenReason openReason)
{
super(desc, components, metadata, partitioner);
this.sstableMetadata = sstableMetadata;
this.maxDataAge = maxDataAge;
this.isOpenEarly = isOpenEarly;
this.openReason = openReason;
deletingTask = new SSTableDeletingTask(this);
// Don't track read rates for tables in the system keyspace and don't bother trying to load or persist
// the read meter when in client mode
if (Keyspace.SYSTEM_KS.equals(desc.ksname) || Config.isClientMode())
// the read meter when in client mode. Also don't track reads for special operations (like early open)
// this is to avoid overflowing the executor queue (see CASSANDRA-8066)
if (Keyspace.SYSTEM_KS.equals(desc.ksname) || Config.isClientMode() || openReason != OpenReason.NORMAL)
{
readMeter = null;
readMeterSyncFuture = null;
@ -532,9 +541,9 @@ public class SSTableReader extends SSTable
IFilter bloomFilter,
long maxDataAge,
StatsMetadata sstableMetadata,
boolean isOpenEarly)
OpenReason openReason)
{
this(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, isOpenEarly);
this(desc, components, metadata, partitioner, maxDataAge, sstableMetadata, openReason);
this.ifile = ifile;
this.dfile = dfile;
@ -953,9 +962,9 @@ public class SSTableReader extends SSTable
}
}
if (readMeterSyncFuture != null)
readMeterSyncFuture.cancel(false);
SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, indexSummary.readOnlyClone(), bf, maxDataAge, sstableMetadata, isOpenEarly);
SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, indexSummary.readOnlyClone(), bf, maxDataAge, sstableMetadata,
openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
replacement.readMeterSyncFuture = this.readMeterSyncFuture;
replacement.readMeter = this.readMeter;
replacement.first = this.last.compareTo(newStart) > 0 ? newStart : this.last;
replacement.last = this.last;
@ -1015,10 +1024,9 @@ public class SSTableReader extends SSTable
StorageMetrics.load.inc(newSize - oldSize);
parent.metric.liveDiskSpaceUsed.inc(newSize - oldSize);
if (readMeterSyncFuture != null)
readMeterSyncFuture.cancel(false);
SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, newSummary, bf, maxDataAge, sstableMetadata, isOpenEarly);
SSTableReader replacement = new SSTableReader(descriptor, components, metadata, partitioner, ifile, dfile, newSummary, bf, maxDataAge, sstableMetadata,
openReason == OpenReason.EARLY ? openReason : OpenReason.METADATA_CHANGE);
replacement.readMeterSyncFuture = this.readMeterSyncFuture;
replacement.readMeter = this.readMeter;
replacement.first = this.first;
replacement.last = this.last;

4
src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
View File

@ -397,7 +397,7 @@ public class SSTableWriter extends SSTable
components, metadata,
partitioner, ifile,
dfile, iwriter.summary.build(partitioner, exclusiveUpperBoundOfReadableIndex),
iwriter.bf, maxDataAge, sstableMetadata, true);
iwriter.bf, maxDataAge, sstableMetadata, SSTableReader.OpenReason.EARLY);
// now it's open, find the ACTUAL last readable key (i.e. for which the data file has also been flushed)
sstable.first = getMinimalKey(first);
@ -448,7 +448,7 @@ public class SSTableWriter extends SSTable
iwriter.bf,
maxDataAge,
sstableMetadata,
false);
SSTableReader.OpenReason.NORMAL);
sstable.first = getMinimalKey(first);
sstable.last = getMinimalKey(last);
// try to save the summaries to disk