mirror of https://github.com/apache/cassandra
Add tool to find why expired sstables are not getting dropped
Patch by marcuse; reviewed by stefania for CASSANDRA-10015
This commit is contained in:
Marcus Eriksson
parent
de84a5c770
commit
028e7cb5af
|
|
@ -1,4 +1,5 @@
|
|||
2.0.17
|
||||
* Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
|
||||
* Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
|
||||
* Don't cast expected bf size to an int (CASSANDRA-9959)
|
||||
* Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,135 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import com.google.common.base.Throwables;
|
||||
import com.google.common.collect.ArrayListMultimap;
|
||||
import com.google.common.collect.Multimap;
|
||||
|
||||
import org.apache.cassandra.config.CFMetaData;
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.config.Schema;
|
||||
import org.apache.cassandra.db.Directories;
|
||||
import org.apache.cassandra.db.Keyspace;
|
||||
import org.apache.cassandra.io.sstable.Component;
|
||||
import org.apache.cassandra.io.sstable.Descriptor;
|
||||
import org.apache.cassandra.io.sstable.SSTableReader;
|
||||
|
||||
/**
|
||||
* During compaction we can drop entire sstables if they only contain expired tombstones and if it is guaranteed
|
||||
* to not cover anything in other sstables. An expired sstable can be blocked from getting dropped if its newest
|
||||
* timestamp is newer than the oldest data in another sstable.
|
||||
*
|
||||
* This class outputs all sstables that are blocking other sstables from getting dropped so that a user can
|
||||
* figure out why certain sstables are still on disk.
|
||||
*/
|
||||
public class SSTableExpiredBlockers
|
||||
{
|
||||
public static void main(String[] args) throws IOException
|
||||
{
|
||||
PrintStream out = System.out;
|
||||
if (args.length < 2)
|
||||
{
|
||||
out.println("Usage: sstableexpiredblockers <keyspace> <table>");
|
||||
System.exit(1);
|
||||
}
|
||||
String keyspace = args[args.length - 2];
|
||||
String columnfamily = args[args.length - 1];
|
||||
DatabaseDescriptor.loadSchemas();
|
||||
|
||||
CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
|
||||
if (metadata == null)
|
||||
throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
|
||||
keyspace,
|
||||
columnfamily));
|
||||
|
||||
Keyspace.openWithoutSSTables(keyspace);
|
||||
Directories directories = Directories.create(keyspace, columnfamily);
|
||||
Set<SSTableReader> sstables = new HashSet<>();
|
||||
|
||||
for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().skipTemporary(true).list().entrySet())
|
||||
{
|
||||
if (sstable.getKey() != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
SSTableReader reader = SSTableReader.open(sstable.getKey());
|
||||
sstables.add(reader);
|
||||
}
|
||||
catch (Throwable t)
|
||||
{
|
||||
out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA));
|
||||
Throwables.propagate(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sstables.isEmpty())
|
||||
{
|
||||
out.println("No sstables for " + keyspace + "." + columnfamily);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds();
|
||||
Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables, gcBefore);
|
||||
for (SSTableReader blocker : blockers.keySet())
|
||||
{
|
||||
out.println(String.format("%s blocks %d expired sstables from getting dropped: %s%n",
|
||||
formatForExpiryTracing(Collections.singleton(blocker)),
|
||||
blockers.get(blocker).size(),
|
||||
formatForExpiryTracing(blockers.get(blocker))));
|
||||
}
|
||||
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
public static Multimap<SSTableReader, SSTableReader> checkForExpiredSSTableBlockers(Iterable<SSTableReader> sstables, int gcBefore)
|
||||
{
|
||||
Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
|
||||
for (SSTableReader sstable : sstables)
|
||||
{
|
||||
if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
|
||||
{
|
||||
for (SSTableReader potentialBlocker : sstables)
|
||||
{
|
||||
if (!potentialBlocker.equals(sstable) &&
|
||||
potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp() &&
|
||||
potentialBlocker.getSSTableMetadata().maxLocalDeletionTime > gcBefore)
|
||||
blockers.put(potentialBlocker, sstable);
|
||||
}
|
||||
}
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
private static String formatForExpiryTracing(Iterable<SSTableReader> sstables)
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
for (SSTableReader sstable : sstables)
|
||||
sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(", ");
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -26,6 +26,7 @@ import java.util.Collections;
|
|||
import java.util.Set;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
|
||||
import com.google.common.collect.Multimap;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
|
@ -41,6 +42,7 @@ import org.apache.cassandra.db.RowMutation;
|
|||
import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
|
||||
import org.apache.cassandra.io.sstable.SSTableReader;
|
||||
import org.apache.cassandra.io.sstable.SSTableScanner;
|
||||
import org.apache.cassandra.tools.SSTableExpiredBlockers;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
|
@ -197,4 +199,31 @@ public class TTLExpiryTest extends SchemaLoader
|
|||
cfs.clearUnsafe();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckForExpiredSSTableBlockers() throws InterruptedException
|
||||
{
|
||||
String KEYSPACE1 = "Keyspace1";
|
||||
ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
|
||||
cfs.truncateBlocking();
|
||||
cfs.disableAutoCompaction();
|
||||
cfs.metadata.gcGraceSeconds(0);
|
||||
|
||||
RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
|
||||
rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
|
||||
rm.applyUnsafe();
|
||||
cfs.forceBlockingFlush();
|
||||
SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
|
||||
rm.delete("Standard1", System.currentTimeMillis());
|
||||
rm.applyUnsafe();
|
||||
cfs.forceBlockingFlush();
|
||||
}
|
||||
Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100);
|
||||
assertEquals(1, blockers.keySet().size());
|
||||
assertTrue(blockers.keySet().contains(blockingSSTable));
|
||||
assertEquals(10, blockers.get(blockingSSTable).size());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
|
||||
for include in "`dirname $0`/cassandra.in.sh" \
|
||||
"$HOME/.cassandra.in.sh" \
|
||||
/usr/share/cassandra/cassandra.in.sh \
|
||||
/usr/local/share/cassandra/cassandra.in.sh \
|
||||
/opt/cassandra/cassandra.in.sh; do
|
||||
if [ -r $include ]; then
|
||||
. $include
|
||||
break
|
||||
fi
|
||||
done
|
||||
elif [ -r $CASSANDRA_INCLUDE ]; then
|
||||
. $CASSANDRA_INCLUDE
|
||||
fi
|
||||
|
||||
|
||||
# Use JAVA_HOME if set, otherwise look for java in PATH
|
||||
if [ -x $JAVA_HOME/bin/java ]; then
|
||||
JAVA=$JAVA_HOME/bin/java
|
||||
else
|
||||
JAVA=`which java`
|
||||
fi
|
||||
|
||||
if [ -z "$CLASSPATH" ]; then
|
||||
echo "You must set the CLASSPATH var" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"$JAVA" -cp "$CLASSPATH" -Dstorage-config=$CASSANDRA_CONF \
|
||||
-Dlog4j.configuration=log4j-tools.properties \
|
||||
org.apache.cassandra.tools.SSTableExpiredBlockers "$@"
|
||||
Loading…
Reference in New Issue