natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Shorten SSTable path 

patch by yukim; reviewed by Josh McKenzie for CASSANDRA-6962
This commit is contained in:
Yuki Morishita 2014-07-07 10:53:01 -05:00
parent 0ae5def831
commit d13a996e45
11 changed files with 268 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -9,6 +9,7 @@
* Make incremental repair default (CASSANDRA-7250)
* Enable code coverage thru JaCoCo (CASSANDRA-7226)
* Switch external naming of 'column families' to 'tables' (CASSANDRA-4369)
* Shorten SSTable path (CASSANDRA-6962)
2.1.1

7
NEWS.txt
View File

@ -16,6 +16,13 @@ using the provided 'sstableupgrade' tool.
3.0
===
New features
------------
- SSTable file name is changed. Now you don't have Keyspace/CF name
in file name. Also, secondary index has its own directory under parent's
directory.
Upgrading
---------
- CQL2 has been removed entirely in this release (previously deprecated

64
src/java/org/apache/cassandra/db/Directories.java
View File

@ -191,7 +191,15 @@ public class Directories
String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(metadata.cfId));
int idx = metadata.cfName.indexOf(SECONDARY_INDEX_NAME_SEPARATOR);
// secondary indicies go in the same directory as the base cf
String directoryName = idx > 0 ? metadata.cfName.substring(0, idx) + "-" + cfId : metadata.cfName + "-" + cfId;
String directoryName;
if (idx >= 0)
{
directoryName = metadata.cfName.substring(0, idx) + "-" + cfId + File.separator + metadata.cfName.substring(idx);
}
else
{
directoryName = metadata.cfName + "-" + cfId;
}
this.dataPaths = new File[dataDirectories.length];
// If upgraded from version less than 2.1, use existing directories
@ -318,7 +326,19 @@ public class Directories
public static File getSnapshotDirectory(Descriptor desc, String snapshotName)
{
return getOrCreate(desc.directory, SNAPSHOT_SUBDIR, snapshotName);
return getSnapshotDirectory(desc.directory, snapshotName);
}
public static File getSnapshotDirectory(File location, String snapshotName)
{
if (location.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR))
{
return getOrCreate(location.getParentFile(), SNAPSHOT_SUBDIR, snapshotName, location.getName());
}
else
{
return getOrCreate(location, SNAPSHOT_SUBDIR, snapshotName);
}
}
public File getSnapshotManifestFile(String snapshotName)
@ -328,7 +348,19 @@ public class Directories
public static File getBackupsDirectory(Descriptor desc)
{
return getOrCreate(desc.directory, BACKUPS_SUBDIR);
return getBackupsDirectory(desc.directory);
}
public static File getBackupsDirectory(File location)
{
if (location.getName().startsWith(SECONDARY_INDEX_NAME_SEPARATOR))
{
return getOrCreate(location.getParentFile(), BACKUPS_SUBDIR, location.getName());
}
else
{
return getOrCreate(location, BACKUPS_SUBDIR);
}
}
public SSTableLister sstableLister()
@ -439,7 +471,7 @@ public class Directories
if (snapshotName != null)
{
new File(location, join(SNAPSHOT_SUBDIR, snapshotName)).listFiles(getFilter());
getSnapshotDirectory(location, snapshotName).listFiles(getFilter());
continue;
}
@ -447,28 +479,29 @@ public class Directories
location.listFiles(getFilter());
if (includeBackups)
new File(location, BACKUPS_SUBDIR).listFiles(getFilter());
getBackupsDirectory(location).listFiles(getFilter());
}
filtered = true;
}
private FileFilter getFilter()
{
// Note: the prefix needs to include cfname + separator to distinguish between a cfs and it's secondary indexes
final String sstablePrefix = getSSTablePrefix();
return new FileFilter()
{
// This function always return false since accepts adds to the components map
public boolean accept(File file)
{
// we are only interested in the SSTable files that belong to the specific ColumnFamily
if (file.isDirectory() || !file.getName().startsWith(sstablePrefix))
if (file.isDirectory())
return false;
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null)
return false;
// we are only interested in the SSTable files that belong to the specific ColumnFamily
if (!pair.left.ksname.equals(metadata.ksName) || !pair.left.cfname.equals(metadata.cfName))
return false;
if (skipTemporary && pair.left.type.isTemporary)
return false;
@ -569,11 +602,6 @@ public class Directories
return result;
}
private String getSSTablePrefix()
{
return metadata.ksName + Component.separator + metadata.cfName + Component.separator;
}
public long getTrueAllocatedSizeIn(File input)
{
if (!input.isDirectory())
@ -662,7 +690,6 @@ public class Directories
private final AtomicLong size = new AtomicLong(0);
private final Set<String> visited = newHashSet(); //count each file only once
private final Set<String> alive;
private final String prefix = getSSTablePrefix();
public TrueFilesSizeVisitor()
{
@ -675,8 +702,11 @@ public class Directories
private boolean isAcceptable(Path file)
{
String fileName = file.toFile().getName();
return fileName.startsWith(prefix)
String fileName = file.toFile().getName();
Pair<Descriptor, Component> pair = SSTable.tryComponentFromFilename(file.getParent().toFile(), fileName);
return pair != null
&& pair.left.ksname.equals(metadata.ksName)
&& pair.left.cfname.equals(metadata.cfName)
&& !visited.contains(fileName)
&& !alive.contains(fileName);
}

2
src/java/org/apache/cassandra/db/compaction/CompactionManager.java
View File

@ -479,7 +479,7 @@ public class CompactionManager implements CompactionManagerMBean
}
// group by keyspace/columnfamily
ColumnFamilyStore cfs = Keyspace.open(desc.ksname).getColumnFamilyStore(desc.cfname);
descriptors.put(cfs, cfs.directories.find(filename.trim()));
descriptors.put(cfs, cfs.directories.find(new File(filename.trim()).getName()));
}
List<Future<?>> futures = new ArrayList<>();

3
src/java/org/apache/cassandra/db/compaction/CompactionManagerMBean.java
View File

@ -66,7 +66,8 @@ public interface CompactionManagerMBean
* If you do so, user defined compaction is performed several times to the groups of files
* in the same keyspace/columnfamily.
*
* @param dataFiles a comma separated list of sstable filename to compact
* @param dataFiles a comma separated list of sstable file to compact.
* must contain keyspace and columnfamily name in path(for 2.1+) or file name itself.
*/
public void forceUserDefinedCompaction(String dataFiles);

108
src/java/org/apache/cassandra/io/sstable/Descriptor.java
View File

@ -18,10 +18,13 @@
package org.apache.cassandra.io.sstable;
import java.io.File;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.StringTokenizer;
import com.google.common.base.Objects;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.io.sstable.metadata.IMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.LegacyMetadataSerializer;
import org.apache.cassandra.io.sstable.metadata.MetadataSerializer;
@ -60,6 +63,7 @@ public class Descriptor
// jb (2.0.1): switch from crc32 to adler32 for compression checksums
// checksum the compressed data
// ka (2.1.0): new Statistics.db file format
// new file name format
// index summaries can be downsampled and the sampling level is persisted
// switch uncompressed checksums to adler32
// tracks presense of legacy (local and remote) counter shards
@ -75,6 +79,7 @@ public class Descriptor
public final boolean hasAllAdlerChecksums;
public final boolean hasRepairedAt;
public final boolean tracksLegacyCounterShards;
public final boolean newFileName;
public Version(String version)
{
@ -86,6 +91,7 @@ public class Descriptor
hasAllAdlerChecksums = version.compareTo("ka") >= 0;
hasRepairedAt = version.compareTo("ka") >= 0;
tracksLegacyCounterShards = version.compareTo("ka") >= 0;
newFileName = version.compareTo("ka") >= 0;
}
/**
@ -188,8 +194,11 @@ public class Descriptor
private void appendFileName(StringBuilder buff)
{
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
if (!version.newFileName)
{
buff.append(ksname).append(separator);
buff.append(cfname).append(separator);
}
if (type.isTemporary)
buff.append(type.marker).append(separator);
buff.append(version).append(separator);
@ -230,13 +239,20 @@ public class Descriptor
return fromFilename(file.getParentFile(), file.getName(), skipComponent).left;
}
public static Pair<Descriptor,String> fromFilename(File directory, String name)
public static Pair<Descriptor, String> fromFilename(File directory, String name)
{
return fromFilename(directory, name, false);
}
/**
* Filename of the form "<ksname>-<cfname>-[tmp-][<version>-]<gen>-<component>"
* Filename of the form is vary by version:
*
* <ul>
* <li>&lt;ksname&gt;-&lt;cfname&gt;-(tmp-)?&lt;version&gt;-&lt;gen&gt;-&lt;component&gt; for cassandra 2.0 and before</li>
* <li>(&lt;tmp marker&gt;-)?&lt;version&gt;-&lt;gen&gt;-&lt;component&gt; for cassandra 2.1 and later</li>
* </ul>
*
* If this is for SSTable of secondary index, directory should ends with index name for 2.1+.
*
* @param directory The directory of the SSTable files
* @param name The name of the SSTable file
@ -244,43 +260,79 @@ public class Descriptor
*
* @return A Descriptor for the SSTable, and the Component remainder.
*/
public static Pair<Descriptor,String> fromFilename(File directory, String name, boolean skipComponent)
public static Pair<Descriptor, String> fromFilename(File directory, String name, boolean skipComponent)
{
File parentDirectory = directory != null ? directory : new File(".");
// tokenize the filename
StringTokenizer st = new StringTokenizer(name, String.valueOf(separator));
String nexttok;
// all filenames must start with keyspace and column family
String ksname = st.nextToken();
String cfname = st.nextToken();
// optional temporary marker
nexttok = st.nextToken();
Type type = Type.FINAL;
if (nexttok.equals(Type.TEMP.marker))
// read tokens backwards to determine version
Deque<String> tokenStack = new ArrayDeque<>();
while (st.hasMoreTokens())
{
type = Type.TEMP;
nexttok = st.nextToken();
}
else if (nexttok.equals(Type.TEMPLINK.marker))
{
type = Type.TEMPLINK;
nexttok = st.nextToken();
tokenStack.push(st.nextToken());
}
// component suffix
String component = skipComponent ? null : tokenStack.pop();
// generation
int generation = Integer.parseInt(tokenStack.pop());
// version
nexttok = tokenStack.pop();
if (!Version.validate(nexttok))
throw new UnsupportedOperationException("SSTable " + name + " is too old to open. Upgrade to 2.0 first, and run upgradesstables");
Version version = new Version(nexttok);
nexttok = st.nextToken();
int generation = Integer.parseInt(nexttok);
// optional temporary marker
Type type = Type.FINAL;
nexttok = tokenStack.peek();
if (Type.TEMP.marker.equals(nexttok))
{
type = Type.TEMP;
tokenStack.pop();
}
else if (Type.TEMPLINK.marker.equals(nexttok))
{
type = Type.TEMPLINK;
tokenStack.pop();
}
// component suffix
String component = null;
if (!skipComponent)
component = st.nextToken();
directory = directory != null ? directory : new File(".");
return Pair.create(new Descriptor(version, directory, ksname, cfname, generation, type), component);
// ks/cf names
String ksname, cfname;
if (version.newFileName)
{
// for 2.1+ read ks and cf names from directory
File cfDirectory = parentDirectory;
// check if this is secondary index
String indexName = "";
if (cfDirectory.getName().startsWith(Directories.SECONDARY_INDEX_NAME_SEPARATOR))
{
indexName = cfDirectory.getName();
cfDirectory = cfDirectory.getParentFile();
}
if (cfDirectory.getName().equals(Directories.BACKUPS_SUBDIR))
{
cfDirectory = cfDirectory.getParentFile();
}
else if (cfDirectory.getParentFile().getName().equals(Directories.SNAPSHOT_SUBDIR))
{
cfDirectory = cfDirectory.getParentFile().getParentFile();
}
cfname = cfDirectory.getName().split("-")[0] + indexName;
ksname = cfDirectory.getParentFile().getName();
}
else
{
cfname = tokenStack.pop();
ksname = tokenStack.pop();
}
assert tokenStack.isEmpty() : "Invalid file name " + name + " in " + directory;
return Pair.create(new Descriptor(version, parentDirectory, ksname, cfname, generation, type), component);
}
/**

9
src/java/org/apache/cassandra/io/sstable/SSTable.java
View File

@ -150,18 +150,17 @@ public abstract class SSTable
}
/**
* @return A Descriptor,Component pair. If component is of unknown type, returns CUSTOM component.
* @return Descriptor and Component pair. null if given file is not acceptable as SSTable component.
* If component is of unknown type, returns CUSTOM component.
*/
public static Pair<Descriptor,Component> tryComponentFromFilename(File dir, String name)
public static Pair<Descriptor, Component> tryComponentFromFilename(File dir, String name)
{
try
{
return Component.fromFilename(dir, name);
}
catch (NoSuchElementException e)
catch (Throwable e)
{
// A NoSuchElementException is thrown if the name does not match the Descriptor format
// This is the less impacting change (all calls to this method test for null return)
return null;
}
}

2
test/unit/org/apache/cassandra/db/DirectoriesTest.java
View File

@ -173,7 +173,7 @@ public class DirectoriesTest
{
if (f.getPath().contains(Directories.SNAPSHOT_SUBDIR) || f.getPath().contains(Directories.BACKUPS_SUBDIR))
assert !listed.contains(f) : f + " should not be listed";
else if (f.getName().contains("-tmp-"))
else if (f.getName().contains("tmp-"))
assert !listed.contains(f) : f + " should not be listed";
else
assert listed.contains(f) : f + " is missing";

2
test/unit/org/apache/cassandra/db/compaction/CompactionsTest.java
View File

@ -367,7 +367,7 @@ public class CompactionsTest
SSTableReader sstable = sstables.iterator().next();
int prevGeneration = sstable.descriptor.generation;
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getName();
String file = new File(sstable.descriptor.filenameFor(Component.DATA)).getAbsolutePath();
// submit user defined compaction on flushed sstable
CompactionManager.instance.forceUserDefinedCompaction(file);
// wait until user defined compaction finishes

120
test/unit/org/apache/cassandra/io/sstable/DescriptorTest.java Normal file
View File

@ -0,0 +1,120 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.io.sstable;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import org.apache.cassandra.db.Directories;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.Pair;
import static org.junit.Assert.*;
public class DescriptorTest
{
private final String ksname = "ks";
private final String cfname = "cf";
private final String cfId = ByteBufferUtil.bytesToHex(ByteBufferUtil.bytes(UUID.randomUUID()));
private final File tempDataDir;
public DescriptorTest() throws IOException
{
// create CF directories, one without CFID and one with it
tempDataDir = File.createTempFile("DescriptorTest", null).getParentFile();
}
@Test
public void testFromFilename() throws Exception
{
File cfIdDir = new File(tempDataDir.getAbsolutePath() + File.separator + ksname + File.separator + cfname + '-' + cfId);
testFromFilenameFor(cfIdDir);
}
@Test
public void testFromFilenameInBackup() throws Exception
{
File backupDir = new File(StringUtils.join(new String[]{tempDataDir.getAbsolutePath(), ksname, cfname + '-' + cfId, Directories.BACKUPS_SUBDIR}, File.separator));
testFromFilenameFor(backupDir);
}
@Test
public void testFromFilenameInSnapshot() throws Exception
{
File snapshotDir = new File(StringUtils.join(new String[]{tempDataDir.getAbsolutePath(), ksname, cfname + '-' + cfId, Directories.SNAPSHOT_SUBDIR, "snapshot_name"}, File.separator));
testFromFilenameFor(snapshotDir);
}
@Test
public void testFromFilenameInLegacyDirectory() throws Exception
{
File cfDir = new File(tempDataDir.getAbsolutePath() + File.separator + ksname + File.separator + cfname);
testFromFilenameFor(cfDir);
}
private void testFromFilenameFor(File dir)
{
// normal
checkFromFilename(new Descriptor(dir, ksname, cfname, 1, Descriptor.Type.FINAL), false);
// skip component (for streaming lock file)
checkFromFilename(new Descriptor(dir, ksname, cfname, 2, Descriptor.Type.FINAL), true);
// tmp
checkFromFilename(new Descriptor(dir, ksname, cfname, 3, Descriptor.Type.TEMP), false);
// secondary index
String idxName = "myidx";
File idxDir = new File(dir.getAbsolutePath() + File.separator + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName);
checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 4, Descriptor.Type.FINAL), false);
// secondary index tmp
checkFromFilename(new Descriptor(idxDir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 5, Descriptor.Type.TEMP), false);
// legacy version
checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 1, Descriptor.Type.FINAL), false);
// legacy tmp
checkFromFilename(new Descriptor("ja", dir, ksname, cfname, 2, Descriptor.Type.TEMP), false);
// legacy secondary index
checkFromFilename(new Descriptor("ja", dir, ksname, cfname + Directories.SECONDARY_INDEX_NAME_SEPARATOR + idxName, 3, Descriptor.Type.FINAL), false);
}
private void checkFromFilename(Descriptor original, boolean skipComponent)
{
File file = new File(skipComponent ? original.baseFilename() : original.filenameFor(Component.DATA));
Pair<Descriptor, String> pair = Descriptor.fromFilename(file.getParentFile(), file.getName(), skipComponent);
Descriptor desc = pair.left;
assertEquals(original.directory, desc.directory);
assertEquals(original.ksname, desc.ksname);
assertEquals(original.cfname, desc.cfname);
assertEquals(original.version, desc.version);
assertEquals(original.generation, desc.generation);
assertEquals(original.type, desc.type);
if (skipComponent)
{
assertNull(pair.right);
}
else
{
assertEquals(Component.DATA.name(), pair.right);
}
}
}

8
test/unit/org/apache/cassandra/io/sstable/SSTableUtils.java
View File

@ -69,10 +69,10 @@ public class SSTableUtils
if(!tempdir.delete() || !tempdir.mkdir())
throw new IOException("Temporary directory creation failed.");
tempdir.deleteOnExit();
File keyspaceDir = new File(tempdir, keyspaceName);
keyspaceDir.mkdir();
keyspaceDir.deleteOnExit();
File datafile = new File(new Descriptor(keyspaceDir, keyspaceName, cfname, generation, Descriptor.Type.FINAL).filenameFor("Data.db"));
File cfDir = new File(tempdir, keyspaceName + File.separator + cfname);
cfDir.mkdirs();
cfDir.deleteOnExit();
File datafile = new File(new Descriptor(cfDir, keyspaceName, cfname, generation, Descriptor.Type.FINAL).filenameFor("Data.db"));
if (!datafile.createNewFile())
throw new IOException("unable to create file " + datafile);
datafile.deleteOnExit();