natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch cassandra-3.0 into cassandra-3.11

This commit is contained in:
Benjamin Lerer 2017-08-08 17:09:06 +02:00
parent 47a2839bf0 1a70dede3a
commit 303dba6504
4 changed files with 191 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -20,6 +20,7 @@ Merged from 3.0:
* Fix secondary index queries on COMPACT tables (CASSANDRA-13627)
* Nodetool listsnapshots output is missing a newline, if there are no snapshots (CASSANDRA-13568)
Merged from 2.2:
* Prevent integer overflow on exabyte filesystems (CASSANDRA-13067)
* Fix queries with LIMIT and filtering on clustering columns (CASSANDRA-11223)
* Fix potential NPE when resume bootstrap fails (CASSANDRA-13272)
* Fix toJSONString for the UDT, tuple and collection types (CASSANDRA-13592)

26
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import java.net.*;
import java.nio.file.FileStore;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -65,6 +64,8 @@ import org.apache.cassandra.utils.FBUtilities;
import org.apache.commons.lang3.StringUtils;
import static org.apache.cassandra.io.util.FileUtils.ONE_GB;
public class DatabaseDescriptor
{
private static final Logger logger = LoggerFactory.getLogger(DatabaseDescriptor.class);
@ -453,7 +454,7 @@ public class DatabaseDescriptor
try
{
// use 1/4 of available space. See discussion on #10013 and #10199
minSize = Ints.checkedCast((guessFileStore(conf.commitlog_directory).getTotalSpace() / 1048576) / 4);
minSize = Ints.saturatedCast((guessFileStore(conf.commitlog_directory).getTotalSpace() / 1048576) / 4);
}
catch (IOException e)
{
@ -527,7 +528,7 @@ public class DatabaseDescriptor
try
{
dataFreeBytes += guessFileStore(datadir).getUnallocatedSpace();
dataFreeBytes = saturatedSum(dataFreeBytes, guessFileStore(datadir).getUnallocatedSpace());
}
catch (IOException e)
{
@ -536,11 +537,10 @@ public class DatabaseDescriptor
datadir), e);
}
}
if (dataFreeBytes < 64L * 1024 * 1048576) // 64 GB
if (dataFreeBytes < 64 * ONE_GB) // 64 GB
logger.warn("Only {} free across all data volumes. Consider adding more capacity to your cluster or removing obsolete snapshots",
FBUtilities.prettyPrintMemory(dataFreeBytes));
if (conf.commitlog_directory.equals(conf.saved_caches_directory))
throw new ConfigurationException("saved_caches_directory must not be the same as the commitlog_directory", false);
if (conf.commitlog_directory.equals(conf.hints_directory))
@ -980,6 +980,20 @@ public class DatabaseDescriptor
paritionerName = partitioner.getClass().getCanonicalName();
}
/**
* Computes the sum of the 2 specified positive values returning {@code Long.MAX_VALUE} if the sum overflow.
*
* @param left the left operand
* @param right the right operand
* @return the sum of the 2 specified positive values of {@code Long.MAX_VALUE} if the sum overflow.
*/
private static long saturatedSum(long left, long right)
{
assert left >= 0 && right >= 0;
long sum = left + right;
return sum < 0 ? Long.MAX_VALUE : sum;
}
private static FileStore guessFileStore(String dir) throws IOException
{
Path path = Paths.get(dir);
@ -987,7 +1001,7 @@ public class DatabaseDescriptor
{
try
{
return Files.getFileStore(path);
return FileUtils.getFileStore(path);
}
catch (IOException e)
{

3
src/java/org/apache/cassandra/db/Directories.java
View File

@ -559,7 +559,7 @@ public class Directories
public long getAvailableSpace()
{
long availableSpace = location.getUsableSpace() - DatabaseDescriptor.getMinFreeSpacePerDriveInBytes();
long availableSpace = FileUtils.getUsableSpace(location) - DatabaseDescriptor.getMinFreeSpacePerDriveInBytes();
return availableSpace > 0 ? availableSpace : 0;
}
@ -572,7 +572,6 @@ public class Directories
DataDirectory that = (DataDirectory) o;
return location.equals(that.location);
}
@Override

181
src/java/org/apache/cassandra/io/util/FileUtils.java
View File

@ -24,6 +24,8 @@ import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.FileAttributeView;
import java.nio.file.attribute.FileStoreAttributeView;
import java.text.DecimalFormat;
import java.util.Arrays;
import java.util.Collections;
@ -54,10 +56,10 @@ public final class FileUtils
public static final Charset CHARSET = StandardCharsets.UTF_8;
private static final Logger logger = LoggerFactory.getLogger(FileUtils.class);
private static final double KB = 1024d;
private static final double MB = 1024*1024d;
private static final double GB = 1024*1024*1024d;
private static final double TB = 1024*1024*1024*1024d;
public static final long ONE_KB = 1024;
public static final long ONE_MB = 1024 * ONE_KB;
public static final long ONE_GB = 1024 * ONE_MB;
public static final long ONE_TB = 1024 * ONE_GB;
private static final DecimalFormat df = new DecimalFormat("#.##");
public static final boolean isCleanerAvailable;
@ -415,27 +417,27 @@ public final class FileUtils
public static String stringifyFileSize(double value)
{
double d;
if ( value >= TB )
if ( value >= ONE_TB )
{
d = value / TB;
d = value / ONE_TB;
String val = df.format(d);
return val + " TiB";
}
else if ( value >= GB )
else if ( value >= ONE_GB )
{
d = value / GB;
d = value / ONE_GB;
String val = df.format(d);
return val + " GiB";
}
else if ( value >= MB )
else if ( value >= ONE_MB )
{
d = value / MB;
d = value / ONE_MB;
String val = df.format(d);
return val + " MiB";
}
else if ( value >= KB )
else if ( value >= ONE_KB )
{
d = value / KB;
d = value / ONE_KB;
String val = df.format(d);
return val + " KiB";
}
@ -608,4 +610,159 @@ public final class FileUtils
{
fsErrorHandler.getAndSet(Optional.ofNullable(handler));
}
/**
* Returns the size of the specified partition.
* <p>This method handles large file system by returning {@code Long.MAX_VALUE} if the size overflow.
* See <a href='https://bugs.openjdk.java.net/browse/JDK-8179320'>JDK-8179320</a> for more information.</p>
*
* @param file the partition
* @return the size, in bytes, of the partition or {@code 0L} if the abstract pathname does not name a partition
*/
public static long getTotalSpace(File file)
{
return handleLargeFileSystem(file.getTotalSpace());
}
/**
* Returns the number of unallocated bytes on the specified partition.
* <p>This method handles large file system by returning {@code Long.MAX_VALUE} if the number of unallocated bytes
* overflow. See <a href='https://bugs.openjdk.java.net/browse/JDK-8179320'>JDK-8179320</a> for more information</p>
*
* @param file the partition
* @return the number of unallocated bytes on the partition or {@code 0L}
* if the abstract pathname does not name a partition.
*/
public static long getFreeSpace(File file)
{
return handleLargeFileSystem(file.getFreeSpace());
}
/**
* Returns the number of available bytes on the specified partition.
* <p>This method handles large file system by returning {@code Long.MAX_VALUE} if the number of available bytes
* overflow. See <a href='https://bugs.openjdk.java.net/browse/JDK-8179320'>JDK-8179320</a> for more information</p>
*
* @param file the partition
* @return the number of available bytes on the partition or {@code 0L}
* if the abstract pathname does not name a partition.
*/
public static long getUsableSpace(File file)
{
return handleLargeFileSystem(file.getUsableSpace());
}
/**
* Returns the {@link FileStore} representing the file store where a file
* is located. This {@link FileStore} handles large file system by returning {@code Long.MAX_VALUE}
* from {@code FileStore#getTotalSpace()}, {@code FileStore#getUnallocatedSpace()} and {@code FileStore#getUsableSpace()}
* it the value is bigger than {@code Long.MAX_VALUE}. See <a href='https://bugs.openjdk.java.net/browse/JDK-8162520'>JDK-8162520</a>
* for more information.
*
* @param path the path to the file
* @return the file store where the file is stored
*/
public static FileStore getFileStore(Path path) throws IOException
{
return new SafeFileStore(Files.getFileStore(path));
}
/**
* Handle large file system by returning {@code Long.MAX_VALUE} when the size overflows.
* @param size returned by the Java's FileStore methods
* @return the size or {@code Long.MAX_VALUE} if the size was bigger than {@code Long.MAX_VALUE}
*/
private static long handleLargeFileSystem(long size)
{
return size < 0 ? Long.MAX_VALUE : size;
}
/**
* Private constructor as the class contains only static methods.
*/
private FileUtils()
{
}
/**
* FileStore decorator used to safely handle large file system.
*
* <p>Java's FileStore methods (getTotalSpace/getUnallocatedSpace/getUsableSpace) are limited to reporting bytes as
* signed long (2^63-1), if the filesystem is any bigger, then the size overflows. {@code SafeFileStore} will
* return {@code Long.MAX_VALUE} if the size overflow.</p>
*
* @see https://bugs.openjdk.java.net/browse/JDK-8162520.
*/
private static final class SafeFileStore extends FileStore
{
/**
* The decorated {@code FileStore}
*/
private final FileStore fileStore;
public SafeFileStore(FileStore fileStore)
{
this.fileStore = fileStore;
}
@Override
public String name()
{
return fileStore.name();
}
@Override
public String type()
{
return fileStore.type();
}
@Override
public boolean isReadOnly()
{
return fileStore.isReadOnly();
}
@Override
public long getTotalSpace() throws IOException
{
return handleLargeFileSystem(fileStore.getTotalSpace());
}
@Override
public long getUsableSpace() throws IOException
{
return handleLargeFileSystem(fileStore.getUsableSpace());
}
@Override
public long getUnallocatedSpace() throws IOException
{
return handleLargeFileSystem(fileStore.getUnallocatedSpace());
}
@Override
public boolean supportsFileAttributeView(Class<? extends FileAttributeView> type)
{
return fileStore.supportsFileAttributeView(type);
}
@Override
public boolean supportsFileAttributeView(String name)
{
return fileStore.supportsFileAttributeView(name);
}
@Override
public <V extends FileStoreAttributeView> V getFileStoreAttributeView(Class<V> type)
{
return fileStore.getFileStoreAttributeView(type);
}
@Override
public Object getAttribute(String attribute) throws IOException
{
return fileStore.getAttribute(attribute);
}
}
}