natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sstablesplit tool unit testing 

patch by Berenguer Blasi; reviewed by Yifan Cai, Mick Semb Wever for CASSANDRA-16012
This commit is contained in:
Bereng 2020-10-01 10:56:17 +02:00 committed by Mick Semb Wever
parent 9347805e60
commit 4564e10268
No known key found for this signature in database
GPG Key ID: E91335D77E3E87CB
3 changed files with 194 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/java/org/apache/cassandra/tools/StandaloneSplitter.java
View File

@ -26,6 +26,7 @@ import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.commons.cli.*;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.compaction.OperationType;
import org.apache.cassandra.db.Directories;
@ -51,7 +52,10 @@ public class StandaloneSplitter
public static void main(String args[])
{
Options options = Options.parseArgs(args);
Util.initDatabaseDescriptor();
if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
else
Util.initDatabaseDescriptor();
try
{

1
src/java/org/apache/cassandra/tools/Util.java
View File

@ -54,6 +54,7 @@ import com.google.common.collect.Lists;
@SuppressWarnings("serial")
public final class Util
{
static final String ALLOW_TOOL_REINIT_FOR_TEST = Util.class.getName() + "ALLOW_TOOL_REINIT_FOR_TEST"; // Necessary for testing
static final String RESET = "\u001B[0m";
static final String BLUE = "\u001B[34m";
static final String CYAN = "\u001B[36m";

188
test/unit/org/apache/cassandra/tools/StandaloneSplitterWithCQLTesterTest.java Normal file
View File

@ -0,0 +1,188 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.cassandra.tools;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.google.common.io.Files;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.apache.cassandra.OrderedJUnit4ClassRunner;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.tools.ToolRunner.ToolResult;
import org.assertj.core.api.Assertions;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@RunWith(OrderedJUnit4ClassRunner.class)
public class StandaloneSplitterWithCQLTesterTest extends CQLTester
{
private static String sstableFileName;
private static File sstablesDir;
private static File sstablesBackupDir;
private static List<File> origSstables;
// CQLTester post test method cleanup needs to be avoided by overriding as it'd clean all sstables, env, etc.
@Override
public void afterTest() throws Throwable
{
}
@Test
public void setupEnv() throws Throwable
{
// Stop the server after setup as we're going to be changing things under it's feet
setupTestSstables();
tearDownClass();
}
@Test
public void testMinFileSizeCheck() throws Throwable
{
restoreOrigSstables();
ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, sstableFileName);
Assertions.assertThat(tool.getStdout()).contains("is less than the split size");
assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
}
@Test
public void testSplittingSSTable() throws Throwable
{
restoreOrigSstables();
ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, "-s", "1", sstableFileName);
List<File> splitFiles = Arrays.asList(sstablesDir.listFiles());
splitFiles.stream().forEach(f -> {
if (f.getName().endsWith("Data.db") && !origSstables.contains(f))
assertTrue(f.getName() + " is way bigger than 1MB: [" + f.length() + "] bytes",
f.length() <= 1024 * 1024 * 1.2); //give a 20% margin on size check
});
assertTrue(origSstables.size() < splitFiles.size());
Assertions.assertThat(tool.getStdout()).contains("sstables snapshotted into");
assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
}
@Test
public void testSplittingMultipleSSTables() throws Throwable
{
restoreOrigSstables();
ArrayList<String> args = new ArrayList<>(Arrays.asList("-s", "1"));
args.addAll(Arrays.asList(sstablesDir.listFiles())
.stream()
.map(f -> f.getAbsolutePath())
.collect(Collectors.toList()));
ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, args.toArray(new String[args.size()]));
List<File> splitFiles = Arrays.asList(sstablesDir.listFiles());
splitFiles.stream().forEach(f -> {
if (f.getName().endsWith("Data.db") && !origSstables.contains(f))
assertTrue(f.getName() + " is way bigger than 1MB: [" + f.length() + "] bytes",
f.length() <= 1024 * 1024 * 1.2); //give a 20% margin on size check
});
assertTrue(origSstables.size() < splitFiles.size());
assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
}
@Test
public void testNoSnapshotOption() throws Throwable
{
restoreOrigSstables();
ToolResult tool = ToolRunner.invokeClass(StandaloneSplitter.class, "-s", "1", "--no-snapshot", sstableFileName);
assertTrue(origSstables.size() < Arrays.asList(sstablesDir.listFiles()).size());
assertTrue(tool.getStdout(), tool.getStdout().isEmpty());
assertTrue(tool.getCleanedStderr(), tool.getCleanedStderr().isEmpty());
assertEquals(0, tool.getExitCode());
}
@Test
public void cleanEnv() throws Throwable
{
super.afterTest();
System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
}
private void setupTestSstables() throws Throwable
{
createTable("CREATE TABLE %s (id text primary key, val text)");
for (int i = 0; i < 100000; i++)
executeFormattedQuery(formatQuery("INSERT INTO %s (id, val) VALUES (?, ?)"), "mockData" + i, "mockData" + i);
ColumnFamilyStore cfs = getCurrentColumnFamilyStore();
cfs.forceBlockingFlush();
Set<SSTableReader> sstables = cfs.getLiveSSTables();
sstableFileName = sstables.iterator().next().getFilename();
assertTrue("Generated sstable must be at least 1MB", (new File(sstableFileName)).length() > 1024*1024);
sstablesDir = new File(sstableFileName).getParentFile();
sstablesBackupDir = new File(sstablesDir.getAbsolutePath() + "/testbackup");
sstablesBackupDir.mkdir();
origSstables = Arrays.asList(sstablesDir.listFiles());
// Back up orig sstables
origSstables.stream().forEach(f -> {
if (f.isFile())
try
{
Files.copy(f, new File(sstablesBackupDir.getAbsolutePath() + "/" + f.getName()));
}
catch(IOException e)
{
throw new RuntimeException(e);
}
});
System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing
}
private void restoreOrigSstables()
{
Arrays.asList(sstablesDir.listFiles()).stream().forEach(f -> {
if (f.isFile())
f.delete();
});
Arrays.asList(sstablesBackupDir.listFiles()).stream().forEach(f -> {
if (f.isFile())
try
{
Files.copy(f, new File(sstablesDir.getAbsolutePath() + "/" + f.getName()));
}
catch(IOException e)
{
throw new RuntimeException(e);
}
});
SSTableReader.resetTidying();
}
}