mirror of https://github.com/apache/cassandra
Sstableverify unit test operate on SSTables
patch by Brian Houser; reviewed by Andres de la Peña, Berenguer Blasi for CASSANDRA-17009 Co-authored-by: Brian Houser <brianh@amazon.com> Co-authored-by: Berenguer Blasi <berenguerblasi@gmail.com>
This commit is contained in:
Bereng
parent
85248da628
commit
95257c0805
|
|
@ -6,6 +6,7 @@ src/gen-java/
|
|||
src/resources/org/apache/cassandra/config/
|
||||
logs/
|
||||
data/
|
||||
!test/data
|
||||
conf/hotspot_compiler
|
||||
doc/cql3/CQL.html
|
||||
doc/build/
|
||||
|
|
|
|||
|
|
@ -18,26 +18,37 @@
|
|||
*/
|
||||
package org.apache.cassandra.tools;
|
||||
|
||||
import org.apache.cassandra.schema.Schema;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.CommandLineParser;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
|
||||
import org.apache.cassandra.config.DatabaseDescriptor;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.Directories;
|
||||
import org.apache.cassandra.db.Keyspace;
|
||||
import org.apache.cassandra.db.compaction.*;
|
||||
import org.apache.cassandra.db.compaction.CompactionManager;
|
||||
import org.apache.cassandra.db.compaction.Verifier;
|
||||
import org.apache.cassandra.dht.Murmur3Partitioner;
|
||||
import org.apache.cassandra.dht.Range;
|
||||
import org.apache.cassandra.dht.Token;
|
||||
import org.apache.cassandra.io.sstable.Component;
|
||||
import org.apache.cassandra.io.sstable.CorruptSSTableException;
|
||||
import org.apache.cassandra.io.sstable.Descriptor;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.schema.Schema;
|
||||
import org.apache.cassandra.utils.JVMStabilityInspector;
|
||||
import org.apache.cassandra.utils.OutputHandler;
|
||||
import org.apache.commons.cli.*;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import static org.apache.cassandra.tools.BulkLoader.CmdLineOptions;
|
||||
|
||||
|
|
@ -56,7 +67,8 @@ public class StandaloneVerifier
|
|||
public static void main(String args[])
|
||||
{
|
||||
Options options = Options.parseArgs(args);
|
||||
Util.initDatabaseDescriptor();
|
||||
initDatabaseDescriptorForTool();
|
||||
|
||||
System.out.println("sstableverify using the following options: " + options);
|
||||
|
||||
try
|
||||
|
|
@ -98,6 +110,7 @@ public class StandaloneVerifier
|
|||
System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
|
||||
if (options.debug)
|
||||
e.printStackTrace(System.err);
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
Verifier.Options verifyOptions = Verifier.options().invokeDiskFailurePolicy(false)
|
||||
|
|
@ -134,6 +147,13 @@ public class StandaloneVerifier
|
|||
}
|
||||
}
|
||||
|
||||
private static void initDatabaseDescriptorForTool() {
|
||||
if (Boolean.getBoolean(Util.ALLOW_TOOL_REINIT_FOR_TEST))
|
||||
DatabaseDescriptor.toolInitialization(false); //Necessary for testing
|
||||
else
|
||||
Util.initDatabaseDescriptor();
|
||||
}
|
||||
|
||||
private static class Options
|
||||
{
|
||||
public final String keyspaceName;
|
||||
|
|
|
|||
|
|
@ -18,13 +18,31 @@
|
|||
*/
|
||||
package org.apache.cassandra.db;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.zip.CRC32;
|
||||
import java.util.zip.CheckedInputStream;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.UpdateBuilder;
|
||||
import org.apache.cassandra.Util;
|
||||
import org.apache.cassandra.batchlog.Batch;
|
||||
import org.apache.cassandra.batchlog.BatchlogManager;
|
||||
import org.apache.cassandra.cache.ChunkCache;
|
||||
import org.apache.cassandra.UpdateBuilder;
|
||||
import org.apache.cassandra.db.compaction.CompactionManager;
|
||||
import org.apache.cassandra.db.compaction.Verifier;
|
||||
import org.apache.cassandra.db.marshal.UUIDType;
|
||||
|
|
@ -45,20 +63,6 @@ import org.apache.cassandra.schema.CompressionParams;
|
|||
import org.apache.cassandra.schema.KeyspaceParams;
|
||||
import org.apache.cassandra.service.StorageService;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.UnknownHostException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.zip.CRC32;
|
||||
import java.util.zip.CheckedInputStream;
|
||||
|
||||
import static org.apache.cassandra.SchemaLoader.counterCFMD;
|
||||
import static org.apache.cassandra.SchemaLoader.createKeyspace;
|
||||
|
|
@ -69,6 +73,14 @@ import static org.junit.Assert.assertFalse;
|
|||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
/**
|
||||
* Test for {@link Verifier}.
|
||||
*
|
||||
* Note: the complete coverage is composed of:
|
||||
* - {@link org.apache.cassandra.tools.StandaloneVerifierOnSSTablesTest}
|
||||
* - {@link org.apache.cassandra.tools.StandaloneVerifierTest}
|
||||
* - {@link VerifyTest}
|
||||
*/
|
||||
public class VerifyTest
|
||||
{
|
||||
public static final String KEYSPACE = "Keyspace1";
|
||||
|
|
|
|||
|
|
@ -33,10 +33,7 @@ import java.util.regex.Pattern;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.cassandra.utils.FBUtilities.preventIllegalAccessWarnings;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,222 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.tools;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import org.apache.cassandra.SchemaLoader;
|
||||
import org.apache.cassandra.UpdateBuilder;
|
||||
import org.apache.cassandra.db.ColumnFamilyStore;
|
||||
import org.apache.cassandra.db.Keyspace;
|
||||
import org.apache.cassandra.db.PartitionPosition;
|
||||
import org.apache.cassandra.db.compaction.CompactionManager;
|
||||
import org.apache.cassandra.io.sstable.Component;
|
||||
import org.apache.cassandra.io.sstable.format.SSTableReader;
|
||||
import org.apache.cassandra.schema.KeyspaceParams;
|
||||
import org.apache.cassandra.service.StorageService;
|
||||
import org.apache.cassandra.tools.ToolRunner.ToolResult;
|
||||
import org.apache.cassandra.utils.ByteBufferUtil;
|
||||
import org.assertj.core.api.Assertions;
|
||||
|
||||
import static org.apache.cassandra.SchemaLoader.standardCFMD;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
/**
|
||||
* Class that tests tables for {@link StandaloneVerifier} by updating using {@link SchemaLoader}
|
||||
* Similar in vein to other {@link SchemaLoader} type tests, as well as {@link StandaloneUpgraderOnSStablesTest}.
|
||||
* Since the tool mainly exercises the {@link org.apache.cassandra.db.compaction.Verifier}, we elect to
|
||||
* not run every conceivable option as many tests are already covered by {@link org.apache.cassandra.db.VerifyTest}.
|
||||
*
|
||||
* Note: the complete coverage is composed of:
|
||||
* - {@link StandaloneVerifierOnSSTablesTest}
|
||||
* - {@link StandaloneVerifierTest}
|
||||
* - {@link org.apache.cassandra.db.VerifyTest}
|
||||
*/
|
||||
public class StandaloneVerifierOnSSTablesTest extends OfflineToolUtils
|
||||
{
|
||||
@BeforeClass
|
||||
public static void setup()
|
||||
{
|
||||
// since legacy tables test data uses ByteOrderedPartitioner that's what we need
|
||||
// for the check version to work
|
||||
System.setProperty("cassandra.partitioner", "org.apache.cassandra.dht.ByteOrderedPartitioner");
|
||||
System.setProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST, "true"); // Necessary for testing`
|
||||
SchemaLoader.loadSchema();
|
||||
StorageService.instance.initServer();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void teardown() throws Exception
|
||||
{
|
||||
SchemaLoader.cleanupAndLeaveDirs();
|
||||
System.clearProperty(Util.ALLOW_TOOL_REINIT_FOR_TEST);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckVersionValidVersion() throws Exception
|
||||
{
|
||||
String keyspaceName = "StandaloneVerifierTestCheckVersionWorking";
|
||||
String workingTable = "workingCheckTable";
|
||||
|
||||
createAndPopulateTable(keyspaceName, workingTable, x -> {});
|
||||
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, workingTable, "-c");
|
||||
assertEquals(0, tool.getExitCode());
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCheckVersionWithWrongVersion() throws Exception
|
||||
{
|
||||
String keyspace = "StandaloneVerifierTestWrongVersions";
|
||||
String tableName = "legacy_ma_simple";
|
||||
|
||||
createAndPopulateTable(keyspace, tableName, cfs -> {
|
||||
// let's just copy old version files from test data into the source dir
|
||||
File testDataDir = new File("test/data/legacy-sstables/ma/legacy_tables/legacy_ma_simple");
|
||||
for (File cfsDir : cfs.getDirectories().getCFDirectories())
|
||||
{
|
||||
FileUtils.copyDirectory(testDataDir, cfsDir);
|
||||
}
|
||||
});
|
||||
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspace, tableName, "-c");
|
||||
|
||||
assertEquals(1, tool.getExitCode());
|
||||
Assertions.assertThat(tool.getStdout()).contains("is not the latest version, run upgradesstables");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkingDataFile() throws Exception
|
||||
{
|
||||
String keyspaceName = "StandaloneVerifierTestWorkingDataKs";
|
||||
String workingTable = "workingTable";
|
||||
|
||||
createAndPopulateTable(keyspaceName, workingTable, x -> {});
|
||||
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, workingTable);
|
||||
assertEquals(0, tool.getExitCode());
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCorruptStatsFile() throws Exception
|
||||
{
|
||||
String keyspaceName = "StandaloneVerifierTestCorruptStatsKs";
|
||||
String corruptStatsTable = "corruptStatsTable";
|
||||
createAndPopulateTable(keyspaceName, corruptStatsTable, cfs -> {
|
||||
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
|
||||
try (RandomAccessFile file = new RandomAccessFile(sstable.descriptor.filenameFor(Component.STATS), "rw"))
|
||||
{
|
||||
file.seek(0);
|
||||
file.writeBytes(StringUtils.repeat('z', 2));
|
||||
}
|
||||
});
|
||||
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, corruptStatsTable);
|
||||
|
||||
assertEquals(1, tool.getExitCode());
|
||||
Assertions.assertThat(tool.getStderr()).contains("Error Loading", corruptStatsTable);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCorruptDataFile() throws Exception
|
||||
{
|
||||
String keyspaceName = "StandaloneVerifierTestCorruptDataKs";
|
||||
String corruptDataTable = "corruptDataTable";
|
||||
|
||||
createAndPopulateTable(keyspaceName, corruptDataTable, cfs -> {
|
||||
SSTableReader sstable = cfs.getLiveSSTables().iterator().next();
|
||||
long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("0"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
|
||||
long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(ByteBufferUtil.bytes("1"), cfs.getPartitioner()), SSTableReader.Operator.EQ).position;
|
||||
long startPosition = Math.min(row0Start, row1Start);
|
||||
|
||||
try (RandomAccessFile file = new RandomAccessFile(sstable.getFilename(), "rw"))
|
||||
{
|
||||
file.seek(startPosition);
|
||||
file.writeBytes(StringUtils.repeat('z', 2));
|
||||
}
|
||||
});
|
||||
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, keyspaceName, corruptDataTable);
|
||||
assertEquals(1, tool.getExitCode());
|
||||
Assertions.assertThat(tool.getStdout()).contains("Invalid SSTable", corruptDataTable);
|
||||
}
|
||||
|
||||
/**
|
||||
* Since we are testing a verifier, we'd like to corrupt files to verify code paths
|
||||
* This function definition is used by {@link this#createAndPopulateTable}.
|
||||
*
|
||||
* CFS is the open ColumnFamilyStore for a given keyspace, table
|
||||
*/
|
||||
@FunctionalInterface
|
||||
private interface CorruptFunction
|
||||
{
|
||||
public void apply(ColumnFamilyStore cfs) throws Exception;
|
||||
}
|
||||
|
||||
/**
|
||||
* This function sets up the keyspace, and table schema for a standardCFMD table.
|
||||
* <p>
|
||||
* This will also populate the tableName with a few rows. After completion the
|
||||
* server will be shutdown.
|
||||
*
|
||||
* @param keyspace the name of the keyspace in which the table should be created
|
||||
* @param tableName new table name of the standard CFMD table
|
||||
* @param corruptionFn function called to corrupt or change the contents on disk, is passed the Cfs of the table name.
|
||||
* @throws Exception on error.
|
||||
*/
|
||||
private static void createAndPopulateTable(String keyspace, String tableName, CorruptFunction corruptionFn) throws Exception
|
||||
{
|
||||
SchemaLoader.createKeyspace(keyspace,
|
||||
KeyspaceParams.simple(1),
|
||||
standardCFMD(keyspace, tableName));
|
||||
|
||||
CompactionManager.instance.disableAutoCompaction();
|
||||
|
||||
Keyspace k = Keyspace.open(keyspace);
|
||||
ColumnFamilyStore cfs = k.getColumnFamilyStore(tableName);
|
||||
|
||||
populateTable(cfs, 2);
|
||||
|
||||
corruptionFn.apply(cfs);
|
||||
}
|
||||
|
||||
private static void populateTable(ColumnFamilyStore cfs, int partitionsPerSSTable)
|
||||
{
|
||||
for (int i = 0; i < partitionsPerSSTable; i++)
|
||||
{
|
||||
UpdateBuilder.create(cfs.metadata(), String.valueOf(i))
|
||||
.newRow("c1").add("val", "1")
|
||||
.newRow("c2").add("val", "2")
|
||||
.apply();
|
||||
}
|
||||
|
||||
cfs.forceBlockingFlush();
|
||||
}
|
||||
}
|
||||
|
|
@ -29,6 +29,12 @@ import org.hamcrest.CoreMatchers;
|
|||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertThat;
|
||||
|
||||
/**
|
||||
* Note: the complete coverage is composed of:
|
||||
* - {@link StandaloneVerifierOnSSTablesTest}
|
||||
* - {@link StandaloneVerifierTest}
|
||||
* - {@link org.apache.cassandra.db.VerifyTest}
|
||||
*/
|
||||
public class StandaloneVerifierTest extends OfflineToolUtils
|
||||
{
|
||||
@Test
|
||||
|
|
@ -70,6 +76,8 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
assertEquals(0, tool.getExitCode());
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -78,8 +86,9 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, "--debug", "system_schema", "tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("debug=true"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -92,8 +101,8 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
"tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("extended=true"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -107,8 +116,8 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
"tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("quick=true"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -122,8 +131,8 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
"tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("mutateRepairStatus=true"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -134,8 +143,8 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, arg);
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -149,8 +158,17 @@ public class StandaloneVerifierTest extends OfflineToolUtils
|
|||
"tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("verbose=true"));
|
||||
Assertions.assertThat(tool.getCleanedStderr()).isEmpty();
|
||||
tool.assertOnExitCode();
|
||||
assertCorrectEnvPostTest();
|
||||
tool.assertOnCleanExit();
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTooManyArgs()
|
||||
{
|
||||
ToolResult tool = ToolRunner.invokeClass(StandaloneVerifier.class, "another arg", "system_schema", "tables");
|
||||
assertThat(tool.getStdout(), CoreMatchers.containsStringIgnoringCase("usage:"));
|
||||
assertThat(tool.getCleanedStderr(), CoreMatchers.containsStringIgnoringCase("Too many arguments"));
|
||||
assertEquals(1, tool.getExitCode());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue