mirror of https://github.com/apache/cassandra
Add a new tool to dump audit logs
Patch by Vinay Chella; reviewed by marcuse for CASSANDRA-14885
This commit is contained in:
Vinay Chella
Marcus Eriksson
parent
4aa022e215
commit
7d138e20ea
|
|
@ -1,4 +1,5 @@
|
|||
4.0
|
||||
* Add a new tool to dump audit logs (CASSANDRA-14885)
|
||||
* Fix generating javadoc with Java11 (CASSANDRA-14988)
|
||||
* Only cancel conflicting compactions when starting anticompactions and sub range compactions (CASSANDRA-14935)
|
||||
* Use a stub IndexRegistry for non-daemon use cases (CASSANDRA-14938)
|
||||
|
|
|
|||
|
|
@ -143,19 +143,44 @@ NodeTool command to reload AuditLog filters
|
|||
``enableauditlog``: NodeTool enableauditlog command can be used to reload auditlog filters when called with default or previous ``loggername`` and updated filters
|
||||
|
||||
E.g.,
|
||||
|
||||
::
|
||||
|
||||
nodetool enableauditlog --loggername <Default/ existing loggerName> --included-keyspaces <New Filter values>
|
||||
|
||||
|
||||
|
||||
View the contents of AuditLog Files
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
``auditlogviewer`` is the new tool introduced to help view the contents of binlog file in human readable text format.
|
||||
|
||||
::
|
||||
|
||||
auditlogviewer <path1> [<path2>...<pathN>] [options]
|
||||
|
||||
Options
|
||||
""""""""
|
||||
|
||||
``-f,--follow``
|
||||
Upon reacahing the end of the log continue indefinitely
|
||||
waiting for more records
|
||||
``-r,--roll_cycle``
|
||||
How often to roll the log file was rolled. May be
|
||||
necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY,
|
||||
DAILY). Default HOURLY.
|
||||
|
||||
``-h,--help``
|
||||
display this help message
|
||||
|
||||
For example, to dump the contents of audit log files on the console
|
||||
|
||||
::
|
||||
|
||||
auditlogviewer /logs/cassandra/audit
|
||||
|
||||
Sample output
|
||||
^^^^^^^^^^^^^^^^
|
||||
"""""""""""""
|
||||
|
||||
::
|
||||
|
||||
LogMessage: user:anonymous|host:localhost/X.X.X.X|source:/X.X.X.X|port:60878|timestamp:1521158923615|type:USE_KS|category:DDL|ks:dev1|operation:USE "dev1"
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.cassandra.audit;
|
|||
|
||||
import java.nio.file.Paths;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.primitives.Ints;
|
||||
|
||||
import net.openhft.chronicle.wire.WireOut;
|
||||
|
|
@ -29,6 +30,10 @@ import org.apache.cassandra.utils.concurrent.WeightedQueue;
|
|||
|
||||
public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger
|
||||
{
|
||||
public static final String TYPE = "type";
|
||||
public static final String AUDITLOG_TYPE = "AuditLog";
|
||||
public static final String AUDITLOG_MESSAGE = "message";
|
||||
|
||||
public BinAuditLogger()
|
||||
{
|
||||
// due to the way that IAuditLogger instance are created in AuditLogManager, via reflection, we can't assume
|
||||
|
|
@ -56,11 +61,12 @@ public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger
|
|||
super.logRecord(new Message(auditLogEntry.getLogString()), binLog);
|
||||
}
|
||||
|
||||
static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable
|
||||
@VisibleForTesting
|
||||
public static class Message extends BinLog.ReleaseableWriteMarshallable implements WeightedQueue.Weighable
|
||||
{
|
||||
private final String message;
|
||||
|
||||
Message(String message)
|
||||
public Message(String message)
|
||||
{
|
||||
this.message = message;
|
||||
}
|
||||
|
|
@ -68,8 +74,8 @@ public class BinAuditLogger extends BinLogAuditLogger implements IAuditLogger
|
|||
@Override
|
||||
public void writeMarshallable(WireOut wire)
|
||||
{
|
||||
wire.write("type").text("AuditLog");
|
||||
wire.write("message").text(message);
|
||||
wire.write(TYPE).text(AUDITLOG_TYPE);
|
||||
wire.write(AUDITLOG_MESSAGE).text(message);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -0,0 +1,212 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.cassandra.tools;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.CommandLineParser;
|
||||
import org.apache.commons.cli.GnuParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
|
||||
import net.openhft.chronicle.core.io.IORuntimeException;
|
||||
import net.openhft.chronicle.queue.ChronicleQueueBuilder;
|
||||
import net.openhft.chronicle.queue.ExcerptTailer;
|
||||
import net.openhft.chronicle.queue.RollCycles;
|
||||
import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue;
|
||||
import net.openhft.chronicle.threads.Pauser;
|
||||
import net.openhft.chronicle.wire.ReadMarshallable;
|
||||
import net.openhft.chronicle.wire.WireIn;
|
||||
import org.apache.cassandra.audit.BinAuditLogger;
|
||||
|
||||
/**
|
||||
* Tool to view the contenst of AuditLog files in human readable format. Default implementation for AuditLog files
|
||||
* logs audit messages in {@link org.apache.cassandra.utils.binlog.BinLog} format, this tool prints the contens of
|
||||
* binary audit log files in text format.
|
||||
*/
|
||||
public class AuditLogViewer
|
||||
{
|
||||
private static final String TOOL_NAME = "auditlogviewer";
|
||||
private static final String ROLL_CYCLE = "roll_cycle";
|
||||
private static final String FOLLOW = "follow";
|
||||
private static final String HELP_OPTION = "help";
|
||||
|
||||
public static void main(String[] args)
|
||||
{
|
||||
AuditLogViewerOptions options = AuditLogViewerOptions.parseArgs(args);
|
||||
|
||||
try
|
||||
{
|
||||
dump(options.pathList, options.rollCycle, options.follow, System.out::print);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
System.err.println(e.getMessage());
|
||||
System.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void dump(List<String> pathList, String rollCycle, boolean follow, Consumer<String> displayFun)
|
||||
{
|
||||
//Backoff strategy for spinning on the queue, not aggressive at all as this doesn't need to be low latency
|
||||
Pauser pauser = Pauser.millis(100);
|
||||
List<ExcerptTailer> tailers = pathList.stream()
|
||||
.distinct()
|
||||
.map(path -> ChronicleQueueBuilder.single(new File(path)).readOnly(true).rollCycle(RollCycles.valueOf(rollCycle)).build())
|
||||
.map(SingleChronicleQueue::createTailer)
|
||||
.collect(Collectors.toList());
|
||||
boolean hadWork = true;
|
||||
while (hadWork)
|
||||
{
|
||||
hadWork = false;
|
||||
for (ExcerptTailer tailer : tailers)
|
||||
{
|
||||
while (tailer.readDocument(new DisplayRecord(displayFun)))
|
||||
{
|
||||
hadWork = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (follow)
|
||||
{
|
||||
if (!hadWork)
|
||||
{
|
||||
//Chronicle queue doesn't support blocking so use this backoff strategy
|
||||
pauser.pause();
|
||||
}
|
||||
//Don't terminate the loop even if there wasn't work
|
||||
hadWork = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static class DisplayRecord implements ReadMarshallable
|
||||
{
|
||||
private final Consumer<String> displayFun;
|
||||
|
||||
DisplayRecord(Consumer<String> displayFun)
|
||||
{
|
||||
this.displayFun = displayFun;
|
||||
}
|
||||
|
||||
public void readMarshallable(WireIn wireIn) throws IORuntimeException
|
||||
{
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
String type = wireIn.read(BinAuditLogger.TYPE).text();
|
||||
sb.append("Type: ")
|
||||
.append(type)
|
||||
.append(System.lineSeparator());
|
||||
|
||||
if (null != type && type.equals(BinAuditLogger.AUDITLOG_TYPE))
|
||||
{
|
||||
sb.append("LogMessage: ").append(wireIn.read(BinAuditLogger.AUDITLOG_MESSAGE).text()).append(System.lineSeparator());
|
||||
}
|
||||
|
||||
displayFun.accept(sb.toString());
|
||||
}
|
||||
}
|
||||
|
||||
private static class AuditLogViewerOptions
|
||||
{
|
||||
private final List<String> pathList;
|
||||
private String rollCycle = "HOURLY";
|
||||
private boolean follow;
|
||||
|
||||
private AuditLogViewerOptions(String[] pathList)
|
||||
{
|
||||
this.pathList = Arrays.asList(pathList);
|
||||
}
|
||||
|
||||
static AuditLogViewerOptions parseArgs(String cmdArgs[])
|
||||
{
|
||||
CommandLineParser parser = new GnuParser();
|
||||
Options options = getCmdLineOptions();
|
||||
try
|
||||
{
|
||||
CommandLine cmd = parser.parse(options, cmdArgs, false);
|
||||
|
||||
if (cmd.hasOption(HELP_OPTION))
|
||||
{
|
||||
printUsage(options);
|
||||
System.exit(0);
|
||||
}
|
||||
|
||||
String[] args = cmd.getArgs();
|
||||
if (args.length <= 0)
|
||||
{
|
||||
System.err.println("Audit log files directory path is a required argument.");
|
||||
printUsage(options);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
AuditLogViewerOptions opts = new AuditLogViewerOptions(args);
|
||||
|
||||
opts.follow = cmd.hasOption(FOLLOW);
|
||||
|
||||
if (cmd.hasOption(ROLL_CYCLE))
|
||||
{
|
||||
opts.rollCycle = cmd.getOptionValue(ROLL_CYCLE);
|
||||
}
|
||||
|
||||
return opts;
|
||||
}
|
||||
catch (ParseException e)
|
||||
{
|
||||
errorMsg(e.getMessage(), options);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
static void errorMsg(String msg, Options options)
|
||||
{
|
||||
System.err.println(msg);
|
||||
printUsage(options);
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
static Options getCmdLineOptions()
|
||||
{
|
||||
Options options = new Options();
|
||||
|
||||
options.addOption(new Option("r", ROLL_CYCLE, false, "How often to roll the log file was rolled. May be necessary for Chronicle to correctly parse file names. (MINUTELY, HOURLY, DAILY). Default HOURLY."));
|
||||
options.addOption(new Option("f", FOLLOW, false, "Upon reacahing the end of the log continue indefinitely waiting for more records"));
|
||||
options.addOption(new Option("h", HELP_OPTION, false, "display this help message"));
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
static void printUsage(Options options)
|
||||
{
|
||||
String usage = String.format("%s <path1> [<path2>...<pathN>] [options]", TOOL_NAME);
|
||||
StringBuilder header = new StringBuilder();
|
||||
header.append("--\n");
|
||||
header.append("View the audit log contents in human readable format");
|
||||
header.append("\n--\n");
|
||||
header.append("Options are:");
|
||||
new HelpFormatter().printHelp(usage, header.toString(), options, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.cassandra.tools;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import net.openhft.chronicle.queue.ChronicleQueue;
|
||||
import net.openhft.chronicle.queue.ChronicleQueueBuilder;
|
||||
import net.openhft.chronicle.queue.ExcerptAppender;
|
||||
import net.openhft.chronicle.queue.RollCycles;
|
||||
import org.apache.cassandra.audit.BinAuditLogger;
|
||||
|
||||
public class AuditLogViewerTest
|
||||
{
|
||||
private Path path;
|
||||
|
||||
@Before
|
||||
public void setUp() throws IOException
|
||||
{
|
||||
path = Files.createTempDirectory("foo");
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws IOException
|
||||
{
|
||||
if (path.toFile().exists() && path.toFile().isDirectory())
|
||||
{
|
||||
//Deletes directory and all of it's contents
|
||||
FileUtils.deleteDirectory(path.toFile());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDisplayRecord()
|
||||
{
|
||||
List<String> records = new ArrayList<>();
|
||||
records.add("Test foo bar 1");
|
||||
records.add("Test foo bar 2");
|
||||
|
||||
try (ChronicleQueue queue = ChronicleQueueBuilder.single(path.toFile()).rollCycle(RollCycles.TEST_SECONDLY).build())
|
||||
{
|
||||
ExcerptAppender appender = queue.acquireAppender();
|
||||
|
||||
//Write bunch of records
|
||||
records.forEach(s -> appender.writeDocument(new BinAuditLogger.Message(s)));
|
||||
|
||||
//Read those written records
|
||||
List<String> actualRecords = new ArrayList<>();
|
||||
AuditLogViewer.dump(ImmutableList.of(path.toString()), RollCycles.TEST_SECONDLY.toString(), false, actualRecords::add);
|
||||
|
||||
for (int i = 0; i < records.size(); i++)
|
||||
{
|
||||
Assert.assertTrue(actualRecords.get(i).contains(records.get(i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
|
||||
# Locations (in order) to use when searching for an include file.
|
||||
for include in "`dirname "$0"`/cassandra.in.sh" \
|
||||
"$HOME/.cassandra.in.sh" \
|
||||
/usr/share/cassandra/cassandra.in.sh \
|
||||
/usr/local/share/cassandra/cassandra.in.sh \
|
||||
/opt/cassandra/cassandra.in.sh; do
|
||||
if [ -r "$include" ]; then
|
||||
. "$include"
|
||||
break
|
||||
fi
|
||||
done
|
||||
elif [ -r "$CASSANDRA_INCLUDE" ]; then
|
||||
. "$CASSANDRA_INCLUDE"
|
||||
fi
|
||||
|
||||
if [ -z "$CLASSPATH" ]; then
|
||||
echo "You must set the CLASSPATH var" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "x$MAX_HEAP_SIZE" = "x" ]; then
|
||||
MAX_HEAP_SIZE="256M"
|
||||
fi
|
||||
|
||||
"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" $JVM_OPTS -Xmx$MAX_HEAP_SIZE \
|
||||
-Dcassandra.storagedir="$cassandra_storagedir" \
|
||||
-Dlogback.configurationFile=logback-tools.xml \
|
||||
org.apache.cassandra.tools.AuditLogViewer "$@"
|
||||
|
||||
# vi:ai sw=4 ts=4 tw=0 et
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
@REM
|
||||
@REM Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
@REM contributor license agreements. See the NOTICE file distributed with
|
||||
@REM this work for additional information regarding copyright ownership.
|
||||
@REM The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
@REM (the "License"); you may not use this file except in compliance with
|
||||
@REM the License. You may obtain a copy of the License at
|
||||
@REM
|
||||
@REM http://www.apache.org/licenses/LICENSE-2.0
|
||||
@REM
|
||||
@REM Unless required by applicable law or agreed to in writing, software
|
||||
@REM distributed under the License is distributed on an "AS IS" BASIS,
|
||||
@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@REM See the License for the specific language governing permissions and
|
||||
@REM limitations under the License.
|
||||
|
||||
@echo off
|
||||
if "%OS%" == "Windows_NT" setlocal
|
||||
|
||||
pushd "%~dp0"
|
||||
call cassandra.in.bat
|
||||
|
||||
if NOT DEFINED CASSANDRA_MAIN set CASSANDRA_MAIN=org.apache.cassandra.tools.AuditLogViewer
|
||||
if NOT DEFINED JAVA_HOME goto :err
|
||||
|
||||
REM ***** JAVA options *****
|
||||
set JAVA_OPTS=^
|
||||
-Dlogback.configurationFile=logback-tools.xml
|
||||
|
||||
set TOOLS_PARAMS=
|
||||
|
||||
"%JAVA_HOME%\bin\java" %JAVA_OPTS% %CASSANDRA_PARAMS% -cp %CASSANDRA_CLASSPATH% "%CASSANDRA_MAIN%" %*
|
||||
goto finally
|
||||
|
||||
:err
|
||||
echo JAVA_HOME environment variable must be set!
|
||||
pause
|
||||
|
||||
:finally
|
||||
|
||||
ENDLOCAL
|
||||
Loading…
Reference in New Issue