diff --git a/test/conf/logback-dtest.xml b/test/conf/logback-dtest.xml index 4282feec4b..9b3216ddcf 100644 --- a/test/conf/logback-dtest.xml +++ b/test/conf/logback-dtest.xml @@ -18,35 +18,18 @@ --> + - - - ./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log - - ./build/test/logs/${cassandra.testtag}/TEST-${suitename}.log.%i.gz - 1 - 20 - - - - 20MB - - + + ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log %-5level [%thread] ${instance_id} %date{ISO8601} %msg%n - false - - - - 0 - 0 - 1024 - + true @@ -79,7 +62,7 @@ - + diff --git a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java index 4b880e496c..934b28d162 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/AbstractCluster.java @@ -22,12 +22,12 @@ import java.io.File; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -35,7 +35,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; import java.util.function.BiPredicate; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,11 +44,11 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.api.ConsistencyLevel; import org.apache.cassandra.distributed.api.Feature; import org.apache.cassandra.distributed.api.ICluster; -import org.apache.cassandra.dht.IPartitioner; -import org.apache.cassandra.dht.Token; import org.apache.cassandra.distributed.api.ICoordinator; import org.apache.cassandra.distributed.api.IInstance; import org.apache.cassandra.distributed.api.IInstanceConfig; @@ -59,6 +58,7 @@ import org.apache.cassandra.distributed.api.IListen; import org.apache.cassandra.distributed.api.IMessage; import org.apache.cassandra.distributed.api.IMessageFilters; import org.apache.cassandra.distributed.api.IUpgradeableInstance; +import org.apache.cassandra.distributed.api.LogAction; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.api.TokenSupplier; import org.apache.cassandra.distributed.shared.AbstractBuilder; @@ -106,6 +106,7 @@ public abstract class AbstractCluster implements ICluster implements ICluster implements ICluster fn) + { + RandomAccessFile reader; + try + { + reader = new RandomAccessFile(file, "r"); + } + catch (FileNotFoundException e) + { + // if file isn't present, don't return an empty stream as it looks the same as no log lines matched + throw new UncheckedIOException(e); + } + if (startPosition > 0) // -1 used to disable, so ignore any negative values or 0 (default offset) + { + try + { + reader.seek(startPosition); + } + catch (IOException e) + { + throw new UncheckedIOException("Unable to seek to " + startPosition, e); + } + } + return new FileLineIterator(reader, fn); + } + + private static final class FileLineIterator extends AbstractIterator implements LineIterator + { + private final RandomAccessFile reader; + private final Predicate fn; + + private FileLineIterator(RandomAccessFile reader, Predicate fn) + { + this.reader = reader; + this.fn = fn; + } + + @Override + public long mark() + { + try + { + return reader.getFilePointer(); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + @Override + protected String computeNext() + { + try + { + String s; + while ((s = reader.readLine()) != null) + { + if (fn.test(s)) + return s; + } + return endOfData(); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + @Override + public void close() + { + try + { + Closeables.close(reader, true); + } + catch (IOException impossible) + { + throw new AssertionError(impossible); + } + } + } +} diff --git a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java index cf66d8f410..2c06066a66 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/Instance.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/Instance.java @@ -24,11 +24,11 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CopyOnWriteArrayList; @@ -37,7 +37,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import java.util.function.BiConsumer; import java.util.function.Function; - import javax.management.ListenerNotFoundException; import javax.management.Notification; import javax.management.NotificationListener; @@ -68,6 +67,7 @@ import org.apache.cassandra.distributed.api.IInstanceConfig; import org.apache.cassandra.distributed.api.IInvokableInstance; import org.apache.cassandra.distributed.api.IListen; import org.apache.cassandra.distributed.api.IMessage; +import org.apache.cassandra.distributed.api.LogAction; import org.apache.cassandra.distributed.api.NodeToolResult; import org.apache.cassandra.distributed.api.SimpleQueryResult; import org.apache.cassandra.distributed.mock.nodetool.InternalNodeProbe; @@ -135,6 +135,8 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance { super("node" + config.num(), classLoader); this.config = config; + Object clusterId = Objects.requireNonNull(config.get("dtest.api.cluster_id"), "cluster_id is not defined"); + ClusterIDDefiner.setId("cluster-" + clusterId); InstanceIDDefiner.setInstanceId(config.num()); FBUtilities.setBroadcastInetAddress(config.broadcastAddress().getAddress()); @@ -144,6 +146,24 @@ public class Instance extends IsolatedExecutor implements IInvokableInstance Config.setOverrideLoadConfig(() -> loadConfig(config)); } + @Override + public boolean getLogsEnabled() + { + return true; + } + + @Override + public LogAction logs() + { + // the path used is defined by test/conf/logback-dtest.xml and looks like the following + // ./build/test/logs/${cassandra.testtag}/${suitename}/${cluster_id}/${instance_id}/system.log + String tag = System.getProperty("cassandra.testtag", "cassandra.testtag_IS_UNDEFINED"); + String suite = System.getProperty("suitename", "suitename_IS_UNDEFINED"); + String clusterId = ClusterIDDefiner.getId(); + String instanceId = InstanceIDDefiner.getInstanceId(); + return new FileLogAction(new File(String.format("build/test/logs/%s/%s/%s/%s/system.log", tag, suite, clusterId, instanceId))); + } + @Override public IInstanceConfig config() { diff --git a/test/distributed/org/apache/cassandra/distributed/impl/InstanceIDDefiner.java b/test/distributed/org/apache/cassandra/distributed/impl/InstanceIDDefiner.java index d32bd7759e..2aa084c21f 100644 --- a/test/distributed/org/apache/cassandra/distributed/impl/InstanceIDDefiner.java +++ b/test/distributed/org/apache/cassandra/distributed/impl/InstanceIDDefiner.java @@ -27,15 +27,21 @@ import org.apache.cassandra.concurrent.NamedThreadFactory; public class InstanceIDDefiner extends PropertyDefinerBase { // Instantiated per classloader, set by Instance - private static volatile String instanceId = "
"; + private static volatile String INSTANCE_ID = "
"; + public static void setInstanceId(int id) { - instanceId = "node" + id; + INSTANCE_ID = "node" + id; NamedThreadFactory.setGlobalPrefix("node" + id + "_"); } + public static String getInstanceId() + { + return INSTANCE_ID; + } + public String getPropertyValue() { - return instanceId; + return INSTANCE_ID; } } diff --git a/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java b/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java index 795b4ea63c..3a1a0a8d1e 100644 --- a/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java +++ b/test/distributed/org/apache/cassandra/distributed/test/JVMDTestTest.java @@ -19,11 +19,20 @@ package org.apache.cassandra.distributed.test; import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.logging.Logger; +import org.junit.Assert; import org.junit.Test; +import org.slf4j.LoggerFactory; + import org.apache.cassandra.distributed.Cluster; import org.apache.cassandra.distributed.api.ConsistencyLevel; +import org.apache.cassandra.distributed.api.Feature; +import org.apache.cassandra.distributed.api.LogAction; +import org.apache.cassandra.service.CassandraDaemon; import org.apache.cassandra.utils.FBUtilities; import static org.junit.Assert.assertEquals; @@ -50,4 +59,23 @@ public class JVMDTestTest extends TestBaseImpl assertEquals(1000, (long) res[0][0]); } } + + @Test + public void instanceLogs() throws IOException, TimeoutException + { + try (Cluster cluster = init(Cluster.build(2).withConfig(c -> c.with(Feature.values())).start())) + { + // debug logging is turned on so we will see debug logs + Assert.assertFalse(cluster.get(1).logs().grep("^DEBUG").getResult().isEmpty()); + // make sure an exception is thrown in the cluster + LogAction logs = cluster.get(2).logs(); + long mark = logs.mark(); // get the current position so watching doesn't see any previous exceptions + cluster.get(2).runOnInstance(() -> { + // pretend that an uncaught exception was thrown + LoggerFactory.getLogger(CassandraDaemon.class).error("Error", new RuntimeException("fail without fail")); + }); + List errors = logs.watchFor(mark, "^ERROR").getResult(); + Assert.assertFalse(errors.isEmpty()); + } + } }