Provide graphing tool w/cassandra-stress

Patch by rmcguire; reviewed by jmckenzie for CASSANDRA-7918

CHANGES.txt

@@ -1,4 +1,5 @@
 3.2
+ * Added graphing option to cassandra-stress (CASSANDRA-7918)
  * Abort in-progress queries that time out (CASSANDRA-7392)
  * Add transparent data encryption core classes (CASSANDRA-9945)
 

build.xml

@@ -802,6 +802,9 @@
                 </path>
             </classpath>
         </javac>
+        <copy todir="${stress.build.classes}">
+            <fileset dir="${stress.build.src}/resources" />
+        </copy>
     </target>
 
 	<target name="_write-poms" depends="maven-declare-dependencies">

tools/stress/src/org/apache/cassandra/stress/Stress.java

@@ -24,6 +24,7 @@ import java.net.SocketException;
 import org.apache.cassandra.stress.settings.StressSettings;
 import org.apache.cassandra.utils.FBUtilities;
 import org.apache.cassandra.utils.WindowsTimer;
+import org.apache.cassandra.stress.util.MultiPrintStream;
 
 public final class Stress
 {
@@ -72,7 +73,10 @@ public final class Stress
                 return;
             }
 
-            PrintStream logout = settings.log.getOutput();
+            MultiPrintStream logout = settings.log.getOutput();
+            if (settings.graph.inGraphMode()) {
+                logout.addStream(new PrintStream(settings.graph.temporaryLogFile));
+            }
 
             if (settings.sendToDaemon != null)
             {
@@ -115,6 +119,9 @@ public final class Stress
             {
                 StressAction stressAction = new StressAction(settings, logout);
                 stressAction.run();
+                logout.flush();
+                if (settings.graph.inGraphMode())
+                    new StressGraph(settings, arguments).generateGraph();
             }
 
         }

tools/stress/src/org/apache/cassandra/stress/StressAction.java

@@ -34,7 +34,6 @@ import org.apache.cassandra.stress.settings.SettingsCommand;
 import org.apache.cassandra.stress.settings.StressSettings;
 import org.apache.cassandra.stress.util.JavaDriverClient;
 import org.apache.cassandra.stress.util.ThriftClient;
-import org.apache.cassandra.stress.util.TimingInterval;
 import org.apache.cassandra.transport.SimpleClient;
 
 public class StressAction implements Runnable
@@ -118,6 +117,7 @@ public class StressAction implements Runnable
         List<String> runIds = new ArrayList<>();
         do
         {
+            output.println("");
             output.println(String.format("Running with %d threadCount", threadCount));
 
             if (settings.command.truncate == SettingsCommand.TruncateWhen.ALWAYS)

tools/stress/src/org/apache/cassandra/stress/StressGraph.java

@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.stress;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.io.ByteStreams;
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.cassandra.stress.settings.StressSettings;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
+import org.json.simple.JSONValue;
+
+
+public class StressGraph
+{
+    private StressSettings stressSettings;
+    private enum ReadingMode
+    {
+        START,
+        METRICS,
+        AGGREGATES,
+        NEXTITERATION
+    }
+    private String[] stressArguments;
+
+    public StressGraph(StressSettings stressSetttings, String[] stressArguments)
+    {
+        this.stressSettings = stressSetttings;
+        this.stressArguments = stressArguments;
+    }
+
+    public void generateGraph()
+    {
+        File htmlFile = new File(stressSettings.graph.file);
+        JSONObject stats;
+        if (htmlFile.isFile())
+        {
+            try
+            {
+                String html = new String(Files.readAllBytes(Paths.get(htmlFile.toURI())), StandardCharsets.UTF_8);
+                stats = parseExistingStats(html);
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException("Couldn't load existing stats html.");
+            }
+            stats = this.createJSONStats(stats);
+        }
+        else
+        {
+            stats = this.createJSONStats(null);
+        }
+
+        try
+        {
+            PrintWriter out = new PrintWriter(htmlFile);
+            String statsBlock = "/* stats start */\nstats = " + stats.toJSONString() + ";\n/* stats end */\n";
+            String html = getGraphHTML().replaceFirst("/\\* stats start \\*/\n\n/\\* stats end \\*/\n", statsBlock);
+            out.write(html);
+            out.close();
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Couldn't write stats html.");
+        }
+    }
+
+    private JSONObject parseExistingStats(String html)
+    {
+        JSONObject stats;
+
+        Pattern pattern = Pattern.compile("(?s).*/\\* stats start \\*/\\nstats = (.*);\\n/\\* stats end \\*/.*");
+        Matcher matcher = pattern.matcher(html);
+        matcher.matches();
+        stats = (JSONObject) JSONValue.parse(matcher.group(1));
+
+        return stats;
+    }
+
+    private String getGraphHTML()
+    {
+        InputStream graphHTMLRes = StressGraph.class.getClassLoader().getResourceAsStream("org/apache/cassandra/stress/graph/graph.html");
+        String graphHTML;
+        try
+        {
+            graphHTML = new String(ByteStreams.toByteArray(graphHTMLRes));
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException(e);
+        }
+        return graphHTML;
+    }
+
+    /** Parse log and append to stats array */
+    private JSONArray parseLogStats(InputStream log, JSONArray stats) {
+        BufferedReader reader = new BufferedReader(new InputStreamReader(log));
+        JSONObject json = new JSONObject();
+        JSONArray intervals = new JSONArray();
+        boolean runningMultipleThreadCounts = false;
+        String currentThreadCount = null;
+        Pattern threadCountMessage = Pattern.compile("Running ([A-Z]+) with ([0-9]+) threads .*");
+        ReadingMode mode = ReadingMode.START;
+
+        try
+        {
+            String line;
+            while ((line = reader.readLine()) != null)
+            {
+                // Detect if we are running multiple thread counts:
+                if (line.startsWith("Thread count was not specified"))
+                    runningMultipleThreadCounts = true;
+
+                // Detect thread count:
+                Matcher tc = threadCountMessage.matcher(line);
+                if (tc.matches())
+                {
+                    if (runningMultipleThreadCounts)
+                    {
+                        currentThreadCount = tc.group(2);
+                    }
+                }
+
+                // Detect mode changes
+                if (line.equals(StressMetrics.HEAD))
+                {
+                    mode = ReadingMode.METRICS;
+                    continue;
+                }
+                else if (line.equals("Results:"))
+                {
+                    mode = ReadingMode.AGGREGATES;
+                    continue;
+                }
+                else if (mode == ReadingMode.AGGREGATES && line.equals(""))
+                {
+                    mode = ReadingMode.NEXTITERATION;
+                }
+                else if (line.equals("END") || line.equals("FAILURE"))
+                {
+                    break;
+                }
+
+                // Process lines
+                if (mode == ReadingMode.METRICS)
+                {
+                    JSONArray metrics = new JSONArray();
+                    String[] parts = line.split(",");
+                    if (parts.length != StressMetrics.HEADMETRICS.length)
+                    {
+                        continue;
+                    }
+                    for (String m : parts)
+                    {
+                        try
+                        {
+                            metrics.add(new BigDecimal(m.trim()));
+                        }
+                        catch (NumberFormatException e)
+                        {
+                            metrics.add(null);
+                        }
+                    }
+                    intervals.add(metrics);
+                }
+                else if (mode == ReadingMode.AGGREGATES)
+                {
+                    String[] parts = line.split(":",2);
+                    if (parts.length != 2)
+                    {
+                        continue;
+                    }
+                    json.put(parts[0].trim(), parts[1].trim());
+                }
+                else if (mode == ReadingMode.NEXTITERATION)
+                {
+                    //Wrap up the results of this test and append to the array.
+                    json.put("metrics", Arrays.asList(StressMetrics.HEADMETRICS));
+                    json.put("test", stressSettings.graph.operation);
+                    if (currentThreadCount == null)
+                        json.put("revision", stressSettings.graph.revision);
+                    else
+                        json.put("revision", String.format("%s - %s threads", stressSettings.graph.revision, currentThreadCount));
+                    json.put("command", StringUtils.join(stressArguments, " "));
+                    json.put("intervals", intervals);
+                    stats.add(json);
+
+                    //Start fresh for next iteration:
+                    json = new JSONObject();
+                    intervals = new JSONArray();
+                    mode = ReadingMode.START;
+                }
+            }
+        }
+        catch (IOException e)
+        {
+            throw new RuntimeException("Couldn't read from temporary stress log file");
+        }
+        stats.add(json);
+        return stats;
+    }
+
+    private JSONObject createJSONStats(JSONObject json)
+    {
+        JSONArray stats;
+        if (json == null)
+        {
+            json = new JSONObject();
+            stats = new JSONArray();
+        }
+        else
+        {
+            stats = (JSONArray) json.get("stats");
+        }
+
+        try
+        {
+            stats = parseLogStats(new FileInputStream(stressSettings.graph.temporaryLogFile), stats);
+        }
+        catch (FileNotFoundException e)
+        {
+            throw new RuntimeException(e);
+        }
+
+        json.put("title", stressSettings.graph.title);
+        json.put("stats", stats);
+        return json;
+    }
+}

tools/stress/src/org/apache/cassandra/stress/StressMetrics.java

@@ -177,10 +177,12 @@ public class StressMetrics
 
     public static final String HEADFORMAT = "%-10s%10s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%8s,%7s,%9s,%7s,%7s,%8s,%8s,%8s,%8s";
     public static final String ROWFORMAT =  "%-10s%10d,%8.0f,%8.0f,%8.0f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%8.1f,%7.1f,%9.5f,%7d,%7.0f,%8.0f,%8.0f,%8.0f,%8.0f";
+    public static final String[] HEADMETRICS = new String[]{"type", "total ops","op/s","pk/s","row/s","mean","med",".95",".99",".999","max","time","stderr", "errors", "gc: #", "max ms", "sum ms", "sdv ms", "mb"};
+    public static final String HEAD = String.format(HEADFORMAT, HEADMETRICS);
 
     private static void printHeader(String prefix, PrintStream output)
     {
-        output.println(prefix + String.format(HEADFORMAT, "type,", "total ops","op/s","pk/s","row/s","mean","med",".95",".99",".999","max","time","stderr", "errors", "gc: #", "max ms", "sum ms", "sdv ms", "mb"));
+        output.println(prefix + HEAD);
     }
 
     private static void printRow(String prefix, String type, TimingInterval interval, TimingInterval total, JmxCollector.GcStats gcStats, Uncertainty opRateUncertainty, PrintStream output)
@@ -233,6 +235,7 @@ public class StressMetrics
         output.println(String.format("stdev gc time(ms)         : %.0f", totalGcStats.sdvms));
         output.println("Total operation time      : " + DurationFormatUtils.formatDuration(
                 history.runTime(), "HH:mm:ss", true));
+        output.println(""); // Newline is important here to separate the aggregates section from the END or the next stress iteration
     }
 
     public static void summarise(List<String> ids, List<StressMetrics> summarise, PrintStream out, int historySampleCount)

tools/stress/src/org/apache/cassandra/stress/settings/CliOption.java

@@ -38,7 +38,8 @@ public enum CliOption
     LOG("Where to log progress to, and the interval at which to do it", SettingsLog.helpPrinter()),
     TRANSPORT("Custom transport factories", SettingsTransport.helpPrinter()),
     PORT("The port to connect to cassandra nodes on", SettingsPort.helpPrinter()),
-    SENDTO("-send-to", "Specify a stress server to send this command to", SettingsMisc.sendToDaemonHelpPrinter())
+    SENDTO("-send-to", "Specify a stress server to send this command to", SettingsMisc.sendToDaemonHelpPrinter()),
+    GRAPH("-graph", "Graph recorded metrics", SettingsGraph.helpPrinter())
     ;
 
     private static final Map<String, CliOption> LOOKUP;

tools/stress/src/org/apache/cassandra/stress/settings/SettingsGraph.java

@@ -0,0 +1,125 @@
+package org.apache.cassandra.stress.settings;
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+
+public class SettingsGraph implements Serializable
+{
+    public final String file;
+    public final String revision;
+    public final String title;
+    public final String operation;
+    public final File temporaryLogFile;
+
+    public SettingsGraph(GraphOptions options, SettingsCommand stressCommand)
+    {
+        file = options.file.value();
+        revision = options.revision.value();
+        title = options.revision.value() == null
+            ? "cassandra-stress - " + new SimpleDateFormat("yyyy-mm-dd hh:mm:ss").format(new Date())
+            : options.title.value();
+
+        operation = options.operation.value() == null
+            ? stressCommand.type.name()
+            : options.operation.value();
+
+        if (inGraphMode())
+        {
+            try
+            {
+                temporaryLogFile = File.createTempFile("cassandra-stress", ".log");
+            }
+            catch (IOException e)
+            {
+                throw new RuntimeException("Cannot open temporary file");
+            }
+        }
+        else
+        {
+            temporaryLogFile = null;
+        }
+    }
+
+    public boolean inGraphMode()
+    {
+        return this.file == null ? false : true;
+    }
+
+    // Option Declarations
+    private static final class GraphOptions extends GroupedOptions
+    {
+        final OptionSimple file = new OptionSimple("file=", ".*", null, "HTML file to create or append to", true);
+        final OptionSimple revision = new OptionSimple("revision=", ".*", "unknown", "Unique name to assign to the current configuration being stressed", false);
+        final OptionSimple title = new OptionSimple("title=", ".*", null, "Title for chart (current date by default)", false);
+        final OptionSimple operation = new OptionSimple("op=", ".*", null, "Alternative name for current operation (stress op name used by default)", false);
+
+        @Override
+        public List<? extends Option> options()
+        {
+            return Arrays.asList(file, revision, title, operation);
+        }
+    }
+
+    // CLI Utility Methods
+    public static SettingsGraph get(Map<String, String[]> clArgs, SettingsCommand stressCommand)
+    {
+        String[] params = clArgs.remove("-graph");
+        if (params == null)
+        {
+            return new SettingsGraph(new GraphOptions(), stressCommand);
+        }
+        GraphOptions options = GroupedOptions.select(params, new GraphOptions());
+        if (options == null)
+        {
+            printHelp();
+            System.out.println("Invalid -graph options provided, see output for valid options");
+            System.exit(1);
+        }
+        return new SettingsGraph(options, stressCommand);
+    }
+
+    public static void printHelp()
+    {
+        GroupedOptions.printOptions(System.out, "-graph", new GraphOptions());
+    }
+
+    public static Runnable helpPrinter()
+    {
+        return new Runnable()
+        {
+            @Override
+            public void run()
+            {
+                printHelp();
+            }
+        };
+    }
+}
+

tools/stress/src/org/apache/cassandra/stress/settings/SettingsLog.java

@@ -21,10 +21,9 @@ package org.apache.cassandra.stress.settings;
  */
 
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.PrintStream;
-import java.io.Serializable;
+import org.apache.cassandra.stress.util.MultiPrintStream;
+
+import java.io.*;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -62,9 +61,15 @@ public class SettingsLog implements Serializable
         level = Level.valueOf(options.level.value().toUpperCase());
     }
 
-    public PrintStream getOutput() throws FileNotFoundException
+    public MultiPrintStream getOutput() throws FileNotFoundException
     {
-        return file == null ? new PrintStream(System.out) : new PrintStream(file);
+        // Always print to stdout regardless of whether we're graphing or not
+        MultiPrintStream stream = new MultiPrintStream(new PrintStream(System.out));
+
+        if (file != null)
+            stream.addStream(new PrintStream(file));
+
+        return stream;
     }
 
     // Option Declarations

tools/stress/src/org/apache/cassandra/stress/settings/StressSettings.java

@@ -54,8 +54,23 @@ public class StressSettings implements Serializable
     public final SettingsTransport transport;
     public final SettingsPort port;
     public final String sendToDaemon;
+    public final SettingsGraph graph;
 
-    public StressSettings(SettingsCommand command, SettingsRate rate, SettingsPopulation generate, SettingsInsert insert, SettingsColumn columns, SettingsSamples samples, SettingsErrors errors, SettingsLog log, SettingsMode mode, SettingsNode node, SettingsSchema schema, SettingsTransport transport, SettingsPort port, String sendToDaemon)
+    public StressSettings(SettingsCommand command,
+                          SettingsRate rate,
+                          SettingsPopulation generate,
+                          SettingsInsert insert,
+                          SettingsColumn columns,
+                          SettingsSamples samples,
+                          SettingsErrors errors,
+                          SettingsLog log,
+                          SettingsMode mode,
+                          SettingsNode node,
+                          SettingsSchema schema,
+                          SettingsTransport transport,
+                          SettingsPort port,
+                          String sendToDaemon,
+                          SettingsGraph graph)
     {
         this.command = command;
         this.rate = rate;
@@ -71,6 +86,7 @@ public class StressSettings implements Serializable
         this.transport = transport;
         this.port = port;
         this.sendToDaemon = sendToDaemon;
+        this.graph = graph;
     }
 
     private SmartThriftClient tclient;
@@ -262,6 +278,7 @@ public class StressSettings implements Serializable
         SettingsNode node = SettingsNode.get(clArgs);
         SettingsSchema schema = SettingsSchema.get(clArgs, command);
         SettingsTransport transport = SettingsTransport.get(clArgs);
+        SettingsGraph graph = SettingsGraph.get(clArgs, command);
         if (!clArgs.isEmpty())
         {
             printHelp();
@@ -278,7 +295,8 @@ public class StressSettings implements Serializable
             }
             System.exit(1);
         }
-        return new StressSettings(command, rate, generate, insert, columns, samples, errors, log, mode, node, schema, transport, port, sendToDaemon);
+
+        return new StressSettings(command, rate, generate, insert, columns, samples, errors, log, mode, node, schema, transport, port, sendToDaemon, graph);
     }
 
     private static Map<String, String[]> parseMap(String[] args)

tools/stress/src/org/apache/cassandra/stress/util/MultiPrintStream.java

@@ -0,0 +1,288 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.cassandra.stress.util;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+/** PrintStream that multiplexes to multiple streams */
+public class MultiPrintStream extends PrintStream
+{
+    private List<PrintStream> newStreams;
+
+    public MultiPrintStream(PrintStream baseStream)
+    {
+        super(baseStream);
+        this.newStreams = new ArrayList();
+    }
+
+    public void addStream(PrintStream printStream)
+    {
+        newStreams.add(printStream);
+    }
+
+    @Override
+    public void flush()
+    {
+        super.flush();
+        for (PrintStream s : newStreams)
+            s.flush();
+    }
+
+    @Override
+    public void close()
+    {
+        super.close();
+        for (PrintStream s : newStreams)
+            s.close();
+    }
+
+    @Override
+    public boolean checkError()
+    {
+        boolean error = super.checkError();
+        for (PrintStream s : newStreams)
+        {
+            if (s.checkError())
+                error = true;
+        }
+        return error;
+    }
+
+    @Override
+    public void write(int b)
+    {
+        super.write(b);
+        for (PrintStream s: newStreams)
+            s.write(b);
+    }
+
+    @Override
+    public void write(byte[] buf, int off, int len)
+    {
+        super.write(buf, off, len);
+        for (PrintStream s: newStreams)
+            s.write(buf, off, len);
+    }
+
+    @Override
+    public void print(boolean b)
+    {
+        super.print(b);
+        for (PrintStream s: newStreams)
+            s.print(b);
+    }
+
+    @Override
+    public void print(char c)
+    {
+        super.print(c);
+        for (PrintStream s: newStreams)
+            s.print(c);
+    }
+
+    @Override
+    public void print(int i)
+    {
+        super.print(i);
+        for (PrintStream s: newStreams)
+            s.print(i);
+    }
+
+    @Override
+    public void print(long l)
+    {
+        super.print(l);
+        for (PrintStream s: newStreams)
+            s.print(l);
+    }
+
+    @Override
+    public void print(float f)
+    {
+        super.print(f);
+        for (PrintStream s: newStreams)
+            s.print(f);
+    }
+
+    @Override
+    public void print(double d)
+    {
+        super.print(d);
+        for (PrintStream s: newStreams)
+            s.print(d);
+    }
+
+    @Override
+    public void print(char[] s)
+    {
+        super.print(s);
+        for (PrintStream stream: newStreams)
+            stream.print(s);
+    }
+
+    @Override
+    public void print(String s)
+    {
+        super.print(s);
+        for (PrintStream stream: newStreams)
+            stream.print(s);
+    }
+
+    @Override
+    public void print(Object obj)
+    {
+        super.print(obj);
+        for (PrintStream s: newStreams)
+            s.print(obj);
+    }
+
+    @Override
+    public void println()
+    {
+        super.println();
+        for (PrintStream s: newStreams)
+            s.println();
+    }
+
+    @Override
+    public void println(boolean x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(char x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(int x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(long x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(float x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(double x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(char[] x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(String x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public void println(Object x)
+    {
+        super.println(x);
+        for (PrintStream s: newStreams)
+            s.println(x);
+    }
+
+    @Override
+    public PrintStream printf(String format, Object... args)
+    {
+        for (PrintStream s: newStreams)
+            s.printf(format, args);
+        return super.printf(format, args);
+    }
+
+    @Override
+    public PrintStream printf(Locale l, String format, Object... args)
+    {
+        for (PrintStream s: newStreams)
+            s.printf(l, format, args);
+        return super.printf(l, format, args);
+    }
+
+    @Override
+    public PrintStream append(CharSequence csq)
+    {
+        for (PrintStream s: newStreams)
+            s.append(csq);
+        return super.append(csq);
+    }
+
+    @Override
+    public PrintStream append(CharSequence csq, int start, int end)
+    {
+        for (PrintStream s: newStreams)
+            s.append(csq, start, end);
+        return super.append(csq, start, end);
+    }
+
+    @Override
+    public PrintStream append(char c)
+    {
+        for (PrintStream s: newStreams)
+            s.append(c);
+        return super.append(c);
+    }
+
+    @Override
+    public void write(byte[] b) throws IOException
+    {
+        super.write(b);
+        for (PrintStream s: newStreams)
+            s.write(b);
+    }
+
+}