From aedefd07bc46878e928ef702040cf3ec14fb228c Mon Sep 17 00:00:00 2001 From: jlizier Date: Mon, 22 May 2017 15:53:42 +1000 Subject: [PATCH 01/78] Fixed analytic expressions for transfer entropies in the simple demos for coupled Gaussians (should have computed directly from the empirical correlations all along, but didn't notice because errors were very small) --- .../clojure/examples/example3TeContinuousDataKernel.clj | 1 + .../clojure/examples/example4TeContinuousDataKraskov.clj | 1 + .../demos/Example3TeContinuousDataKernel.java | 9 +++++++-- .../demos/Example4TeContinuousDataKraskov.java | 7 ++++++- demos/julia/example3TeContinuousDataKernel.jl | 8 ++++++-- demos/julia/example4TeContinuousDataKraskov.jl | 6 +++++- demos/octave/example3TeContinuousDataKernel.m | 8 ++++++-- demos/octave/example4TeContinuousDataKraskov.m | 6 +++++- demos/python/example3TeContinuousDataKernel.py | 8 ++++++-- demos/python/example4TeContinuousDataKraskov.py | 6 +++++- demos/r/example3TeContinuousDataKernel.r | 8 ++++++-- demos/r/example4TeContinuousDataKraskov.r | 6 +++++- 12 files changed, 59 insertions(+), 15 deletions(-) diff --git a/demos/clojure/examples/example3TeContinuousDataKernel.clj b/demos/clojure/examples/example3TeContinuousDataKernel.clj index ea6d63a..1dc5af2 100755 --- a/demos/clojure/examples/example3TeContinuousDataKernel.clj +++ b/demos/clojure/examples/example3TeContinuousDataKernel.clj @@ -45,6 +45,7 @@ (.setObservations teCalc sourceArray destArray) ; For copied source, should give something close to expected value for correlated Gaussians: +; TODO The analytic result quoted here isn't quite right, see e.g. octave demos (can't be bothered fixing here...) (println "TE result " (.computeAverageLocalOfObservations teCalc) " expected to be close to " (/ (Math/log (/ 1 (- 1 (* covariance covariance)))) (Math/log 2)) " for these correlated Gaussians but biased upward") diff --git a/demos/clojure/examples/example4TeContinuousDataKraskov.clj b/demos/clojure/examples/example4TeContinuousDataKraskov.clj index bb63fe7..1c04be0 100755 --- a/demos/clojure/examples/example4TeContinuousDataKraskov.clj +++ b/demos/clojure/examples/example4TeContinuousDataKraskov.clj @@ -49,6 +49,7 @@ ; data is a set of random variables) - the result will be of the order ; of what we expect, but not exactly equal to it; in fact, there will ; be a large variance around it. +; TODO The analytic result quoted here isn't quite right, see e.g. octave demos (can't be bothered fixing here...) (println "TE result " (.computeAverageLocalOfObservations teCalc) " nats expected to be close to " (Math/log (/ 1 (- 1 (* covariance covariance)))) " nats for these correlated Gaussians") diff --git a/demos/java/infodynamics/demos/Example3TeContinuousDataKernel.java b/demos/java/infodynamics/demos/Example3TeContinuousDataKernel.java index c65595b..4775f83 100755 --- a/demos/java/infodynamics/demos/Example3TeContinuousDataKernel.java +++ b/demos/java/infodynamics/demos/Example3TeContinuousDataKernel.java @@ -58,11 +58,16 @@ public class Example3TeContinuousDataKernel { teCalc.setProperty("NORMALISE", "true"); // Normalise the individual variables (default) teCalc.initialise(1, 0.5); // Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units teCalc.setObservations(sourceArray, destArray); - // For copied source, should give something close to expected value for correlated Gaussians: double result = teCalc.computeAverageLocalOfObservations(); + // For copied source, should give something close to expected value for correlated Gaussians:. + // Expected correlation is expected covariance / product of expected standard deviations: + // (where square of destArray standard dev is sum of squares of std devs of + // underlying distributions) + double corr_expected = covariance / + (1.0 * Math.sqrt(Math.pow(covariance,2) + Math.pow(1-covariance,2))); System.out.printf("TE result %.4f bits; expected to be close to " + "%.4f bits for these correlated Gaussians but biased upwards\n", - result, Math.log(1.0/(1-Math.pow(covariance,2)))/Math.log(2)); + result, -0.5 * Math.log(1-Math.pow(corr_expected,2))/Math.log(2)); teCalc.initialise(); // Initialise leaving the parameters the same teCalc.setObservations(sourceArray2, destArray); diff --git a/demos/java/infodynamics/demos/Example4TeContinuousDataKraskov.java b/demos/java/infodynamics/demos/Example4TeContinuousDataKraskov.java index ccbc221..a1276d4 100755 --- a/demos/java/infodynamics/demos/Example4TeContinuousDataKraskov.java +++ b/demos/java/infodynamics/demos/Example4TeContinuousDataKraskov.java @@ -65,9 +65,14 @@ public class Example4TeContinuousDataKraskov { // data is a set of random variables) - the result will be of the order // of what we expect, but not exactly equal to it; in fact, there will // be a large variance around it. + // Expected correlation is expected covariance / product of expected standard deviations: + // (where square of destArray standard dev is sum of squares of std devs of + // underlying distributions) + double corr_expected = covariance / + (1.0 * Math.sqrt(Math.pow(covariance,2) + Math.pow(1-covariance,2))); System.out.printf("TE result %.4f nats; expected to be close to " + "%.4f nats for these correlated Gaussians\n", - result, Math.log(1.0/(1-Math.pow(covariance,2)))); + result, -0.5 * Math.log(1-Math.pow(corr_expected,2))); // Perform calculation with uncorrelated source: teCalc.initialise(); // Initialise leaving the parameters the same diff --git a/demos/julia/example3TeContinuousDataKernel.jl b/demos/julia/example3TeContinuousDataKernel.jl index 2a9572a..ee3f115 100755 --- a/demos/julia/example3TeContinuousDataKernel.jl +++ b/demos/julia/example3TeContinuousDataKernel.jl @@ -42,9 +42,13 @@ jcall(teCalc, "setProperty", Void, (JString, JString), "NORMALISE", "true"); # N jcall(teCalc, "initialise", Void, (jint, jdouble), 1, 0.5); # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units jcall(teCalc, "setObservations", Void, (Array{jdouble,1}, Array{jdouble,1}), sourceArray, destArray); -# For copied source, should give something close to expected value for correlated Gaussians: result = jcall(teCalc, "computeAverageLocalOfObservations", jdouble, ()); -@printf("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n", result, log(1/(1-covariance^2))/log(2)); +# For copied source, should give something close to expected value for correlated Gaussians: +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected = covariance / (1 * sqrt(covariance^2 + (1-covariance)^2)); +@printf("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n", result, -0.5*log(1-corr_expected^2)/log(2)); jcall(teCalc, "initialise", Void, ()); # Initialise leaving the parameters the same jcall(teCalc, "setObservations", Void, (Array{jdouble,1}, Array{jdouble,1}), diff --git a/demos/julia/example4TeContinuousDataKraskov.jl b/demos/julia/example4TeContinuousDataKraskov.jl index 1a58930..a69dc7f 100755 --- a/demos/julia/example4TeContinuousDataKraskov.jl +++ b/demos/julia/example4TeContinuousDataKraskov.jl @@ -49,8 +49,12 @@ result = jcall(teCalc, "computeAverageLocalOfObservations", jdouble, ()); # data is a set of random variables) - the result will be of the order # of what we expect, but not exactly equal to it; in fact, there will # be a large variance around it. +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected = covariance / (1 * sqrt(covariance^2 + (1-covariance)^2)); @printf("TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians\n", - result, log(1/(1-covariance^2))); + result, -0.5*log(1-corr_expected^2)); # Perform calculation with uncorrelated source: jcall(teCalc, "initialise", Void, ()); # Initialise leaving the parameters the same diff --git a/demos/octave/example3TeContinuousDataKernel.m b/demos/octave/example3TeContinuousDataKernel.m index 594cf39..212af5a 100755 --- a/demos/octave/example3TeContinuousDataKernel.m +++ b/demos/octave/example3TeContinuousDataKernel.m @@ -34,10 +34,14 @@ teCalc=javaObject('infodynamics.measures.continuous.kernel.TransferEntropyCalcul teCalc.setProperty('NORMALISE', 'true'); % Normalise the individual variables teCalc.initialise(1, 0.5); % Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units teCalc.setObservations(sourceArray, destArray); -% For copied source, should give something close to expected value for correlated Gaussians: result = teCalc.computeAverageLocalOfObservations(); +% For copied source, should give something close to expected value for correlated Gaussians. +% Expected correlation is expected covariance / product of expected standard deviations: +% (where square of destArray standard dev is sum of squares of std devs of +% underlying distributions) +corr_expected = covariance ./ (1 * sqrt(covariance^2 + (1-covariance)^2)); fprintf('TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n', ... - result, log(1/(1-covariance^2))/log(2)); + result, -0.5*log(1-corr_expected^2)/log(2)); teCalc.initialise(); % Initialise leaving the parameters the same teCalc.setObservations(sourceArray2, destArray); % For random source, it should give something close to 0 bits diff --git a/demos/octave/example4TeContinuousDataKraskov.m b/demos/octave/example4TeContinuousDataKraskov.m index 7d353c9..5e43395 100755 --- a/demos/octave/example4TeContinuousDataKraskov.m +++ b/demos/octave/example4TeContinuousDataKraskov.m @@ -40,8 +40,12 @@ result = teCalc.computeAverageLocalOfObservations(); % data is a set of random variables) - the result will be of the order % of what we expect, but not exactly equal to it; in fact, there will % be a large variance around it. +% Expected correlation is expected covariance / product of expected standard deviations: +% (where square of destArray standard dev is sum of squares of std devs of +% underlying distributions) +corr_expected = covariance ./ (1 * sqrt(covariance^2 + (1-covariance)^2)); fprintf('TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians\n', ... - result, log(1/(1-covariance^2))); + result, - 0.5 * log(1-corr_expected^2)); % Perform calculation with uncorrelated source: teCalc.initialise(); % Initialise leaving the parameters the same teCalc.setObservations(sourceArray2, destArray); diff --git a/demos/python/example3TeContinuousDataKernel.py b/demos/python/example3TeContinuousDataKernel.py index f29b027..bf7f0e6 100755 --- a/demos/python/example3TeContinuousDataKernel.py +++ b/demos/python/example3TeContinuousDataKernel.py @@ -45,10 +45,14 @@ teCalc = teCalcClass() teCalc.setProperty("NORMALISE", "true") # Normalise the individual variables teCalc.initialise(1, 0.5) # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units teCalc.setObservations(JArray(JDouble, 1)(sourceArray), JArray(JDouble, 1)(destArray)) -# For copied source, should give something close to 1 bit: result = teCalc.computeAverageLocalOfObservations() +# For copied source, should give something close to 1 bit: +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected = covariance / (1 * math.sqrt(covariance**2 + (1-covariance)**2)); print("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards" % \ - (result, math.log(1/(1-math.pow(covariance,2)))/math.log(2))) + (result, -0.5*math.log(1-math.pow(corr_expected,2))/math.log(2))) teCalc.initialise() # Initialise leaving the parameters the same teCalc.setObservations(JArray(JDouble, 1)(sourceArray2), JArray(JDouble, 1)(destArray)) # For random source, it should give something close to 0 bits diff --git a/demos/python/example4TeContinuousDataKraskov.py b/demos/python/example4TeContinuousDataKraskov.py index ac65bab..6f9132b 100755 --- a/demos/python/example4TeContinuousDataKraskov.py +++ b/demos/python/example4TeContinuousDataKraskov.py @@ -52,8 +52,12 @@ result = teCalc.computeAverageLocalOfObservations() # data is a set of random variables) - the result will be of the order # of what we expect, but not exactly equal to it; in fact, there will # be a large variance around it. +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected = covariance / (1 * math.sqrt(covariance**2 + (1-covariance)**2)); print("TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians" % \ - (result, math.log(1/(1-math.pow(covariance,2))))) + (result, -0.5 * math.log(1-corr_expected**2))) # Perform calculation with uncorrelated source: teCalc.initialise() # Initialise leaving the parameters the same teCalc.setObservations(JArray(JDouble, 1)(sourceArray2), JArray(JDouble, 1)(destArray)) diff --git a/demos/r/example3TeContinuousDataKernel.r b/demos/r/example3TeContinuousDataKernel.r index 44e557b..47e7393 100755 --- a/demos/r/example3TeContinuousDataKernel.r +++ b/demos/r/example3TeContinuousDataKernel.r @@ -41,9 +41,13 @@ teCalc<-.jnew("infodynamics/measures/continuous/kernel/TransferEntropyCalculator .jcall(teCalc,"V","setProperty", "NORMALISE", "true") # Normalise the individual variables .jcall(teCalc,"V","initialise", 1L, 0.5) # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units .jcall(teCalc,"V","setObservations", sourceArray, destArray) -# For copied source, should give something close to expected value for correlated Gaussians: result <- .jcall(teCalc,"D","computeAverageLocalOfObservations") -cat("TE result ", result, "bits; expected to be close to ", log(1/(1-covariance^2))/log(2), " bits for these correlated Gaussians but biased upwards\n") +# For copied source, should give something close to expected value for correlated Gaussians. +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected <- covariance / (1 * sqrt(covariance^2 + (1-covariance)^2)); +cat("TE result ", result, "bits; expected to be close to ", -0.5*log(1-corr_expected^2)/log(2), " bits for these correlated Gaussians but biased upwards\n") .jcall(teCalc,"V","initialise") # Initialise leaving the parameters the same .jcall(teCalc,"V","setObservations", sourceArray2, destArray) diff --git a/demos/r/example4TeContinuousDataKraskov.r b/demos/r/example4TeContinuousDataKraskov.r index d2ea19c..3bdb982 100755 --- a/demos/r/example4TeContinuousDataKraskov.r +++ b/demos/r/example4TeContinuousDataKraskov.r @@ -48,7 +48,11 @@ result <- .jcall(teCalc,"D","computeAverageLocalOfObservations") # data is a set of random variables) - the result will be of the order # of what we expect, but not exactly equal to it; in fact, there will # be a large variance around it. -cat("TE result ", result, "nats; expected to be close to ", log(1/(1-covariance^2)), " nats for these correlated Gaussians\n") +# Expected correlation is expected covariance / product of expected standard deviations: +# (where square of destArray standard dev is sum of squares of std devs of +# underlying distributions) +corr_expected <- covariance / (1 * sqrt(covariance^2 + (1-covariance)^2)); +cat("TE result ", result, "nats; expected to be close to ", -0.5*log(1-corr_expected^2), " nats for these correlated Gaussians\n") # Perform calculation with uncorrelated source: .jcall(teCalc,"V","initialise") # Initialise leaving the parameters the same From b05399e40f2fdc28202f3e0a7b5ec499f26213e4 Mon Sep 17 00:00:00 2001 From: jlizier Date: Mon, 22 May 2017 15:55:17 +1000 Subject: [PATCH 02/78] Fixed octave-matlab translation error in plotting CAs --- demos/octave/CellularAutomata/plotRawCa.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/octave/CellularAutomata/plotRawCa.m b/demos/octave/CellularAutomata/plotRawCa.m index 368876f..650f648 100755 --- a/demos/octave/CellularAutomata/plotRawCa.m +++ b/demos/octave/CellularAutomata/plotRawCa.m @@ -40,7 +40,7 @@ function plotRawCa(states, rule, plotOptions, saveIt, saveFormat) plotOptions.plotRows = size(states, 1); end if not(isfield(plotOptions, 'plotCols')) - plotOptions.plotCols = columns(states); + plotOptions.plotCols = size(states, 2); end if not(isfield(plotOptions, 'plotStartRow')) plotOptions.plotStartRow = 1; From b2218a976dacb1081b4f393ccaebd265e2bb3c6a Mon Sep 17 00:00:00 2001 From: jlizier Date: Mon, 22 May 2017 15:57:33 +1000 Subject: [PATCH 03/78] In CA running demo, adding capability for rule table specifying CA to be provided rather than a string or integer rule number --- demos/octave/CellularAutomata/runCA.m | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/demos/octave/CellularAutomata/runCA.m b/demos/octave/CellularAutomata/runCA.m index 969cc8a..3c39c4f 100755 --- a/demos/octave/CellularAutomata/runCA.m +++ b/demos/octave/CellularAutomata/runCA.m @@ -44,8 +44,8 @@ % - cells - number of cells in the CA % - steps - number of rows to execute the CA for (including the random initial row) % - debug - turn on various debug messages -% - seedOrState - if a scalar, it is the state input for the random number generator (so one can repeat CA investigations for the same initial state). -% - if a vector, it is the initial state for the CA (must be of length cells) +% - seedOrState (optional) - if a scalar, it is the state input for the random number generator (so one can repeat CA investigations for the same initial state). +% - if a vector, it is the initial state for the CA (must be of length cells) % % Outputs: % - caStates - a run, from random initial conditions, of a CA of the given parameters. @@ -128,7 +128,7 @@ function [caStates, ruleTable, executedRules] = runCA(neighbourhood, base, rule, fprintf('%d', ruleTable(i)); end fprintf('\n'); - else + elseif (isscalar(rule)) % The rule is specified as an integer if (rule > base .^ (base .^ neighbourhood) - 1) error(sprintf('Rule %d is not within the limits of this base %d and neighbourhood %d (max is %d)', ... @@ -156,6 +156,12 @@ function [caStates, ruleTable, executedRules] = runCA(neighbourhood, base, rule, fprintf('%d', ruleTable(i)); end fprintf('\n'); + else + % We have a vector supplied for the rule, as a rule table already + if (length(rule) ~= base .^ neighbourhood) + error('Rule table vector (length %d) is not of expected length %d\n', length(rule), base .^ neighbourhood); + end + ruleTable = rule; end caStates = zeros(steps, cells); From 2e4a58d2c93721e8decf447ac9926772715a9bfd Mon Sep 17 00:00:00 2001 From: Pedro Martinez Mediano Date: Thu, 25 May 2017 03:04:21 +1000 Subject: [PATCH 04/78] Turn off GPU by default in master branch. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 728ca0b..67210b0 100755 --- a/build.xml +++ b/build.xml @@ -21,7 +21,7 @@ - + From ab393a6d1fc8d5b0bc61ec1c64d9b3e8ca52bd2c Mon Sep 17 00:00:00 2001 From: Joseph Lizier Date: Fri, 26 May 2017 13:27:15 +1000 Subject: [PATCH 05/78] Added Pedro to the main authors list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a2550a5..b17cbdf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Java Information Dynamics Toolkit (JIDT) -Copyright (C) 2012-2014 [Joseph T. Lizier](http://lizier.me/joseph/); 2014-2016 [Joseph T. Lizier](http://lizier.me/joseph/) and Ipek Özdemir +Copyright (C) 2012-2014 [Joseph T. Lizier](http://lizier.me/joseph/); 2014-2016 [Joseph T. Lizier](http://lizier.me/joseph/) and Ipek Özdemir; 2017- [Joseph T. Lizier](http://lizier.me/joseph/), Ipek Özdemir and [Pedro Mediano](https://www.doc.ic.ac.uk/~pam213/) *JIDT* provides a stand-alone, open-source code Java implementation (also usable in [Matlab, Octave](../../wiki/UseInOctaveMatlab), [Python](../../wiki/UseInPython), [R](../../wiki/UseInR), [Julia](../../wiki/UseInJulia) and [Clojure](../../wiki/UseInClojure)) of information-theoretic measures of distributed computation in complex systems: i.e. information storage, transfer and modification. From 4b8e26fc05c94d442684905e323720444182028c Mon Sep 17 00:00:00 2001 From: jlizier Date: Tue, 13 Jun 2017 12:03:56 +1000 Subject: [PATCH 06/78] Changed various computeLocal methods from private to public (these were mistakenly private) --- .../ConditionalTransferEntropyCalculatorDiscrete.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java index 68f0125..26e0722 100644 --- a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java +++ b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java @@ -640,6 +640,7 @@ public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCal // (Not necessary to check for distinct random perturbations) int[][] newOrderings = rg.generateRandomPerturbations(observations, numPermutationsToCheck); + // TODO stop using deprecated method ConditionalTransferEntropyCalculatorDiscrete cte = newInstance(base, k, numOtherInfoContributors); cte.initialise(); cte.observations = observations; @@ -897,7 +898,7 @@ public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCal * (indexed only by time step) * @return time-series of local conditional TE values */ - private double[] computeLocalFromPreviousObservations + public double[] computeLocalFromPreviousObservations (int source[], int dest[], int[] conditional){ int rows = source.length; @@ -955,7 +956,7 @@ public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCal * (indexed first by time step then by variable number) * @return time-series of local conditional TE values */ - private double[] computeLocalFromPreviousObservations + public double[] computeLocalFromPreviousObservations (int source[], int dest[], int[][] conditionals){ int rows = source.length; From 77f4c1afd5c494815a22f788bd65063646dc7ad0 Mon Sep 17 00:00:00 2001 From: jlizier Date: Tue, 13 Jun 2017 12:04:41 +1000 Subject: [PATCH 07/78] Added partialNeighbourCountFromObservations() method for Leo's analytic work on neighbour counts --- ...ualInfoCalculatorMultiVariateKraskov1.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) mode change 100755 => 100644 java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java diff --git a/java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java b/java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java old mode 100755 new mode 100644 index 363d16c..6e3c786 --- a/java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java +++ b/java/source/infodynamics/measures/continuous/kraskov/MutualInfoCalculatorMultiVariateKraskov1.java @@ -118,5 +118,48 @@ public class MutualInfoCalculatorMultiVariateKraskov1 } else { return new double[] {sumDiGammas, sumNx, sumNy}; } + } + + public int[][] partialNeighbourCountFromObservations( + int startTimePoint, int numTimePoints) throws Exception { + + double startTime = Calendar.getInstance().getTimeInMillis(); + + ensureKdTreesConstructed(); + + int[][] neighbourCounts = new int[numTimePoints][2]; + + for (int t = startTimePoint; t < startTimePoint + numTimePoints; t++) { + // Compute eps for this time step by + // finding the kth closest neighbour for point t: + PriorityQueue nnPQ = + kdTreeJoint.findKNearestNeighbours(k, t, dynCorrExclTime); + // First element in the PQ is the kth NN, + // and epsilon = kthNnData.distance + NeighbourNodeData kthNnData = nnPQ.poll(); + + // Count the number of points whose x distance is less + // than eps, and whose y distance is less than + // epsilon = kthNnData.distance + int n_x = nnSearcherSource.countPointsStrictlyWithinR( + t, kthNnData.distance, dynCorrExclTime); + int n_y = nnSearcherDest.countPointsStrictlyWithinR( + t, kthNnData.distance, dynCorrExclTime); + + neighbourCounts[t - startTimePoint][0] = n_x; + neighbourCounts[t - startTimePoint][1] = n_y; + + } + + if (debug) { + Calendar rightNow2 = Calendar.getInstance(); + long endTime = rightNow2.getTimeInMillis(); + System.out.println("Subset " + startTimePoint + ":" + + (startTimePoint + numTimePoints) + " Calculation time: " + + ((endTime - startTime)/1000.0) + " sec" ); + } + + return neighbourCounts; + } } From ea40f48bbc7bd18bbdd399b08ebf1320d6fb5c2f Mon Sep 17 00:00:00 2001 From: jlizier Date: Wed, 14 Jun 2017 11:09:01 +1000 Subject: [PATCH 08/78] Adding new methods computePValueForGivenEstimate() and computeEstimateForGivenPValue to AnalyticMeasurementDistribution and implementing in ChiSquareMeasurementDistribution. Involves importing (and refactoring) a significantly larger chunk of commons.maths3 classes (and updating existing ones to latest 3.6.1 version for consistency). Also addresses issue #23 in changing the actual value of the ChiSquareMeasurementDistribution to be that of the information theoretic measurement rather than 2*N times it (which was the value that is actually chi squared distributed), which also necessitates changes to the Discrete and Gaussian calculators computeSignificance() methods. --- ...ualInfoCalculatorMultiVariateGaussian.java | 3 +- ...ualInfoCalculatorMultiVariateGaussian.java | 4 +- ...alMutualInformationCalculatorDiscrete.java | 3 +- .../MutualInformationCalculatorDiscrete.java | 3 +- .../TransferEntropyCalculatorDiscrete.java | 5 +- .../AnalyticMeasurementDistribution.java | 77 +- .../ChiSquareMeasurementDistribution.java | 45 +- .../source/infodynamics/utils/MathsUtils.java | 17 +- .../utils/commonsmath3/Field.java | 87 + .../utils/commonsmath3/FieldElement.java | 116 + .../utils/commonsmath3/RealFieldElement.java | 431 ++++ .../analysis/UnivariateFunction.java | 110 + .../solvers/AbstractUnivariateSolver.java | 89 + .../analysis/solvers/AllowedSolution.java | 104 + .../solvers/BaseAbstractUnivariateSolver.java | 347 +++ .../solvers/BaseUnivariateSolver.java | 171 ++ .../solvers/BracketedUnivariateSolver.java | 121 + .../analysis/solvers/BrentSolver.java | 272 +++ .../analysis/solvers/UnivariateSolver.java | 57 + .../solvers/UnivariateSolverUtils.java | 496 +++++ .../AbstractIntegerDistribution.java | 282 +++ .../AbstractRealDistribution.java | 336 +++ .../distribution/BetaDistribution.java | 435 ++++ .../distribution/BinomialDistribution.java | 227 ++ .../distribution/CauchyDistribution.java | 285 +++ .../distribution/ChiSquaredDistribution.java | 225 ++ .../distribution/ExponentialDistribution.java | 380 ++++ .../distribution/FDistribution.java | 357 +++ .../distribution/GammaDistribution.java | 542 +++++ .../HypergeometricDistribution.java | 376 ++++ .../distribution/IntegerDistribution.java | 184 ++ .../distribution/NormalDistribution.java | 340 +++ .../distribution/PascalDistribution.java | 277 +++ .../distribution/PoissonDistribution.java | 424 ++++ .../distribution/RealDistribution.java | 225 ++ .../distribution/SaddlePointExpansion.java | 229 ++ .../distribution/TDistribution.java | 301 +++ .../UniformIntegerDistribution.java | 210 ++ .../distribution/WeibullDistribution.java | 382 ++++ .../distribution/ZipfDistribution.java | 517 +++++ .../exception/ConvergenceException.java | 7 +- .../exception/DimensionMismatchException.java | 5 +- .../exception/MathArithmeticException.java | 5 +- .../MathIllegalArgumentException.java | 5 +- .../exception/MathIllegalNumberException.java | 5 +- .../exception/MathIllegalStateException.java | 10 +- .../exception/MathInternalError.java | 86 + .../MathUnsupportedOperationException.java | 101 + .../exception/MaxCountExceededException.java | 5 +- .../exception/NoBracketingException.java | 135 ++ .../exception/NoDataException.java | 75 + .../NonMonotonicSequenceException.java | 149 ++ .../exception/NotANumberException.java | 66 + .../exception/NotFiniteNumberException.java | 83 + .../exception/NotPositiveException.java | 77 + .../NotStrictlyPositiveException.java | 78 + .../exception/NullArgumentException.java | 80 + .../exception/NumberIsTooLargeException.java | 5 +- .../exception/NumberIsTooSmallException.java | 5 +- .../exception/OutOfRangeException.java | 107 + .../TooManyEvaluationsException.java | 68 + .../commonsmath3/exception/ZeroException.java | 77 + .../commonsmath3/exception/util/ArgUtils.java | 5 +- .../exception/util/ExceptionContext.java | 13 +- .../util/ExceptionContextProvider.java | 5 +- .../exception/util/Localizable.java | 5 +- .../exception/util/LocalizedFormats.java | 17 +- .../commonsmath3/random/AbstractWell.java | 216 ++ .../random/BitsStreamGenerator.java | 299 +++ .../utils/commonsmath3/random/RandomData.java | 293 +++ .../random/RandomDataGenerator.java | 811 +++++++ .../commonsmath3/random/RandomDataImpl.java | 614 ++++++ .../commonsmath3/random/RandomGenerator.java | 176 ++ .../random/RandomGeneratorFactory.java | 150 ++ .../utils/commonsmath3/random/Well19937c.java | 143 ++ .../utils/commonsmath3/special/Beta.java | 542 +++++ .../utils/commonsmath3/special/Erf.java | 272 +++ .../utils/commonsmath3/special/Gamma.java | 17 +- .../commonsmath3/util/ArithmeticUtils.java | 936 ++++++++ .../utils/commonsmath3/util/Combinations.java | 434 ++++ .../commonsmath3/util/CombinatoricsUtils.java | 491 +++++ .../commonsmath3/util/ContinuedFraction.java | 5 +- .../utils/commonsmath3/util/DoubleArray.java | 139 ++ .../utils/commonsmath3/util/FastMath.java | 843 +++++-- .../utils/commonsmath3/util/FastMathCalc.java | 5 +- .../util/FastMathLiteralArrays.java | 5 +- .../commonsmath3/util/IntegerSequence.java | 391 ++++ .../utils/commonsmath3/util/MathArrays.java | 1964 +++++++++++++++++ .../utils/commonsmath3/util/MathUtils.java | 338 +++ .../utils/commonsmath3/util/Precision.java | 44 +- .../util/ResizableDoubleArray.java | 1239 +++++++++++ .../ChiSquareMeasurementDistributionTest.java | 52 + .../infodynamics/utils/MathsUtilsTest.java | 68 + 93 files changed, 20540 insertions(+), 318 deletions(-) create mode 100644 java/source/infodynamics/utils/commonsmath3/Field.java create mode 100644 java/source/infodynamics/utils/commonsmath3/FieldElement.java create mode 100644 java/source/infodynamics/utils/commonsmath3/RealFieldElement.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/UnivariateFunction.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/AbstractUnivariateSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/AllowedSolution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseAbstractUnivariateSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseUnivariateSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/BracketedUnivariateSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/BrentSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolver.java create mode 100644 java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolverUtils.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/AbstractIntegerDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/AbstractRealDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/BetaDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/BinomialDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/CauchyDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/ChiSquaredDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/ExponentialDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/FDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/GammaDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/HypergeometricDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/IntegerDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/NormalDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/PascalDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/PoissonDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/RealDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/SaddlePointExpansion.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/TDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/UniformIntegerDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/WeibullDistribution.java create mode 100644 java/source/infodynamics/utils/commonsmath3/distribution/ZipfDistribution.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/ConvergenceException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/DimensionMismatchException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/MathArithmeticException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/MathIllegalArgumentException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/MathIllegalNumberException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/MathIllegalStateException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/MathInternalError.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/MathUnsupportedOperationException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/MaxCountExceededException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NoBracketingException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NoDataException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NonMonotonicSequenceException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NotANumberException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NotFiniteNumberException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NotPositiveException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NotStrictlyPositiveException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/NullArgumentException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooLargeException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooSmallException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/OutOfRangeException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/TooManyEvaluationsException.java create mode 100644 java/source/infodynamics/utils/commonsmath3/exception/ZeroException.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/util/ArgUtils.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContext.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContextProvider.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/util/Localizable.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/exception/util/LocalizedFormats.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/AbstractWell.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/BitsStreamGenerator.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/RandomData.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/RandomDataGenerator.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/RandomDataImpl.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/RandomGenerator.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/RandomGeneratorFactory.java create mode 100644 java/source/infodynamics/utils/commonsmath3/random/Well19937c.java create mode 100644 java/source/infodynamics/utils/commonsmath3/special/Beta.java create mode 100644 java/source/infodynamics/utils/commonsmath3/special/Erf.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/special/Gamma.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/ArithmeticUtils.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/Combinations.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/CombinatoricsUtils.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/util/ContinuedFraction.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/DoubleArray.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/util/FastMath.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/util/FastMathCalc.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/util/FastMathLiteralArrays.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/IntegerSequence.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/MathArrays.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/MathUtils.java mode change 100755 => 100644 java/source/infodynamics/utils/commonsmath3/util/Precision.java create mode 100644 java/source/infodynamics/utils/commonsmath3/util/ResizableDoubleArray.java create mode 100644 java/unittests/infodynamics/utils/ChiSquareMeasurementDistributionTest.java diff --git a/java/source/infodynamics/measures/continuous/gaussian/ConditionalMutualInfoCalculatorMultiVariateGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/ConditionalMutualInfoCalculatorMultiVariateGaussian.java index 8a7aed7..6521a60 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/ConditionalMutualInfoCalculatorMultiVariateGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/ConditionalMutualInfoCalculatorMultiVariateGaussian.java @@ -528,7 +528,8 @@ public class ConditionalMutualInfoCalculatorMultiVariateGaussian // return new ChiSquareMeasurementDistribution(2.0*((double)totalObservations)*lastAverage, // dimensionsVar1 * dimensionsVar2); // Taking the subsets into account: - return new ChiSquareMeasurementDistribution(2.0*((double)totalObservations)*lastAverage, + return new ChiSquareMeasurementDistribution(lastAverage, + totalObservations, var1IndicesInCovariance.length * var2IndicesInCovariance.length); } diff --git a/java/source/infodynamics/measures/continuous/gaussian/MutualInfoCalculatorMultiVariateGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/MutualInfoCalculatorMultiVariateGaussian.java index f2995c4..4e0cb3f 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/MutualInfoCalculatorMultiVariateGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/MutualInfoCalculatorMultiVariateGaussian.java @@ -342,7 +342,8 @@ public class MutualInfoCalculatorMultiVariateGaussian * @throws Exception */ public ChiSquareMeasurementDistribution computeSignificance() { - return new ChiSquareMeasurementDistribution(2*totalObservations*lastAverage, + return new ChiSquareMeasurementDistribution(lastAverage, + totalObservations, dimensionsSource * dimensionsDest); } @@ -517,5 +518,4 @@ public class MutualInfoCalculatorMultiVariateGaussian return localValues; } - } diff --git a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java index 11a6d11..f11db0c 100755 --- a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java +++ b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java @@ -544,7 +544,8 @@ public class ConditionalMutualInformationCalculatorDiscrete if (!condMiComputed) { computeAverageLocalOfObservations(); } - return new ChiSquareMeasurementDistribution(2.0*((double)observations)*average, + return new ChiSquareMeasurementDistribution(average, + observations, (base1 - 1)*(base2 - 1)*condBase); } diff --git a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java index 56d710a..d7a5c19 100755 --- a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java +++ b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java @@ -317,7 +317,8 @@ public class MutualInformationCalculatorDiscrete extends InfoMeasureCalculatorDi if (!miComputed) { computeAverageLocalOfObservations(); } - return new ChiSquareMeasurementDistribution(2.0*((double)observations)*average, + return new ChiSquareMeasurementDistribution(average, + observations, (base - 1) * (base - 1)); } diff --git a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java index fa398f3..0697d96 100755 --- a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java +++ b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java @@ -1287,9 +1287,8 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu if (!estimateComputed) { computeAverageLocalOfObservations(); } - // We set the "actual value" as 2*N*TE, since this marries up properly - // with the requisite chi^2 distribution - return new ChiSquareMeasurementDistribution(2.0*((double)observations)*average, + return new ChiSquareMeasurementDistribution(average, + observations, (sourceHistoryEmbedLength*base - 1)*(base - 1)*(k*base)); } diff --git a/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java b/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java index 822583e..e547349 100755 --- a/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java +++ b/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java @@ -27,16 +27,89 @@ package infodynamics.utils; * @author Joseph Lizier (email, * www) */ -public class AnalyticMeasurementDistribution extends MeasurementDistribution { +public abstract class AnalyticMeasurementDistribution extends MeasurementDistribution { /** * Construct the instance * * @param actualValue observed value of the information-theoretic measure * @param pValue p-value that surrogate measurements are greater than - * the observed value. + * the observed value. (1 - CDF of null at the observed value) */ protected AnalyticMeasurementDistribution(double actualValue, double pValue) { super(actualValue, pValue); } + + /** + * Compute the analytic p-value + * for the given estimate (i.e. the input argument, not the + * estimate produced by any particular observations supplied) + * under a null hypothesis that the source values of our + * samples had no relation to the destination value (possibly + * in the context of a conditional value). + * + *

See Section II.E "Statistical significance testing" of + * the JIDT paper below for a description of how this is done for MI, + * conditional MI and TE as per the references below. + *

+ * + *

References:
+ *

+ * + * @return p-value for the given channel measure score under this null hypothesis. + * @throws Exception + */ + public abstract double computePValueForGivenEstimate(double estimate); + + /** + * Compute the estimated observed measured value corresponding to + * a given p-value + * (derived from how the given estimates are analytically distributed + * under a null hypothesis that the source values of our + * samples had no relation to the destination value, possibly + * in the context of a conditional value). + * + *

See Section II.E "Statistical significance testing" of + * the JIDT paper below for a description of how this is done for MI, + * conditional MI and TE as per the references below. + *

+ * + *

References:
+ *

+ * + * @return the estimate of the channel measure score corresponding to + * the given p-value under this null hypothesis. + * @throws Exception + */ + public abstract double computeEstimateForGivenPValue(double pValue); } diff --git a/java/source/infodynamics/utils/ChiSquareMeasurementDistribution.java b/java/source/infodynamics/utils/ChiSquareMeasurementDistribution.java index 9169c04..18faafd 100755 --- a/java/source/infodynamics/utils/ChiSquareMeasurementDistribution.java +++ b/java/source/infodynamics/utils/ChiSquareMeasurementDistribution.java @@ -18,10 +18,13 @@ package infodynamics.utils; +import infodynamics.utils.commonsmath3.distribution.ChiSquaredDistribution; + /** * Class to represent analytic distributions of info theoretic measurements under - * some null hypothesis of a relationship between the variables, where that - * distribution is a Chi Square distribution. + * some null hypothesis of a relationship between the variables, where the + * distribution of a function of those information-theoretic measurements + * is a Chi Square distribution. * * @author Joseph Lizier (email, * www) @@ -35,13 +38,43 @@ public class ChiSquareMeasurementDistribution extends protected int degreesOfFreedom; /** - * Construct the distribution + * The number of observations that the information theoretic estimate + * is computed from + */ + protected int numObservations; + + /** + * An object of the chi2dist class from commons.math3, + * which we'll use to access the distribution values + */ + protected ChiSquaredDistribution chi2dist; + + /** + * Construct the distribution. + * Note: the Chi squared distribution is technically + * of 2*numObservations*(the info theoretic estimate), not + * of the info theoretic measurement itself. * - * @param actualValue actual observed value + * @param actualValue actual observed information-theoretic value + * @param numObservations the number of observations that the information theoretic estimate + * is computed from * @param degreesOfFreedom degrees of freedom for the distribution */ - public ChiSquareMeasurementDistribution(double actualValue, int degreesOfFreedom) { - super(actualValue, 1 - MathsUtils.chiSquareCdf(actualValue, degreesOfFreedom)); + public ChiSquareMeasurementDistribution(double actualValue, + int numObservations, int degreesOfFreedom) { + super(actualValue, 1 - MathsUtils.chiSquareCdf(2.0*((double)numObservations)*actualValue, degreesOfFreedom)); + this.numObservations = numObservations; this.degreesOfFreedom = degreesOfFreedom; + chi2dist = new ChiSquaredDistribution(degreesOfFreedom); + } + + public double computePValueForGivenEstimate(double estimate) { + return 1 - MathsUtils.chiSquareCdf(2.0*((double)numObservations)*estimate, degreesOfFreedom); + } + + public double computeEstimateForGivenPValue(double pValue) { + return chi2dist.inverseCumulativeProbability(1 - pValue) / (2.0*((double)numObservations)); + // Could also call the following, but this doesn't re-use our objects: + // return MathsUtils.chiSquareInv(1 - pValue, degreesOfFreedom); } } diff --git a/java/source/infodynamics/utils/MathsUtils.java b/java/source/infodynamics/utils/MathsUtils.java index bb40844..7682b64 100755 --- a/java/source/infodynamics/utils/MathsUtils.java +++ b/java/source/infodynamics/utils/MathsUtils.java @@ -18,6 +18,7 @@ package infodynamics.utils; +import infodynamics.utils.commonsmath3.distribution.ChiSquaredDistribution; import infodynamics.utils.commonsmath3.special.Gamma; /** @@ -336,7 +337,7 @@ public class MathsUtils { } /** - *

Return the value of the cummulative distribution function of the + *

Return the value of the cumulative distribution function of the * chi-square distribution, evaluated at x, for k degrees of freedom.

* *

Note that this relies on our approximation of the error function, @@ -360,6 +361,20 @@ public class MathsUtils { return Gamma.regularizedGammaP(((double)k)/2.0, x/2.0); } + /** + * Return the value of the quantile (inverse of the CDF) for the + * given cdf value (between 0 and 1) of the chi-square distribution + * with l degrees of freedom. + * + * @param cdf value of the CDF to evaluate the inverse for + * @param k degrees of freedom (must have k>0) + * @return + */ + public static double chiSquareInv(double cdf, int k) { + ChiSquaredDistribution chi2dist = new ChiSquaredDistribution(k); + return chi2dist.inverseCumulativeProbability(cdf); + } + /** * Return the value of the lower Incomplete Gamma function, * given arguments s/2 and x/2. diff --git a/java/source/infodynamics/utils/commonsmath3/Field.java b/java/source/infodynamics/utils/commonsmath3/Field.java new file mode 100644 index 0000000..49352c7 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/Field.java @@ -0,0 +1,87 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3; + +/** + * Interface representing a field. + *

+ * Classes implementing this interface will often be singletons. + *

+ * @param the type of the field elements + * @see FieldElement + * @since 2.0 + */ +public interface Field { + + /** Get the additive identity of the field. + *

+ * The additive identity is the element e0 of the field such that + * for all elements a of the field, the equalities a + e0 = + * e0 + a = a hold. + *

+ * @return additive identity of the field + */ + T getZero(); + + /** Get the multiplicative identity of the field. + *

+ * The multiplicative identity is the element e1 of the field such that + * for all elements a of the field, the equalities a × e1 = + * e1 × a = a hold. + *

+ * @return multiplicative identity of the field + */ + T getOne(); + + /** + * Returns the runtime class of the FieldElement. + * + * @return The {@code Class} object that represents the runtime + * class of this object. + */ + Class> getRuntimeClass(); + +} diff --git a/java/source/infodynamics/utils/commonsmath3/FieldElement.java b/java/source/infodynamics/utils/commonsmath3/FieldElement.java new file mode 100644 index 0000000..5a03ab0 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/FieldElement.java @@ -0,0 +1,116 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3; + +import infodynamics.utils.commonsmath3.exception.MathArithmeticException; +import infodynamics.utils.commonsmath3.exception.NullArgumentException; + + +/** + * Interface representing field elements. + * @param the type of the field elements + * @see Field + * @since 2.0 + */ +public interface FieldElement { + + /** Compute this + a. + * @param a element to add + * @return a new element representing this + a + * @throws NullArgumentException if {@code a} is {@code null}. + */ + T add(T a) throws NullArgumentException; + + /** Compute this - a. + * @param a element to subtract + * @return a new element representing this - a + * @throws NullArgumentException if {@code a} is {@code null}. + */ + T subtract(T a) throws NullArgumentException; + + /** + * Returns the additive inverse of {@code this} element. + * @return the opposite of {@code this}. + */ + T negate(); + + /** Compute n × this. Multiplication by an integer number is defined + * as the following sum + *
+ * n × this = ∑i=1n this. + *
+ * @param n Number of times {@code this} must be added to itself. + * @return A new element representing n × this. + */ + T multiply(int n); + + /** Compute this × a. + * @param a element to multiply + * @return a new element representing this × a + * @throws NullArgumentException if {@code a} is {@code null}. + */ + T multiply(T a) throws NullArgumentException; + + /** Compute this ÷ a. + * @param a element to divide by + * @return a new element representing this ÷ a + * @throws NullArgumentException if {@code a} is {@code null}. + * @throws MathArithmeticException if {@code a} is zero + */ + T divide(T a) throws NullArgumentException, MathArithmeticException; + + /** + * Returns the multiplicative inverse of {@code this} element. + * @return the inverse of {@code this}. + * @throws MathArithmeticException if {@code this} is zero + */ + T reciprocal() throws MathArithmeticException; + + /** Get the {@link Field} to which the instance belongs. + * @return {@link Field} to which the instance belongs + */ + Field getField(); +} diff --git a/java/source/infodynamics/utils/commonsmath3/RealFieldElement.java b/java/source/infodynamics/utils/commonsmath3/RealFieldElement.java new file mode 100644 index 0000000..86fc4bf --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/RealFieldElement.java @@ -0,0 +1,431 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3; + +import infodynamics.utils.commonsmath3.exception.DimensionMismatchException; + +/** + * Interface representing a real + * field. + * @param the type of the field elements + * @see FieldElement + * @since 3.2 + */ +public interface RealFieldElement extends FieldElement { + + /** Get the real value of the number. + * @return real value + */ + double getReal(); + + /** '+' operator. + * @param a right hand side parameter of the operator + * @return this+a + */ + T add(double a); + + /** '-' operator. + * @param a right hand side parameter of the operator + * @return this-a + */ + T subtract(double a); + + /** '×' operator. + * @param a right hand side parameter of the operator + * @return this×a + */ + T multiply(double a); + + /** '÷' operator. + * @param a right hand side parameter of the operator + * @return this÷a + */ + T divide(double a); + + /** IEEE remainder operator. + * @param a right hand side parameter of the operator + * @return this - n × a where n is the closest integer to this/a + * (the even integer is chosen for n if this/a is halfway between two integers) + */ + T remainder(double a); + + /** IEEE remainder operator. + * @param a right hand side parameter of the operator + * @return this - n × a where n is the closest integer to this/a + * (the even integer is chosen for n if this/a is halfway between two integers) + * @exception DimensionMismatchException if number of free parameters or orders are inconsistent + */ + T remainder(T a) + throws DimensionMismatchException; + + /** absolute value. + * @return abs(this) + */ + T abs(); + + /** Get the smallest whole number larger than instance. + * @return ceil(this) + */ + T ceil(); + + /** Get the largest whole number smaller than instance. + * @return floor(this) + */ + T floor(); + + /** Get the whole number that is the nearest to the instance, or the even one if x is exactly half way between two integers. + * @return a double number r such that r is an integer r - 0.5 ≤ this ≤ r + 0.5 + */ + T rint(); + + /** Get the closest long to instance value. + * @return closest long to {@link #getReal()} + */ + long round(); + + /** Compute the signum of the instance. + * The signum is -1 for negative numbers, +1 for positive numbers and 0 otherwise + * @return -1.0, -0.0, +0.0, +1.0 or NaN depending on sign of a + */ + T signum(); + + /** + * Returns the instance with the sign of the argument. + * A NaN {@code sign} argument is treated as positive. + * + * @param sign the sign for the returned value + * @return the instance with the same sign as the {@code sign} argument + */ + T copySign(T sign); + + /** + * Returns the instance with the sign of the argument. + * A NaN {@code sign} argument is treated as positive. + * + * @param sign the sign for the returned value + * @return the instance with the same sign as the {@code sign} argument + */ + T copySign(double sign); + + /** + * Multiply the instance by a power of 2. + * @param n power of 2 + * @return this × 2n + */ + T scalb(int n); + + /** + * Returns the hypotenuse of a triangle with sides {@code this} and {@code y} + * - sqrt(this2 +y2) + * avoiding intermediate overflow or underflow. + * + *
    + *
  • If either argument is infinite, then the result is positive infinity.
    • + *
  • else, if either argument is NaN then the result is NaN.
    • + *
+ * + * @param y a value + * @return sqrt(this2 +y2) + * @exception DimensionMismatchException if number of free parameters or orders are inconsistent + */ + T hypot(T y) + throws DimensionMismatchException; + + /** {@inheritDoc} */ + T reciprocal(); + + /** Square root. + * @return square root of the instance + */ + T sqrt(); + + /** Cubic root. + * @return cubic root of the instance + */ + T cbrt(); + + /** Nth root. + * @param n order of the root + * @return nth root of the instance + */ + T rootN(int n); + + /** Power operation. + * @param p power to apply + * @return thisp + */ + T pow(double p); + + /** Integer power operation. + * @param n power to apply + * @return thisn + */ + T pow(int n); + + /** Power operation. + * @param e exponent + * @return thise + * @exception DimensionMismatchException if number of free parameters or orders are inconsistent + */ + T pow(T e) + throws DimensionMismatchException; + + /** Exponential. + * @return exponential of the instance + */ + T exp(); + + /** Exponential minus 1. + * @return exponential minus one of the instance + */ + T expm1(); + + /** Natural logarithm. + * @return logarithm of the instance + */ + T log(); + + /** Shifted natural logarithm. + * @return logarithm of one plus the instance + */ + T log1p(); + +// TODO: add this method in 4.0, as it is not possible to do it in 3.2 +// due to incompatibility of the return type in the Dfp class +// /** Base 10 logarithm. +// * @return base 10 logarithm of the instance +// */ +// T log10(); + + /** Cosine operation. + * @return cos(this) + */ + T cos(); + + /** Sine operation. + * @return sin(this) + */ + T sin(); + + /** Tangent operation. + * @return tan(this) + */ + T tan(); + + /** Arc cosine operation. + * @return acos(this) + */ + T acos(); + + /** Arc sine operation. + * @return asin(this) + */ + T asin(); + + /** Arc tangent operation. + * @return atan(this) + */ + T atan(); + + /** Two arguments arc tangent operation. + * @param x second argument of the arc tangent + * @return atan2(this, x) + * @exception DimensionMismatchException if number of free parameters or orders are inconsistent + */ + T atan2(T x) + throws DimensionMismatchException; + + /** Hyperbolic cosine operation. + * @return cosh(this) + */ + T cosh(); + + /** Hyperbolic sine operation. + * @return sinh(this) + */ + T sinh(); + + /** Hyperbolic tangent operation. + * @return tanh(this) + */ + T tanh(); + + /** Inverse hyperbolic cosine operation. + * @return acosh(this) + */ + T acosh(); + + /** Inverse hyperbolic sine operation. + * @return asin(this) + */ + T asinh(); + + /** Inverse hyperbolic tangent operation. + * @return atanh(this) + */ + T atanh(); + + /** + * Compute a linear combination. + * @param a Factors. + * @param b Factors. + * @return Σi ai bi. + * @throws DimensionMismatchException if arrays dimensions don't match + * @since 3.2 + */ + T linearCombination(T[] a, T[] b) + throws DimensionMismatchException; + + /** + * Compute a linear combination. + * @param a Factors. + * @param b Factors. + * @return Σi ai bi. + * @throws DimensionMismatchException if arrays dimensions don't match + * @since 3.2 + */ + T linearCombination(double[] a, T[] b) + throws DimensionMismatchException; + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @return a1×b1 + + * a2×b2 + * @see #linearCombination(Object, Object, Object, Object, Object, Object) + * @see #linearCombination(Object, Object, Object, Object, Object, Object, Object, Object) + * @since 3.2 + */ + T linearCombination(T a1, T b1, T a2, T b2); + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @return a1×b1 + + * a2×b2 + * @see #linearCombination(double, Object, double, Object, double, Object) + * @see #linearCombination(double, Object, double, Object, double, Object, double, Object) + * @since 3.2 + */ + T linearCombination(double a1, T b1, double a2, T b2); + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @param a3 first factor of the third term + * @param b3 second factor of the third term + * @return a1×b1 + + * a2×b2 + a3×b3 + * @see #linearCombination(Object, Object, Object, Object) + * @see #linearCombination(Object, Object, Object, Object, Object, Object, Object, Object) + * @since 3.2 + */ + T linearCombination(T a1, T b1, T a2, T b2, T a3, T b3); + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @param a3 first factor of the third term + * @param b3 second factor of the third term + * @return a1×b1 + + * a2×b2 + a3×b3 + * @see #linearCombination(double, Object, double, Object) + * @see #linearCombination(double, Object, double, Object, double, Object, double, Object) + * @since 3.2 + */ + T linearCombination(double a1, T b1, double a2, T b2, double a3, T b3); + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @param a3 first factor of the third term + * @param b3 second factor of the third term + * @param a4 first factor of the third term + * @param b4 second factor of the third term + * @return a1×b1 + + * a2×b2 + a3×b3 + + * a4×b4 + * @see #linearCombination(Object, Object, Object, Object) + * @see #linearCombination(Object, Object, Object, Object, Object, Object) + * @since 3.2 + */ + T linearCombination(T a1, T b1, T a2, T b2, T a3, T b3, T a4, T b4); + + /** + * Compute a linear combination. + * @param a1 first factor of the first term + * @param b1 second factor of the first term + * @param a2 first factor of the second term + * @param b2 second factor of the second term + * @param a3 first factor of the third term + * @param b3 second factor of the third term + * @param a4 first factor of the third term + * @param b4 second factor of the third term + * @return a1×b1 + + * a2×b2 + a3×b3 + + * a4×b4 + * @see #linearCombination(double, Object, double, Object) + * @see #linearCombination(double, Object, double, Object, double, Object) + * @since 3.2 + */ + T linearCombination(double a1, T b1, double a2, T b2, double a3, T b3, double a4, T b4); + +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/UnivariateFunction.java b/java/source/infodynamics/utils/commonsmath3/analysis/UnivariateFunction.java new file mode 100644 index 0000000..664f4f6 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/UnivariateFunction.java @@ -0,0 +1,110 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.analysis; + +/** + * An interface representing a univariate real function. + *

+ * When a user-defined function encounters an error during + * evaluation, the {@link #value(double) value} method should throw a + * user-defined unchecked exception.

+ *

+ * The following code excerpt shows the recommended way to do that using + * a root solver as an example, but the same construct is applicable to + * ODE integrators or optimizers.

+ * + * 
+ * private static class LocalException extends RuntimeException {
+ *     // The x value that caused the problem.
+ *     private final double x;
+ *
+ *     public LocalException(double x) {
+ *         this.x = x;
+ *     }
+ *
+ *     public double getX() {
+ *         return x;
+ *     }
+ * }
+ *
+ * private static class MyFunction implements UnivariateFunction {
+ *     public double value(double x) {
+ *         double y = hugeFormula(x);
+ *         if (somethingBadHappens) {
+ *           throw new LocalException(x);
+ *         }
+ *         return y;
+ *     }
+ * }
+ *
+ * public void compute() {
+ *     try {
+ *         solver.solve(maxEval, new MyFunction(a, b, c), min, max);
+ *     } catch (LocalException le) {
+ *         // Retrieve the x value.
+ *     }
+ * }
+ *
+ * + * As shown, the exception is local to the user's code and it is guaranteed + * that Apache Commons Math will not catch it. + * + */ +public interface UnivariateFunction { + /** + * Compute the value of the function. + * + * @param x Point at which the function value should be computed. + * @return the value of the function. + * @throws IllegalArgumentException when the activated method itself can + * ascertain that a precondition, specified in the API expressed at the + * level of the activated method, has been violated. + * When Commons Math throws an {@code IllegalArgumentException}, it is + * usually the consequence of checking the actual parameters passed to + * the method. + */ + double value(double x); +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AbstractUnivariateSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AbstractUnivariateSolver.java new file mode 100644 index 0000000..9baa16e --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AbstractUnivariateSolver.java @@ -0,0 +1,89 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; + +/** + * Base class for solvers. + * + * @since 3.0 + */ +public abstract class AbstractUnivariateSolver + extends BaseAbstractUnivariateSolver + implements UnivariateSolver { + /** + * Construct a solver with given absolute accuracy. + * + * @param absoluteAccuracy Maximum absolute error. + */ + protected AbstractUnivariateSolver(final double absoluteAccuracy) { + super(absoluteAccuracy); + } + /** + * Construct a solver with given accuracies. + * + * @param relativeAccuracy Maximum relative error. + * @param absoluteAccuracy Maximum absolute error. + */ + protected AbstractUnivariateSolver(final double relativeAccuracy, + final double absoluteAccuracy) { + super(relativeAccuracy, absoluteAccuracy); + } + /** + * Construct a solver with given accuracies. + * + * @param relativeAccuracy Maximum relative error. + * @param absoluteAccuracy Maximum absolute error. + * @param functionValueAccuracy Maximum function value error. + */ + protected AbstractUnivariateSolver(final double relativeAccuracy, + final double absoluteAccuracy, + final double functionValueAccuracy) { + super(relativeAccuracy, absoluteAccuracy, functionValueAccuracy); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AllowedSolution.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AllowedSolution.java new file mode 100644 index 0000000..2094345 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/AllowedSolution.java @@ -0,0 +1,104 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.analysis.solvers; + + +/** The kinds of solutions that a {@link BracketedUnivariateSolver + * (bracketed univariate real) root-finding algorithm} may accept as solutions. + * This basically controls whether or not under-approximations and + * over-approximations are allowed. + * + *

If all solutions are accepted ({@link #ANY_SIDE}), then the solution + * that the root-finding algorithm returns for a given root may be equal to the + * actual root, but it may also be an approximation that is slightly smaller + * or slightly larger than the actual root. Root-finding algorithms generally + * only guarantee that the returned solution is within the requested + * tolerances. In certain cases however, in particular for + * {@link infodynamics.utils.commonsmath3.ode.events.EventHandler state events} of + * {@link infodynamics.utils.commonsmath3.ode.ODEIntegrator ODE solvers}, it + * may be necessary to guarantee that a solution is returned that lies on a + * specific side the solution.

+ * + * @see BracketedUnivariateSolver + * @since 3.0 + */ +public enum AllowedSolution { + /** There are no additional side restriction on the solutions for + * root-finding. That is, both under-approximations and over-approximations + * are allowed. So, if a function f(x) has a root at x = x0, then the + * root-finding result s may be smaller than x0, equal to x0, or greater + * than x0. + */ + ANY_SIDE, + + /** Only solutions that are less than or equal to the actual root are + * acceptable as solutions for root-finding. In other words, + * over-approximations are not allowed. So, if a function f(x) has a root + * at x = x0, then the root-finding result s must satisfy s <= x0. + */ + LEFT_SIDE, + + /** Only solutions that are greater than or equal to the actual root are + * acceptable as solutions for root-finding. In other words, + * under-approximations are not allowed. So, if a function f(x) has a root + * at x = x0, then the root-finding result s must satisfy s >= x0. + */ + RIGHT_SIDE, + + /** Only solutions for which values are less than or equal to zero are + * acceptable as solutions for root-finding. So, if a function f(x) has + * a root at x = x0, then the root-finding result s must satisfy f(s) <= 0. + */ + BELOW_SIDE, + + /** Only solutions for which values are greater than or equal to zero are + * acceptable as solutions for root-finding. So, if a function f(x) has + * a root at x = x0, then the root-finding result s must satisfy f(s) >= 0. + */ + ABOVE_SIDE; + +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseAbstractUnivariateSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseAbstractUnivariateSolver.java new file mode 100644 index 0000000..eba217d --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseAbstractUnivariateSolver.java @@ -0,0 +1,347 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; +import infodynamics.utils.commonsmath3.exception.MaxCountExceededException; +import infodynamics.utils.commonsmath3.exception.NoBracketingException; +import infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.NullArgumentException; +import infodynamics.utils.commonsmath3.util.IntegerSequence; +import infodynamics.utils.commonsmath3.util.MathUtils; + +/** + * Provide a default implementation for several functions useful to generic + * solvers. + * The default values for relative and function tolerances are 1e-14 + * and 1e-15, respectively. It is however highly recommended to not + * rely on the default, but rather carefully consider values that match + * user's expectations, as well as the specifics of each implementation. + * + * @param Type of function to solve. + * + * @since 2.0 + */ +public abstract class BaseAbstractUnivariateSolver + implements BaseUnivariateSolver { + /** Default relative accuracy. */ + private static final double DEFAULT_RELATIVE_ACCURACY = 1e-14; + /** Default function value accuracy. */ + private static final double DEFAULT_FUNCTION_VALUE_ACCURACY = 1e-15; + /** Function value accuracy. */ + private final double functionValueAccuracy; + /** Absolute accuracy. */ + private final double absoluteAccuracy; + /** Relative accuracy. */ + private final double relativeAccuracy; + /** Evaluations counter. */ + private IntegerSequence.Incrementor evaluations; + /** Lower end of search interval. */ + private double searchMin; + /** Higher end of search interval. */ + private double searchMax; + /** Initial guess. */ + private double searchStart; + /** Function to solve. */ + private FUNC function; + + /** + * Construct a solver with given absolute accuracy. + * + * @param absoluteAccuracy Maximum absolute error. + */ + protected BaseAbstractUnivariateSolver(final double absoluteAccuracy) { + this(DEFAULT_RELATIVE_ACCURACY, + absoluteAccuracy, + DEFAULT_FUNCTION_VALUE_ACCURACY); + } + + /** + * Construct a solver with given accuracies. + * + * @param relativeAccuracy Maximum relative error. + * @param absoluteAccuracy Maximum absolute error. + */ + protected BaseAbstractUnivariateSolver(final double relativeAccuracy, + final double absoluteAccuracy) { + this(relativeAccuracy, + absoluteAccuracy, + DEFAULT_FUNCTION_VALUE_ACCURACY); + } + + /** + * Construct a solver with given accuracies. + * + * @param relativeAccuracy Maximum relative error. + * @param absoluteAccuracy Maximum absolute error. + * @param functionValueAccuracy Maximum function value error. + */ + protected BaseAbstractUnivariateSolver(final double relativeAccuracy, + final double absoluteAccuracy, + final double functionValueAccuracy) { + this.absoluteAccuracy = absoluteAccuracy; + this.relativeAccuracy = relativeAccuracy; + this.functionValueAccuracy = functionValueAccuracy; + this.evaluations = IntegerSequence.Incrementor.create(); + } + + /** {@inheritDoc} */ + public int getMaxEvaluations() { + return evaluations.getMaximalCount(); + } + /** {@inheritDoc} */ + public int getEvaluations() { + return evaluations.getCount(); + } + /** + * @return the lower end of the search interval. + */ + public double getMin() { + return searchMin; + } + /** + * @return the higher end of the search interval. + */ + public double getMax() { + return searchMax; + } + /** + * @return the initial guess. + */ + public double getStartValue() { + return searchStart; + } + /** + * {@inheritDoc} + */ + public double getAbsoluteAccuracy() { + return absoluteAccuracy; + } + /** + * {@inheritDoc} + */ + public double getRelativeAccuracy() { + return relativeAccuracy; + } + /** + * {@inheritDoc} + */ + public double getFunctionValueAccuracy() { + return functionValueAccuracy; + } + + /** + * Compute the objective function value. + * + * @param point Point at which the objective function must be evaluated. + * @return the objective function value at specified point. + * @throws TooManyEvaluationsException if the maximal number of evaluations + * is exceeded. + */ + protected double computeObjectiveValue(double point) + throws TooManyEvaluationsException { + incrementEvaluationCount(); + return function.value(point); + } + + /** + * Prepare for computation. + * Subclasses must call this method if they override any of the + * {@code solve} methods. + * + * @param f Function to solve. + * @param min Lower bound for the interval. + * @param max Upper bound for the interval. + * @param startValue Start value to use. + * @param maxEval Maximum number of evaluations. + * @exception NullArgumentException if f is null + */ + protected void setup(int maxEval, + FUNC f, + double min, double max, + double startValue) + throws NullArgumentException { + // Checks. + MathUtils.checkNotNull(f); + + // Reset. + searchMin = min; + searchMax = max; + searchStart = startValue; + function = f; + evaluations = evaluations.withMaximalCount(maxEval).withStart(0); + } + + /** {@inheritDoc} */ + public double solve(int maxEval, FUNC f, double min, double max, double startValue) + throws TooManyEvaluationsException, + NoBracketingException { + // Initialization. + setup(maxEval, f, min, max, startValue); + + // Perform computation. + return doSolve(); + } + + /** {@inheritDoc} */ + public double solve(int maxEval, FUNC f, double min, double max) { + return solve(maxEval, f, min, max, min + 0.5 * (max - min)); + } + + /** {@inheritDoc} */ + public double solve(int maxEval, FUNC f, double startValue) + throws TooManyEvaluationsException, + NoBracketingException { + return solve(maxEval, f, Double.NaN, Double.NaN, startValue); + } + + /** + * Method for implementing actual optimization algorithms in derived + * classes. + * + * @return the root. + * @throws TooManyEvaluationsException if the maximal number of evaluations + * is exceeded. + * @throws NoBracketingException if the initial search interval does not bracket + * a root and the solver requires it. + */ + protected abstract double doSolve() + throws TooManyEvaluationsException, NoBracketingException; + + /** + * Check whether the function takes opposite signs at the endpoints. + * + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @return {@code true} if the function values have opposite signs at the + * given points. + */ + protected boolean isBracketing(final double lower, + final double upper) { + return UnivariateSolverUtils.isBracketing(function, lower, upper); + } + + /** + * Check whether the arguments form a (strictly) increasing sequence. + * + * @param start First number. + * @param mid Second number. + * @param end Third number. + * @return {@code true} if the arguments form an increasing sequence. + */ + protected boolean isSequence(final double start, + final double mid, + final double end) { + return UnivariateSolverUtils.isSequence(start, mid, end); + } + + /** + * Check that the endpoints specify an interval. + * + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + protected void verifyInterval(final double lower, + final double upper) + throws NumberIsTooLargeException { + UnivariateSolverUtils.verifyInterval(lower, upper); + } + + /** + * Check that {@code lower < initial < upper}. + * + * @param lower Lower endpoint. + * @param initial Initial value. + * @param upper Upper endpoint. + * @throws NumberIsTooLargeException if {@code lower >= initial} or + * {@code initial >= upper}. + */ + protected void verifySequence(final double lower, + final double initial, + final double upper) + throws NumberIsTooLargeException { + UnivariateSolverUtils.verifySequence(lower, initial, upper); + } + + /** + * Check that the endpoints specify an interval and the function takes + * opposite signs at the endpoints. + * + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @throws NullArgumentException if the function has not been set. + * @throws NoBracketingException if the function has the same sign at + * the endpoints. + */ + protected void verifyBracketing(final double lower, + final double upper) + throws NullArgumentException, + NoBracketingException { + UnivariateSolverUtils.verifyBracketing(function, lower, upper); + } + + /** + * Increment the evaluation count by one. + * Method {@link #computeObjectiveValue(double)} calls this method internally. + * It is provided for subclasses that do not exclusively use + * {@code computeObjectiveValue} to solve the function. + * See e.g. {@link AbstractUnivariateDifferentiableSolver}. + * + * @throws TooManyEvaluationsException when the allowed number of function + * evaluations has been exhausted. + */ + protected void incrementEvaluationCount() + throws TooManyEvaluationsException { + try { + evaluations.increment(); + } catch (MaxCountExceededException e) { + throw new TooManyEvaluationsException(e.getMax()); + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseUnivariateSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseUnivariateSolver.java new file mode 100644 index 0000000..fb70cb9 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BaseUnivariateSolver.java @@ -0,0 +1,171 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; +import infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException; +import infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException; + + +/** + * Interface for (univariate real) rootfinding algorithms. + * Implementations will search for only one zero in the given interval. + * + * This class is not intended for use outside of the Apache Commons Math + * library, regular user should rely on more specific interfaces like + * {@link UnivariateSolver}, {@link PolynomialSolver} or {@link + * DifferentiableUnivariateSolver}. + * @param Type of function to solve. + * + * @since 3.0 + * @see UnivariateSolver + * @see PolynomialSolver + * @see DifferentiableUnivariateSolver + */ +public interface BaseUnivariateSolver { + /** + * Get the maximum number of function evaluations. + * + * @return the maximum number of function evaluations. + */ + int getMaxEvaluations(); + + /** + * Get the number of evaluations of the objective function. + * The number of evaluations corresponds to the last call to the + * {@code optimize} method. It is 0 if the method has not been + * called yet. + * + * @return the number of evaluations of the objective function. + */ + int getEvaluations(); + + /** + * Get the absolute accuracy of the solver. Solutions returned by the + * solver should be accurate to this tolerance, i.e., if ε is the + * absolute accuracy of the solver and {@code v} is a value returned by + * one of the {@code solve} methods, then a root of the function should + * exist somewhere in the interval ({@code v} - ε, {@code v} + ε). + * + * @return the absolute accuracy. + */ + double getAbsoluteAccuracy(); + + /** + * Get the relative accuracy of the solver. The contract for relative + * accuracy is the same as {@link #getAbsoluteAccuracy()}, but using + * relative, rather than absolute error. If ρ is the relative accuracy + * configured for a solver and {@code v} is a value returned, then a root + * of the function should exist somewhere in the interval + * ({@code v} - ρ {@code v}, {@code v} + ρ {@code v}). + * + * @return the relative accuracy. + */ + double getRelativeAccuracy(); + + /** + * Get the function value accuracy of the solver. If {@code v} is + * a value returned by the solver for a function {@code f}, + * then by contract, {@code |f(v)|} should be less than or equal to + * the function value accuracy configured for the solver. + * + * @return the function value accuracy. + */ + double getFunctionValueAccuracy(); + + /** + * Solve for a zero root in the given interval. + * A solver may require that the interval brackets a single zero root. + * Solvers that do require bracketing should be able to handle the case + * where one of the endpoints is itself a root. + * + * @param maxEval Maximum number of evaluations. + * @param f Function to solve. + * @param min Lower bound for the interval. + * @param max Upper bound for the interval. + * @return a value where the function is zero. + * @throws MathIllegalArgumentException + * if the arguments do not satisfy the requirements specified by the solver. + * @throws TooManyEvaluationsException if + * the allowed number of evaluations is exceeded. + */ + double solve(int maxEval, FUNC f, double min, double max) + throws MathIllegalArgumentException, TooManyEvaluationsException; + + /** + * Solve for a zero in the given interval, start at {@code startValue}. + * A solver may require that the interval brackets a single zero root. + * Solvers that do require bracketing should be able to handle the case + * where one of the endpoints is itself a root. + * + * @param maxEval Maximum number of evaluations. + * @param f Function to solve. + * @param min Lower bound for the interval. + * @param max Upper bound for the interval. + * @param startValue Start value to use. + * @return a value where the function is zero. + * @throws MathIllegalArgumentException + * if the arguments do not satisfy the requirements specified by the solver. + * @throws TooManyEvaluationsException if + * the allowed number of evaluations is exceeded. + */ + double solve(int maxEval, FUNC f, double min, double max, double startValue) + throws MathIllegalArgumentException, TooManyEvaluationsException; + + /** + * Solve for a zero in the vicinity of {@code startValue}. + * + * @param f Function to solve. + * @param startValue Start value to use. + * @return a value where the function is zero. + * @param maxEval Maximum number of evaluations. + * @throws infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException + * if the arguments do not satisfy the requirements specified by the solver. + * @throws infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException if + * the allowed number of evaluations is exceeded. + */ + double solve(int maxEval, FUNC f, double startValue); +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BracketedUnivariateSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BracketedUnivariateSolver.java new file mode 100644 index 0000000..998a1cb --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BracketedUnivariateSolver.java @@ -0,0 +1,121 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; + +/** Interface for {@link UnivariateSolver (univariate real) root-finding + * algorithms} that maintain a bracketed solution. There are several advantages + * to having such root-finding algorithms: + *
    + *
  • The bracketed solution guarantees that the root is kept within the + * interval. As such, these algorithms generally also guarantee + * convergence.
    • + *
  • The bracketed solution means that we have the opportunity to only + * return roots that are greater than or equal to the actual root, or + * are less than or equal to the actual root. That is, we can control + * whether under-approximations and over-approximations are + * {@link AllowedSolution allowed solutions}. Other root-finding + * algorithms can usually only guarantee that the solution (the root that + * was found) is around the actual root.
    • + *
+ * + *

For backwards compatibility, all root-finding algorithms must have + * {@link AllowedSolution#ANY_SIDE ANY_SIDE} as default for the allowed + * solutions.

+ * @param Type of function to solve. + * + * @see AllowedSolution + * @since 3.0 + */ +public interface BracketedUnivariateSolver + extends BaseUnivariateSolver { + + /** + * Solve for a zero in the given interval. + * A solver may require that the interval brackets a single zero root. + * Solvers that do require bracketing should be able to handle the case + * where one of the endpoints is itself a root. + * + * @param maxEval Maximum number of evaluations. + * @param f Function to solve. + * @param min Lower bound for the interval. + * @param max Upper bound for the interval. + * @param allowedSolution The kind of solutions that the root-finding algorithm may + * accept as solutions. + * @return A value where the function is zero. + * @throws infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException + * if the arguments do not satisfy the requirements specified by the solver. + * @throws infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException if + * the allowed number of evaluations is exceeded. + */ + double solve(int maxEval, FUNC f, double min, double max, + AllowedSolution allowedSolution); + + /** + * Solve for a zero in the given interval, start at {@code startValue}. + * A solver may require that the interval brackets a single zero root. + * Solvers that do require bracketing should be able to handle the case + * where one of the endpoints is itself a root. + * + * @param maxEval Maximum number of evaluations. + * @param f Function to solve. + * @param min Lower bound for the interval. + * @param max Upper bound for the interval. + * @param startValue Start value to use. + * @param allowedSolution The kind of solutions that the root-finding algorithm may + * accept as solutions. + * @return A value where the function is zero. + * @throws infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException + * if the arguments do not satisfy the requirements specified by the solver. + * @throws infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException if + * the allowed number of evaluations is exceeded. + */ + double solve(int maxEval, FUNC f, double min, double max, double startValue, + AllowedSolution allowedSolution); + +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BrentSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BrentSolver.java new file mode 100644 index 0000000..e6ca684 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/BrentSolver.java @@ -0,0 +1,272 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.analysis.solvers; + + +import infodynamics.utils.commonsmath3.exception.NoBracketingException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.TooManyEvaluationsException; +import infodynamics.utils.commonsmath3.util.FastMath; +import infodynamics.utils.commonsmath3.util.Precision; + +/** + * This class implements the + * Brent algorithm for finding zeros of real univariate functions. + * The function should be continuous but not necessarily smooth. + * The {@code solve} method returns a zero {@code x} of the function {@code f} + * in the given interval {@code [a, b]} to within a tolerance + * {@code 2 eps abs(x) + t} where {@code eps} is the relative accuracy and + * {@code t} is the absolute accuracy. + *

The given interval must bracket the root.

+ *

+ * The reference implementation is given in chapter 4 of + *

+ * Algorithms for Minimization Without Derivatives, + * Richard P. Brent, + * Dover, 2002 + *
+ * + * @see BaseAbstractUnivariateSolver + */ +public class BrentSolver extends AbstractUnivariateSolver { + + /** Default absolute accuracy. */ + private static final double DEFAULT_ABSOLUTE_ACCURACY = 1e-6; + + /** + * Construct a solver with default absolute accuracy (1e-6). + */ + public BrentSolver() { + this(DEFAULT_ABSOLUTE_ACCURACY); + } + /** + * Construct a solver. + * + * @param absoluteAccuracy Absolute accuracy. + */ + public BrentSolver(double absoluteAccuracy) { + super(absoluteAccuracy); + } + /** + * Construct a solver. + * + * @param relativeAccuracy Relative accuracy. + * @param absoluteAccuracy Absolute accuracy. + */ + public BrentSolver(double relativeAccuracy, + double absoluteAccuracy) { + super(relativeAccuracy, absoluteAccuracy); + } + /** + * Construct a solver. + * + * @param relativeAccuracy Relative accuracy. + * @param absoluteAccuracy Absolute accuracy. + * @param functionValueAccuracy Function value accuracy. + * + * @see BaseAbstractUnivariateSolver#BaseAbstractUnivariateSolver(double,double,double) + */ + public BrentSolver(double relativeAccuracy, + double absoluteAccuracy, + double functionValueAccuracy) { + super(relativeAccuracy, absoluteAccuracy, functionValueAccuracy); + } + + /** + * {@inheritDoc} + */ + @Override + protected double doSolve() + throws NoBracketingException, + TooManyEvaluationsException, + NumberIsTooLargeException { + double min = getMin(); + double max = getMax(); + final double initial = getStartValue(); + final double functionValueAccuracy = getFunctionValueAccuracy(); + + verifySequence(min, initial, max); + + // Return the initial guess if it is good enough. + double yInitial = computeObjectiveValue(initial); + if (FastMath.abs(yInitial) <= functionValueAccuracy) { + return initial; + } + + // Return the first endpoint if it is good enough. + double yMin = computeObjectiveValue(min); + if (FastMath.abs(yMin) <= functionValueAccuracy) { + return min; + } + + // Reduce interval if min and initial bracket the root. + if (yInitial * yMin < 0) { + return brent(min, initial, yMin, yInitial); + } + + // Return the second endpoint if it is good enough. + double yMax = computeObjectiveValue(max); + if (FastMath.abs(yMax) <= functionValueAccuracy) { + return max; + } + + // Reduce interval if initial and max bracket the root. + if (yInitial * yMax < 0) { + return brent(initial, max, yInitial, yMax); + } + + throw new NoBracketingException(min, max, yMin, yMax); + } + + /** + * Search for a zero inside the provided interval. + * This implementation is based on the algorithm described at page 58 of + * the book + *
+ * Algorithms for Minimization Without Derivatives, + * Richard P. Brent, + * Dover 0-486-41998-3 + *
+ * + * @param lo Lower bound of the search interval. + * @param hi Higher bound of the search interval. + * @param fLo Function value at the lower bound of the search interval. + * @param fHi Function value at the higher bound of the search interval. + * @return the value where the function is zero. + */ + private double brent(double lo, double hi, + double fLo, double fHi) { + double a = lo; + double fa = fLo; + double b = hi; + double fb = fHi; + double c = a; + double fc = fa; + double d = b - a; + double e = d; + + final double t = getAbsoluteAccuracy(); + final double eps = getRelativeAccuracy(); + + while (true) { + if (FastMath.abs(fc) < FastMath.abs(fb)) { + a = b; + b = c; + c = a; + fa = fb; + fb = fc; + fc = fa; + } + + final double tol = 2 * eps * FastMath.abs(b) + t; + final double m = 0.5 * (c - b); + + if (FastMath.abs(m) <= tol || + Precision.equals(fb, 0)) { + return b; + } + if (FastMath.abs(e) < tol || + FastMath.abs(fa) <= FastMath.abs(fb)) { + // Force bisection. + d = m; + e = d; + } else { + double s = fb / fa; + double p; + double q; + // The equality test (a == c) is intentional, + // it is part of the original Brent's method and + // it should NOT be replaced by proximity test. + if (a == c) { + // Linear interpolation. + p = 2 * m * s; + q = 1 - s; + } else { + // Inverse quadratic interpolation. + q = fa / fc; + final double r = fb / fc; + p = s * (2 * m * q * (q - r) - (b - a) * (r - 1)); + q = (q - 1) * (r - 1) * (s - 1); + } + if (p > 0) { + q = -q; + } else { + p = -p; + } + s = e; + e = d; + if (p >= 1.5 * m * q - FastMath.abs(tol * q) || + p >= FastMath.abs(0.5 * s * q)) { + // Inverse quadratic interpolation gives a value + // in the wrong direction, or progress is slow. + // Fall back to bisection. + d = m; + e = d; + } else { + d = p / q; + } + } + a = b; + fa = fb; + + if (FastMath.abs(d) > tol) { + b += d; + } else if (m > 0) { + b += tol; + } else { + b -= tol; + } + fb = computeObjectiveValue(b); + if ((fb > 0 && fc > 0) || + (fb <= 0 && fc <= 0)) { + c = a; + fc = fa; + d = b - a; + e = d; + } + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolver.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolver.java new file mode 100644 index 0000000..7cd6023 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolver.java @@ -0,0 +1,57 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; + + +/** + * Interface for (univariate real) root-finding algorithms. + * Implementations will search for only one zero in the given interval. + * + */ +public interface UnivariateSolver + extends BaseUnivariateSolver {} diff --git a/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolverUtils.java b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolverUtils.java new file mode 100644 index 0000000..44e5475 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/analysis/solvers/UnivariateSolverUtils.java @@ -0,0 +1,496 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.analysis.solvers; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; +import infodynamics.utils.commonsmath3.exception.NoBracketingException; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NullArgumentException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Utility routines for {@link UnivariateSolver} objects. + * + */ +public class UnivariateSolverUtils { + /** + * Class contains only static methods. + */ + private UnivariateSolverUtils() {} + + /** + * Convenience method to find a zero of a univariate real function. A default + * solver is used. + * + * @param function Function. + * @param x0 Lower bound for the interval. + * @param x1 Upper bound for the interval. + * @return a value where the function is zero. + * @throws NoBracketingException if the function has the same sign at the + * endpoints. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static double solve(UnivariateFunction function, double x0, double x1) + throws NullArgumentException, + NoBracketingException { + if (function == null) { + throw new NullArgumentException(LocalizedFormats.FUNCTION); + } + final UnivariateSolver solver = new BrentSolver(); + return solver.solve(Integer.MAX_VALUE, function, x0, x1); + } + + /** + * Convenience method to find a zero of a univariate real function. A default + * solver is used. + * + * @param function Function. + * @param x0 Lower bound for the interval. + * @param x1 Upper bound for the interval. + * @param absoluteAccuracy Accuracy to be used by the solver. + * @return a value where the function is zero. + * @throws NoBracketingException if the function has the same sign at the + * endpoints. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static double solve(UnivariateFunction function, + double x0, double x1, + double absoluteAccuracy) + throws NullArgumentException, + NoBracketingException { + if (function == null) { + throw new NullArgumentException(LocalizedFormats.FUNCTION); + } + final UnivariateSolver solver = new BrentSolver(absoluteAccuracy); + return solver.solve(Integer.MAX_VALUE, function, x0, x1); + } + + /** + * Force a root found by a non-bracketing solver to lie on a specified side, + * as if the solver were a bracketing one. + * + * @param maxEval maximal number of new evaluations of the function + * (evaluations already done for finding the root should have already been subtracted + * from this number) + * @param f function to solve + * @param bracketing bracketing solver to use for shifting the root + * @param baseRoot original root found by a previous non-bracketing solver + * @param min minimal bound of the search interval + * @param max maximal bound of the search interval + * @param allowedSolution the kind of solutions that the root-finding algorithm may + * accept as solutions. + * @return a root approximation, on the specified side of the exact root + * @throws NoBracketingException if the function has the same sign at the + * endpoints. + */ + public static double forceSide(final int maxEval, final UnivariateFunction f, + final BracketedUnivariateSolver bracketing, + final double baseRoot, final double min, final double max, + final AllowedSolution allowedSolution) + throws NoBracketingException { + + if (allowedSolution == AllowedSolution.ANY_SIDE) { + // no further bracketing required + return baseRoot; + } + + // find a very small interval bracketing the root + final double step = FastMath.max(bracketing.getAbsoluteAccuracy(), + FastMath.abs(baseRoot * bracketing.getRelativeAccuracy())); + double xLo = FastMath.max(min, baseRoot - step); + double fLo = f.value(xLo); + double xHi = FastMath.min(max, baseRoot + step); + double fHi = f.value(xHi); + int remainingEval = maxEval - 2; + while (remainingEval > 0) { + + if ((fLo >= 0 && fHi <= 0) || (fLo <= 0 && fHi >= 0)) { + // compute the root on the selected side + return bracketing.solve(remainingEval, f, xLo, xHi, baseRoot, allowedSolution); + } + + // try increasing the interval + boolean changeLo = false; + boolean changeHi = false; + if (fLo < fHi) { + // increasing function + if (fLo >= 0) { + changeLo = true; + } else { + changeHi = true; + } + } else if (fLo > fHi) { + // decreasing function + if (fLo <= 0) { + changeLo = true; + } else { + changeHi = true; + } + } else { + // unknown variation + changeLo = true; + changeHi = true; + } + + // update the lower bound + if (changeLo) { + xLo = FastMath.max(min, xLo - step); + fLo = f.value(xLo); + remainingEval--; + } + + // update the higher bound + if (changeHi) { + xHi = FastMath.min(max, xHi + step); + fHi = f.value(xHi); + remainingEval--; + } + + } + + throw new NoBracketingException(LocalizedFormats.FAILED_BRACKETING, + xLo, xHi, fLo, fHi, + maxEval - remainingEval, maxEval, baseRoot, + min, max); + + } + + /** + * This method simply calls {@link #bracket(UnivariateFunction, double, double, double, + * double, double, int) bracket(function, initial, lowerBound, upperBound, q, r, maximumIterations)} + * with {@code q} and {@code r} set to 1.0 and {@code maximumIterations} set to {@code Integer.MAX_VALUE}. + *

+ * Note: this method can take {@code Integer.MAX_VALUE} + * iterations to throw a {@code ConvergenceException.} Unless you are + * confident that there is a root between {@code lowerBound} and + * {@code upperBound} near {@code initial}, it is better to use + * {@link #bracket(UnivariateFunction, double, double, double, double,double, int) + * bracket(function, initial, lowerBound, upperBound, q, r, maximumIterations)}, + * explicitly specifying the maximum number of iterations.

+ * + * @param function Function. + * @param initial Initial midpoint of interval being expanded to + * bracket a root. + * @param lowerBound Lower bound (a is never lower than this value) + * @param upperBound Upper bound (b never is greater than this + * value). + * @return a two-element array holding a and b. + * @throws NoBracketingException if a root cannot be bracketted. + * @throws NotStrictlyPositiveException if {@code maximumIterations <= 0}. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static double[] bracket(UnivariateFunction function, + double initial, + double lowerBound, double upperBound) + throws NullArgumentException, + NotStrictlyPositiveException, + NoBracketingException { + return bracket(function, initial, lowerBound, upperBound, 1.0, 1.0, Integer.MAX_VALUE); + } + + /** + * This method simply calls {@link #bracket(UnivariateFunction, double, double, double, + * double, double, int) bracket(function, initial, lowerBound, upperBound, q, r, maximumIterations)} + * with {@code q} and {@code r} set to 1.0. + * @param function Function. + * @param initial Initial midpoint of interval being expanded to + * bracket a root. + * @param lowerBound Lower bound (a is never lower than this value). + * @param upperBound Upper bound (b never is greater than this + * value). + * @param maximumIterations Maximum number of iterations to perform + * @return a two element array holding a and b. + * @throws NoBracketingException if the algorithm fails to find a and b + * satisfying the desired conditions. + * @throws NotStrictlyPositiveException if {@code maximumIterations <= 0}. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static double[] bracket(UnivariateFunction function, + double initial, + double lowerBound, double upperBound, + int maximumIterations) + throws NullArgumentException, + NotStrictlyPositiveException, + NoBracketingException { + return bracket(function, initial, lowerBound, upperBound, 1.0, 1.0, maximumIterations); + } + + /** + * This method attempts to find two values a and b satisfying
    + *
  • {@code lowerBound <= a < initial < b <= upperBound}
    • + *
  • {@code f(a) * f(b) <= 0}
    • + *
+ * If {@code f} is continuous on {@code [a,b]}, this means that {@code a} + * and {@code b} bracket a root of {@code f}. + *

+ * The algorithm checks the sign of \( f(l_k) \) and \( f(u_k) \) for increasing + * values of k, where \( l_k = max(lower, initial - \delta_k) \), + * \( u_k = min(upper, initial + \delta_k) \), using recurrence + * \( \delta_{k+1} = r \delta_k + q, \delta_0 = 0\) and starting search with \( k=1 \). + * The algorithm stops when one of the following happens:

    + *
  • at least one positive and one negative value have been found -- success!
    • + *
  • both endpoints have reached their respective limits -- NoBracketingException
    • + *
  • {@code maximumIterations} iterations elapse -- NoBracketingException
+ *

+ * If different signs are found at first iteration ({@code k=1}), then the returned + * interval will be \( [a, b] = [l_1, u_1] \). If different signs are found at a later + * iteration {@code k>1}, then the returned interval will be either + * \( [a, b] = [l_{k+1}, l_{k}] \) or \( [a, b] = [u_{k}, u_{k+1}] \). A root solver called + * with these parameters will therefore start with the smallest bracketing interval known + * at this step. + *

+ *

+ * Interval expansion rate is tuned by changing the recurrence parameters {@code r} and + * {@code q}. When the multiplicative factor {@code r} is set to 1, the sequence is a + * simple arithmetic sequence with linear increase. When the multiplicative factor {@code r} + * is larger than 1, the sequence has an asymptotically exponential rate. Note than the + * additive parameter {@code q} should never be set to zero, otherwise the interval would + * degenerate to the single initial point for all values of {@code k}. + *

+ *

+ * As a rule of thumb, when the location of the root is expected to be approximately known + * within some error margin, {@code r} should be set to 1 and {@code q} should be set to the + * order of magnitude of the error margin. When the location of the root is really a wild guess, + * then {@code r} should be set to a value larger than 1 (typically 2 to double the interval + * length at each iteration) and {@code q} should be set according to half the initial + * search interval length. + *

+ *

+ * As an example, if we consider the trivial function {@code f(x) = 1 - x} and use + * {@code initial = 4}, {@code r = 1}, {@code q = 2}, the algorithm will compute + * {@code f(4-2) = f(2) = -1} and {@code f(4+2) = f(6) = -5} for {@code k = 1}, then + * {@code f(4-4) = f(0) = +1} and {@code f(4+4) = f(8) = -7} for {@code k = 2}. Then it will + * return the interval {@code [0, 2]} as the smallest one known to be bracketing the root. + * As shown by this example, the initial value (here {@code 4}) may lie outside of the returned + * bracketing interval. + *

+ * @param function function to check + * @param initial Initial midpoint of interval being expanded to + * bracket a root. + * @param lowerBound Lower bound (a is never lower than this value). + * @param upperBound Upper bound (b never is greater than this + * value). + * @param q additive offset used to compute bounds sequence (must be strictly positive) + * @param r multiplicative factor used to compute bounds sequence + * @param maximumIterations Maximum number of iterations to perform + * @return a two element array holding the bracketing values. + * @exception NoBracketingException if function cannot be bracketed in the search interval + */ + public static double[] bracket(final UnivariateFunction function, final double initial, + final double lowerBound, final double upperBound, + final double q, final double r, final int maximumIterations) + throws NoBracketingException { + + if (function == null) { + throw new NullArgumentException(LocalizedFormats.FUNCTION); + } + if (q <= 0) { + throw new NotStrictlyPositiveException(q); + } + if (maximumIterations <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.INVALID_MAX_ITERATIONS, maximumIterations); + } + verifySequence(lowerBound, initial, upperBound); + + // initialize the recurrence + double a = initial; + double b = initial; + double fa = Double.NaN; + double fb = Double.NaN; + double delta = 0; + + for (int numIterations = 0; + (numIterations < maximumIterations) && (a > lowerBound || b < upperBound); + ++numIterations) { + + final double previousA = a; + final double previousFa = fa; + final double previousB = b; + final double previousFb = fb; + + delta = r * delta + q; + a = FastMath.max(initial - delta, lowerBound); + b = FastMath.min(initial + delta, upperBound); + fa = function.value(a); + fb = function.value(b); + + if (numIterations == 0) { + // at first iteration, we don't have a previous interval + // we simply compare both sides of the initial interval + if (fa * fb <= 0) { + // the first interval already brackets a root + return new double[] { a, b }; + } + } else { + // we have a previous interval with constant sign and expand it, + // we expect sign changes to occur at boundaries + if (fa * previousFa <= 0) { + // sign change detected at near lower bound + return new double[] { a, previousA }; + } else if (fb * previousFb <= 0) { + // sign change detected at near upper bound + return new double[] { previousB, b }; + } + } + + } + + // no bracketing found + throw new NoBracketingException(a, b, fa, fb); + + } + + /** + * Compute the midpoint of two values. + * + * @param a first value. + * @param b second value. + * @return the midpoint. + */ + public static double midpoint(double a, double b) { + return (a + b) * 0.5; + } + + /** + * Check whether the interval bounds bracket a root. That is, if the + * values at the endpoints are not equal to zero, then the function takes + * opposite signs at the endpoints. + * + * @param function Function. + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @return {@code true} if the function values have opposite signs at the + * given points. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static boolean isBracketing(UnivariateFunction function, + final double lower, + final double upper) + throws NullArgumentException { + if (function == null) { + throw new NullArgumentException(LocalizedFormats.FUNCTION); + } + final double fLo = function.value(lower); + final double fHi = function.value(upper); + return (fLo >= 0 && fHi <= 0) || (fLo <= 0 && fHi >= 0); + } + + /** + * Check whether the arguments form a (strictly) increasing sequence. + * + * @param start First number. + * @param mid Second number. + * @param end Third number. + * @return {@code true} if the arguments form an increasing sequence. + */ + public static boolean isSequence(final double start, + final double mid, + final double end) { + return (start < mid) && (mid < end); + } + + /** + * Check that the endpoints specify an interval. + * + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + public static void verifyInterval(final double lower, + final double upper) + throws NumberIsTooLargeException { + if (lower >= upper) { + throw new NumberIsTooLargeException(LocalizedFormats.ENDPOINTS_NOT_AN_INTERVAL, + lower, upper, false); + } + } + + /** + * Check that {@code lower < initial < upper}. + * + * @param lower Lower endpoint. + * @param initial Initial value. + * @param upper Upper endpoint. + * @throws NumberIsTooLargeException if {@code lower >= initial} or + * {@code initial >= upper}. + */ + public static void verifySequence(final double lower, + final double initial, + final double upper) + throws NumberIsTooLargeException { + verifyInterval(lower, initial); + verifyInterval(initial, upper); + } + + /** + * Check that the endpoints specify an interval and the end points + * bracket a root. + * + * @param function Function. + * @param lower Lower endpoint. + * @param upper Upper endpoint. + * @throws NoBracketingException if the function has the same sign at the + * endpoints. + * @throws NullArgumentException if {@code function} is {@code null}. + */ + public static void verifyBracketing(UnivariateFunction function, + final double lower, + final double upper) + throws NullArgumentException, + NoBracketingException { + if (function == null) { + throw new NullArgumentException(LocalizedFormats.FUNCTION); + } + verifyInterval(lower, upper); + if (!isBracketing(function, lower, upper)) { + throw new NoBracketingException(lower, upper, + function.value(lower), + function.value(upper)); + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/AbstractIntegerDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/AbstractIntegerDistribution.java new file mode 100644 index 0000000..0192acf --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/AbstractIntegerDistribution.java @@ -0,0 +1,282 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import java.io.Serializable; + +import infodynamics.utils.commonsmath3.exception.MathInternalError; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Base class for integer-valued discrete distributions. Default + * implementations are provided for some of the methods that do not vary + * from distribution to distribution. + * + */ +public abstract class AbstractIntegerDistribution implements IntegerDistribution, Serializable { + + /** Serializable version identifier */ + private static final long serialVersionUID = -1146319659338487221L; + + /** + * RandomData instance used to generate samples from the distribution. + * @deprecated As of 3.1, to be removed in 4.0. Please use the + * {@link #random} instance variable instead. + */ + @Deprecated + protected final infodynamics.utils.commonsmath3.random.RandomDataImpl randomData = + new infodynamics.utils.commonsmath3.random.RandomDataImpl(); + + /** + * RNG instance used to generate samples from the distribution. + * @since 3.1 + */ + protected final RandomGenerator random; + + /** + * @deprecated As of 3.1, to be removed in 4.0. Please use + * {@link #AbstractIntegerDistribution(RandomGenerator)} instead. + */ + @Deprecated + protected AbstractIntegerDistribution() { + // Legacy users are only allowed to access the deprecated "randomData". + // New users are forbidden to use this constructor. + random = null; + } + + /** + * @param rng Random number generator. + * @since 3.1 + */ + protected AbstractIntegerDistribution(RandomGenerator rng) { + random = rng; + } + + /** + * {@inheritDoc} + * + * The default implementation uses the identity + *

{@code P(x0 < X <= x1) = P(X <= x1) - P(X <= x0)}

+ */ + public double cumulativeProbability(int x0, int x1) throws NumberIsTooLargeException { + if (x1 < x0) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_ENDPOINT_ABOVE_UPPER_ENDPOINT, + x0, x1, true); + } + return cumulativeProbability(x1) - cumulativeProbability(x0); + } + + /** + * {@inheritDoc} + * + * The default implementation returns + *
    + *
  • {@link #getSupportLowerBound()} for {@code p = 0},
    • + *
  • {@link #getSupportUpperBound()} for {@code p = 1}, and
    • + *
  • {@link #solveInverseCumulativeProbability(double, int, int)} for + * {@code 0 < p < 1}.
    • + *
+ */ + public int inverseCumulativeProbability(final double p) throws OutOfRangeException { + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0, 1); + } + + int lower = getSupportLowerBound(); + if (p == 0.0) { + return lower; + } + if (lower == Integer.MIN_VALUE) { + if (checkedCumulativeProbability(lower) >= p) { + return lower; + } + } else { + lower -= 1; // this ensures cumulativeProbability(lower) < p, which + // is important for the solving step + } + + int upper = getSupportUpperBound(); + if (p == 1.0) { + return upper; + } + + // use the one-sided Chebyshev inequality to narrow the bracket + // cf. AbstractRealDistribution.inverseCumulativeProbability(double) + final double mu = getNumericalMean(); + final double sigma = FastMath.sqrt(getNumericalVariance()); + final boolean chebyshevApplies = !(Double.isInfinite(mu) || Double.isNaN(mu) || + Double.isInfinite(sigma) || Double.isNaN(sigma) || sigma == 0.0); + if (chebyshevApplies) { + double k = FastMath.sqrt((1.0 - p) / p); + double tmp = mu - k * sigma; + if (tmp > lower) { + lower = ((int) FastMath.ceil(tmp)) - 1; + } + k = 1.0 / k; + tmp = mu + k * sigma; + if (tmp < upper) { + upper = ((int) FastMath.ceil(tmp)) - 1; + } + } + + return solveInverseCumulativeProbability(p, lower, upper); + } + + /** + * This is a utility function used by {@link + * #inverseCumulativeProbability(double)}. It assumes {@code 0 < p < 1} and + * that the inverse cumulative probability lies in the bracket {@code + * (lower, upper]}. The implementation does simple bisection to find the + * smallest {@code p}-quantile inf{x in Z | P(X<=x) >= p}. + * + * @param p the cumulative probability + * @param lower a value satisfying {@code cumulativeProbability(lower) < p} + * @param upper a value satisfying {@code p <= cumulativeProbability(upper)} + * @return the smallest {@code p}-quantile of this distribution + */ + protected int solveInverseCumulativeProbability(final double p, int lower, int upper) { + while (lower + 1 < upper) { + int xm = (lower + upper) / 2; + if (xm < lower || xm > upper) { + /* + * Overflow. + * There will never be an overflow in both calculation methods + * for xm at the same time + */ + xm = lower + (upper - lower) / 2; + } + + double pm = checkedCumulativeProbability(xm); + if (pm >= p) { + upper = xm; + } else { + lower = xm; + } + } + return upper; + } + + /** {@inheritDoc} */ + public void reseedRandomGenerator(long seed) { + random.setSeed(seed); + randomData.reSeed(seed); + } + + /** + * {@inheritDoc} + * + * The default implementation uses the + * + * inversion method. + */ + public int sample() { + return inverseCumulativeProbability(random.nextDouble()); + } + + /** + * {@inheritDoc} + * + * The default implementation generates the sample by calling + * {@link #sample()} in a loop. + */ + public int[] sample(int sampleSize) { + if (sampleSize <= 0) { + throw new NotStrictlyPositiveException( + LocalizedFormats.NUMBER_OF_SAMPLES, sampleSize); + } + int[] out = new int[sampleSize]; + for (int i = 0; i < sampleSize; i++) { + out[i] = sample(); + } + return out; + } + + /** + * Computes the cumulative probability function and checks for {@code NaN} + * values returned. Throws {@code MathInternalError} if the value is + * {@code NaN}. Rethrows any exception encountered evaluating the cumulative + * probability function. Throws {@code MathInternalError} if the cumulative + * probability function returns {@code NaN}. + * + * @param argument input value + * @return the cumulative probability + * @throws MathInternalError if the cumulative probability is {@code NaN} + */ + private double checkedCumulativeProbability(int argument) + throws MathInternalError { + double result = Double.NaN; + result = cumulativeProbability(argument); + if (Double.isNaN(result)) { + throw new MathInternalError(LocalizedFormats + .DISCRETE_CUMULATIVE_PROBABILITY_RETURNED_NAN, argument); + } + return result; + } + + /** + * For a random variable {@code X} whose values are distributed according to + * this distribution, this method returns {@code log(P(X = x))}, where + * {@code log} is the natural logarithm. In other words, this method + * represents the logarithm of the probability mass function (PMF) for the + * distribution. Note that due to the floating point precision and + * under/overflow issues, this method will for some distributions be more + * precise and faster than computing the logarithm of + * {@link #probability(int)}. + *

+ * The default implementation simply computes the logarithm of {@code probability(x)}.

+ * + * @param x the point at which the PMF is evaluated + * @return the logarithm of the value of the probability mass function at {@code x} + */ + public double logProbability(int x) { + return FastMath.log(probability(x)); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/AbstractRealDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/AbstractRealDistribution.java new file mode 100644 index 0000000..160e76d --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/AbstractRealDistribution.java @@ -0,0 +1,336 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import java.io.Serializable; + +import infodynamics.utils.commonsmath3.analysis.UnivariateFunction; +import infodynamics.utils.commonsmath3.analysis.solvers.UnivariateSolverUtils; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Base class for probability distributions on the reals. + * Default implementations are provided for some of the methods + * that do not vary from distribution to distribution. + * + * @since 3.0 + */ +public abstract class AbstractRealDistribution +implements RealDistribution, Serializable { + /** Default accuracy. */ + public static final double SOLVER_DEFAULT_ABSOLUTE_ACCURACY = 1e-6; + /** Serializable version identifier */ + private static final long serialVersionUID = -38038050983108802L; + /** + * RandomData instance used to generate samples from the distribution. + * @deprecated As of 3.1, to be removed in 4.0. Please use the + * {@link #random} instance variable instead. + */ + @Deprecated + protected infodynamics.utils.commonsmath3.random.RandomDataImpl randomData = + new infodynamics.utils.commonsmath3.random.RandomDataImpl(); + + /** + * RNG instance used to generate samples from the distribution. + * @since 3.1 + */ + protected final RandomGenerator random; + + /** Solver absolute accuracy for inverse cumulative computation */ + private double solverAbsoluteAccuracy = SOLVER_DEFAULT_ABSOLUTE_ACCURACY; + + /** + * @deprecated As of 3.1, to be removed in 4.0. Please use + * {@link #AbstractRealDistribution(RandomGenerator)} instead. + */ + @Deprecated + protected AbstractRealDistribution() { + // Legacy users are only allowed to access the deprecated "randomData". + // New users are forbidden to use this constructor. + random = null; + } + /** + * @param rng Random number generator. + * @since 3.1 + */ + protected AbstractRealDistribution(RandomGenerator rng) { + random = rng; + } + + /** + * {@inheritDoc} + * + * The default implementation uses the identity + *

{@code P(x0 < X <= x1) = P(X <= x1) - P(X <= x0)}

+ * + * @deprecated As of 3.1 (to be removed in 4.0). Please use + * {@link #probability(double,double)} instead. + */ + @Deprecated + public double cumulativeProbability(double x0, double x1) throws NumberIsTooLargeException { + return probability(x0, x1); + } + + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(x0 < X <= x1)}. + * + * @param x0 Lower bound (excluded). + * @param x1 Upper bound (included). + * @return the probability that a random variable with this distribution + * takes a value between {@code x0} and {@code x1}, excluding the lower + * and including the upper endpoint. + * @throws NumberIsTooLargeException if {@code x0 > x1}. + * + * The default implementation uses the identity + * {@code P(x0 < X <= x1) = P(X <= x1) - P(X <= x0)} + * + * @since 3.1 + */ + public double probability(double x0, + double x1) { + if (x0 > x1) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_ENDPOINT_ABOVE_UPPER_ENDPOINT, + x0, x1, true); + } + return cumulativeProbability(x1) - cumulativeProbability(x0); + } + + /** + * {@inheritDoc} + * + * The default implementation returns + *
    + *
  • {@link #getSupportLowerBound()} for {@code p = 0},
    • + *
  • {@link #getSupportUpperBound()} for {@code p = 1}.
    • + *
+ */ + public double inverseCumulativeProbability(final double p) throws OutOfRangeException { + /* + * IMPLEMENTATION NOTES + * -------------------- + * Where applicable, use is made of the one-sided Chebyshev inequality + * to bracket the root. This inequality states that + * P(X - mu >= k * sig) <= 1 / (1 + k^2), + * mu: mean, sig: standard deviation. Equivalently + * 1 - P(X < mu + k * sig) <= 1 / (1 + k^2), + * F(mu + k * sig) >= k^2 / (1 + k^2). + * + * For k = sqrt(p / (1 - p)), we find + * F(mu + k * sig) >= p, + * and (mu + k * sig) is an upper-bound for the root. + * + * Then, introducing Y = -X, mean(Y) = -mu, sd(Y) = sig, and + * P(Y >= -mu + k * sig) <= 1 / (1 + k^2), + * P(-X >= -mu + k * sig) <= 1 / (1 + k^2), + * P(X <= mu - k * sig) <= 1 / (1 + k^2), + * F(mu - k * sig) <= 1 / (1 + k^2). + * + * For k = sqrt((1 - p) / p), we find + * F(mu - k * sig) <= p, + * and (mu - k * sig) is a lower-bound for the root. + * + * In cases where the Chebyshev inequality does not apply, geometric + * progressions 1, 2, 4, ... and -1, -2, -4, ... are used to bracket + * the root. + */ + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0, 1); + } + + double lowerBound = getSupportLowerBound(); + if (p == 0.0) { + return lowerBound; + } + + double upperBound = getSupportUpperBound(); + if (p == 1.0) { + return upperBound; + } + + final double mu = getNumericalMean(); + final double sig = FastMath.sqrt(getNumericalVariance()); + final boolean chebyshevApplies; + chebyshevApplies = !(Double.isInfinite(mu) || Double.isNaN(mu) || + Double.isInfinite(sig) || Double.isNaN(sig)); + + if (lowerBound == Double.NEGATIVE_INFINITY) { + if (chebyshevApplies) { + lowerBound = mu - sig * FastMath.sqrt((1. - p) / p); + } else { + lowerBound = -1.0; + while (cumulativeProbability(lowerBound) >= p) { + lowerBound *= 2.0; + } + } + } + + if (upperBound == Double.POSITIVE_INFINITY) { + if (chebyshevApplies) { + upperBound = mu + sig * FastMath.sqrt(p / (1. - p)); + } else { + upperBound = 1.0; + while (cumulativeProbability(upperBound) < p) { + upperBound *= 2.0; + } + } + } + + final UnivariateFunction toSolve = new UnivariateFunction() { + /** {@inheritDoc} */ + public double value(final double x) { + return cumulativeProbability(x) - p; + } + }; + + double x = UnivariateSolverUtils.solve(toSolve, + lowerBound, + upperBound, + getSolverAbsoluteAccuracy()); + + if (!isSupportConnected()) { + /* Test for plateau. */ + final double dx = getSolverAbsoluteAccuracy(); + if (x - dx >= getSupportLowerBound()) { + double px = cumulativeProbability(x); + if (cumulativeProbability(x - dx) == px) { + upperBound = x; + while (upperBound - lowerBound > dx) { + final double midPoint = 0.5 * (lowerBound + upperBound); + if (cumulativeProbability(midPoint) < px) { + lowerBound = midPoint; + } else { + upperBound = midPoint; + } + } + return upperBound; + } + } + } + return x; + } + + /** + * Returns the solver absolute accuracy for inverse cumulative computation. + * You can override this method in order to use a Brent solver with an + * absolute accuracy different from the default. + * + * @return the maximum absolute error in inverse cumulative probability estimates + */ + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** {@inheritDoc} */ + public void reseedRandomGenerator(long seed) { + random.setSeed(seed); + randomData.reSeed(seed); + } + + /** + * {@inheritDoc} + * + * The default implementation uses the + * + * inversion method. + * + */ + public double sample() { + return inverseCumulativeProbability(random.nextDouble()); + } + + /** + * {@inheritDoc} + * + * The default implementation generates the sample by calling + * {@link #sample()} in a loop. + */ + public double[] sample(int sampleSize) { + if (sampleSize <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.NUMBER_OF_SAMPLES, + sampleSize); + } + double[] out = new double[sampleSize]; + for (int i = 0; i < sampleSize; i++) { + out[i] = sample(); + } + return out; + } + + /** + * {@inheritDoc} + * + * @return zero. + * @since 3.1 + */ + public double probability(double x) { + return 0d; + } + + /** + * Returns the natural logarithm of the probability density function (PDF) of this distribution + * evaluated at the specified point {@code x}. In general, the PDF is the derivative of the + * {@link #cumulativeProbability(double) CDF}. If the derivative does not exist at {@code x}, + * then an appropriate replacement should be returned, e.g. {@code Double.POSITIVE_INFINITY}, + * {@code Double.NaN}, or the limit inferior or limit superior of the difference quotient. Note + * that due to the floating point precision and under/overflow issues, this method will for some + * distributions be more precise and faster than computing the logarithm of + * {@link #density(double)}. The default implementation simply computes the logarithm of + * {@code density(x)}. + * + * @param x the point at which the PDF is evaluated + * @return the logarithm of the value of the probability density function at point {@code x} + */ + public double logDensity(double x) { + return FastMath.log(density(x)); + } +} + diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/BetaDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/BetaDistribution.java new file mode 100644 index 0000000..60e108e --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/BetaDistribution.java @@ -0,0 +1,435 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NumberIsTooSmallException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Beta; +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.FastMath; +import infodynamics.utils.commonsmath3.util.Precision; + +/** + * Implements the Beta distribution. + * + * @see Beta distribution + * @since 2.0 (changed to concrete class in 3.0) + */ +public class BetaDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier. */ + private static final long serialVersionUID = -1221965979403477668L; + /** First shape parameter. */ + private final double alpha; + /** Second shape parameter. */ + private final double beta; + /** Normalizing factor used in density computations. + * updated whenever alpha or beta are changed. + */ + private double z; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + + /** + * Build a new instance. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param alpha First shape parameter (must be positive). + * @param beta Second shape parameter (must be positive). + */ + public BetaDistribution(double alpha, double beta) { + this(alpha, beta, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Build a new instance. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param alpha First shape parameter (must be positive). + * @param beta Second shape parameter (must be positive). + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @since 2.1 + */ + public BetaDistribution(double alpha, double beta, double inverseCumAccuracy) { + this(new Well19937c(), alpha, beta, inverseCumAccuracy); + } + + /** + * Creates a β distribution. + * + * @param rng Random number generator. + * @param alpha First shape parameter (must be positive). + * @param beta Second shape parameter (must be positive). + * @since 3.3 + */ + public BetaDistribution(RandomGenerator rng, double alpha, double beta) { + this(rng, alpha, beta, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a β distribution. + * + * @param rng Random number generator. + * @param alpha First shape parameter (must be positive). + * @param beta Second shape parameter (must be positive). + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @since 3.1 + */ + public BetaDistribution(RandomGenerator rng, + double alpha, + double beta, + double inverseCumAccuracy) { + super(rng); + + this.alpha = alpha; + this.beta = beta; + z = Double.NaN; + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * Access the first shape parameter, {@code alpha}. + * + * @return the first shape parameter. + */ + public double getAlpha() { + return alpha; + } + + /** + * Access the second shape parameter, {@code beta}. + * + * @return the second shape parameter. + */ + public double getBeta() { + return beta; + } + + /** Recompute the normalization factor. */ + private void recomputeZ() { + if (Double.isNaN(z)) { + z = Gamma.logGamma(alpha) + Gamma.logGamma(beta) - Gamma.logGamma(alpha + beta); + } + } + + /** {@inheritDoc} */ + public double density(double x) { + final double logDensity = logDensity(x); + return logDensity == Double.NEGATIVE_INFINITY ? 0 : FastMath.exp(logDensity); + } + + /** {@inheritDoc} **/ + @Override + public double logDensity(double x) { + recomputeZ(); + if (x < 0 || x > 1) { + return Double.NEGATIVE_INFINITY; + } else if (x == 0) { + if (alpha < 1) { + throw new NumberIsTooSmallException(LocalizedFormats.CANNOT_COMPUTE_BETA_DENSITY_AT_0_FOR_SOME_ALPHA, alpha, 1, false); + } + return Double.NEGATIVE_INFINITY; + } else if (x == 1) { + if (beta < 1) { + throw new NumberIsTooSmallException(LocalizedFormats.CANNOT_COMPUTE_BETA_DENSITY_AT_1_FOR_SOME_BETA, beta, 1, false); + } + return Double.NEGATIVE_INFINITY; + } else { + double logX = FastMath.log(x); + double log1mX = FastMath.log1p(-x); + return (alpha - 1) * logX + (beta - 1) * log1mX - z; + } + } + + /** {@inheritDoc} */ + public double cumulativeProbability(double x) { + if (x <= 0) { + return 0; + } else if (x >= 1) { + return 1; + } else { + return Beta.regularizedBeta(x, alpha, beta); + } + } + + /** + * Return the absolute accuracy setting of the solver used to estimate + * inverse cumulative probabilities. + * + * @return the solver absolute accuracy. + * @since 2.1 + */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For first shape parameter {@code alpha} and second shape parameter + * {@code beta}, the mean is {@code alpha / (alpha + beta)}. + */ + public double getNumericalMean() { + final double a = getAlpha(); + return a / (a + getBeta()); + } + + /** + * {@inheritDoc} + * + * For first shape parameter {@code alpha} and second shape parameter + * {@code beta}, the variance is + * {@code (alpha * beta) / [(alpha + beta)^2 * (alpha + beta + 1)]}. + */ + public double getNumericalVariance() { + final double a = getAlpha(); + final double b = getBeta(); + final double alphabetasum = a + b; + return (a * b) / ((alphabetasum * alphabetasum) * (alphabetasum + 1)); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the parameters. + * + * @return lower bound of the support (always 0) + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always 1 no matter the parameters. + * + * @return upper bound of the support (always 1) + */ + public double getSupportUpperBound() { + return 1; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + + /** {@inheritDoc} + *

+ * Sampling is performed using Cheng algorithms: + *

+ *

+ * R. C. H. Cheng, "Generating beta variates with nonintegral shape parameters.". + * Communications of the ACM, 21, 317–322, 1978. + *

+ */ + @Override + public double sample() { + return ChengBetaSampler.sample(random, alpha, beta); + } + + /** Utility class implementing Cheng's algorithms for beta distribution sampling. + *

+ * R. C. H. Cheng, "Generating beta variates with nonintegral shape parameters.". + * Communications of the ACM, 21, 317–322, 1978. + *

+ * @since 3.6 + */ + private static final class ChengBetaSampler { + + /** + * Returns one sample using Cheng's sampling algorithm. + * @param random random generator to use + * @param alpha distribution first shape parameter + * @param beta distribution second shape parameter + * @return sampled value + */ + static double sample(RandomGenerator random, final double alpha, final double beta) { + final double a = FastMath.min(alpha, beta); + final double b = FastMath.max(alpha, beta); + + if (a > 1) { + return algorithmBB(random, alpha, a, b); + } else { + return algorithmBC(random, alpha, b, a); + } + } + + /** + * Returns one sample using Cheng's BB algorithm, when both α and β are greater than 1. + * @param random random generator to use + * @param a0 distribution first shape parameter (α) + * @param a min(α, β) where α, β are the two distribution shape parameters + * @param b max(α, β) where α, β are the two distribution shape parameters + * @return sampled value + */ + private static double algorithmBB(RandomGenerator random, + final double a0, + final double a, + final double b) { + final double alpha = a + b; + final double beta = FastMath.sqrt((alpha - 2.) / (2. * a * b - alpha)); + final double gamma = a + 1. / beta; + + double r; + double w; + double t; + do { + final double u1 = random.nextDouble(); + final double u2 = random.nextDouble(); + final double v = beta * (FastMath.log(u1) - FastMath.log1p(-u1)); + w = a * FastMath.exp(v); + final double z = u1 * u1 * u2; + r = gamma * v - 1.3862944; + final double s = a + r - w; + if (s + 2.609438 >= 5 * z) { + break; + } + + t = FastMath.log(z); + if (s >= t) { + break; + } + } while (r + alpha * (FastMath.log(alpha) - FastMath.log(b + w)) < t); + + w = FastMath.min(w, Double.MAX_VALUE); + return Precision.equals(a, a0) ? w / (b + w) : b / (b + w); + } + + /** + * Returns one sample using Cheng's BC algorithm, when at least one of α and β is smaller than 1. + * @param random random generator to use + * @param a0 distribution first shape parameter (α) + * @param a max(α, β) where α, β are the two distribution shape parameters + * @param b min(α, β) where α, β are the two distribution shape parameters + * @return sampled value + */ + private static double algorithmBC(RandomGenerator random, + final double a0, + final double a, + final double b) { + final double alpha = a + b; + final double beta = 1. / b; + final double delta = 1. + a - b; + final double k1 = delta * (0.0138889 + 0.0416667 * b) / (a * beta - 0.777778); + final double k2 = 0.25 + (0.5 + 0.25 / delta) * b; + + double w; + for (;;) { + final double u1 = random.nextDouble(); + final double u2 = random.nextDouble(); + final double y = u1 * u2; + final double z = u1 * y; + if (u1 < 0.5) { + if (0.25 * u2 + z - y >= k1) { + continue; + } + } else { + if (z <= 0.25) { + final double v = beta * (FastMath.log(u1) - FastMath.log1p(-u1)); + w = a * FastMath.exp(v); + break; + } + + if (z >= k2) { + continue; + } + } + + final double v = beta * (FastMath.log(u1) - FastMath.log1p(-u1)); + w = a * FastMath.exp(v); + if (alpha * (FastMath.log(alpha) - FastMath.log(b + w) + v) - 1.3862944 >= FastMath.log(z)) { + break; + } + } + + w = FastMath.min(w, Double.MAX_VALUE); + return Precision.equals(a, a0) ? w / (b + w) : b / (b + w); + } + + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/BinomialDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/BinomialDistribution.java new file mode 100644 index 0000000..37e4ace --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/BinomialDistribution.java @@ -0,0 +1,227 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Beta; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the binomial distribution. + * + * @see Binomial distribution (Wikipedia) + * @see Binomial Distribution (MathWorld) + */ +public class BinomialDistribution extends AbstractIntegerDistribution { + /** Serializable version identifier. */ + private static final long serialVersionUID = 6751309484392813623L; + /** The number of trials. */ + private final int numberOfTrials; + /** The probability of success. */ + private final double probabilityOfSuccess; + + /** + * Create a binomial distribution with the given number of trials and + * probability of success. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param trials Number of trials. + * @param p Probability of success. + * @throws NotPositiveException if {@code trials < 0}. + * @throws OutOfRangeException if {@code p < 0} or {@code p > 1}. + */ + public BinomialDistribution(int trials, double p) { + this(new Well19937c(), trials, p); + } + + /** + * Creates a binomial distribution. + * + * @param rng Random number generator. + * @param trials Number of trials. + * @param p Probability of success. + * @throws NotPositiveException if {@code trials < 0}. + * @throws OutOfRangeException if {@code p < 0} or {@code p > 1}. + * @since 3.1 + */ + public BinomialDistribution(RandomGenerator rng, + int trials, + double p) { + super(rng); + + if (trials < 0) { + throw new NotPositiveException(LocalizedFormats.NUMBER_OF_TRIALS, + trials); + } + if (p < 0 || p > 1) { + throw new OutOfRangeException(p, 0, 1); + } + + probabilityOfSuccess = p; + numberOfTrials = trials; + } + + /** + * Access the number of trials for this distribution. + * + * @return the number of trials. + */ + public int getNumberOfTrials() { + return numberOfTrials; + } + + /** + * Access the probability of success for this distribution. + * + * @return the probability of success. + */ + public double getProbabilityOfSuccess() { + return probabilityOfSuccess; + } + + /** {@inheritDoc} */ + public double probability(int x) { + final double logProbability = logProbability(x); + return logProbability == Double.NEGATIVE_INFINITY ? 0 : FastMath.exp(logProbability); + } + + /** {@inheritDoc} **/ + @Override + public double logProbability(int x) { + if (numberOfTrials == 0) { + return (x == 0) ? 0. : Double.NEGATIVE_INFINITY; + } + double ret; + if (x < 0 || x > numberOfTrials) { + ret = Double.NEGATIVE_INFINITY; + } else { + ret = SaddlePointExpansion.logBinomialProbability(x, + numberOfTrials, probabilityOfSuccess, + 1.0 - probabilityOfSuccess); + } + return ret; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(int x) { + double ret; + if (x < 0) { + ret = 0.0; + } else if (x >= numberOfTrials) { + ret = 1.0; + } else { + ret = 1.0 - Beta.regularizedBeta(probabilityOfSuccess, + x + 1.0, numberOfTrials - x); + } + return ret; + } + + /** + * {@inheritDoc} + * + * For {@code n} trials and probability parameter {@code p}, the mean is + * {@code n * p}. + */ + public double getNumericalMean() { + return numberOfTrials * probabilityOfSuccess; + } + + /** + * {@inheritDoc} + * + * For {@code n} trials and probability parameter {@code p}, the variance is + * {@code n * p * (1 - p)}. + */ + public double getNumericalVariance() { + final double p = probabilityOfSuccess; + return numberOfTrials * p * (1 - p); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 except for the probability + * parameter {@code p = 1}. + * + * @return lower bound of the support (0 or the number of trials) + */ + public int getSupportLowerBound() { + return probabilityOfSuccess < 1.0 ? 0 : numberOfTrials; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is the number of trials except for the + * probability parameter {@code p = 0}. + * + * @return upper bound of the support (number of trials or 0) + */ + public int getSupportUpperBound() { + return probabilityOfSuccess > 0.0 ? numberOfTrials : 0; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/CauchyDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/CauchyDistribution.java new file mode 100644 index 0000000..03ca4ac --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/CauchyDistribution.java @@ -0,0 +1,285 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the Cauchy distribution. + * + * @see Cauchy distribution (Wikipedia) + * @see Cauchy Distribution (MathWorld) + * @since 1.1 (changed to concrete class in 3.0) + */ +public class CauchyDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier */ + private static final long serialVersionUID = 8589540077390120676L; + /** The median of this distribution. */ + private final double median; + /** The scale of this distribution. */ + private final double scale; + /** Inverse cumulative probability accuracy */ + private final double solverAbsoluteAccuracy; + + /** + * Creates a Cauchy distribution with the median equal to zero and scale + * equal to one. + */ + public CauchyDistribution() { + this(0, 1); + } + + /** + * Creates a Cauchy distribution using the given median and scale. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param median Median for this distribution. + * @param scale Scale parameter for this distribution. + */ + public CauchyDistribution(double median, double scale) { + this(median, scale, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a Cauchy distribution using the given median and scale. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param median Median for this distribution. + * @param scale Scale parameter for this distribution. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code scale <= 0}. + * @since 2.1 + */ + public CauchyDistribution(double median, double scale, + double inverseCumAccuracy) { + this(new Well19937c(), median, scale, inverseCumAccuracy); + } + + /** + * Creates a Cauchy distribution. + * + * @param rng Random number generator. + * @param median Median for this distribution. + * @param scale Scale parameter for this distribution. + * @throws NotStrictlyPositiveException if {@code scale <= 0}. + * @since 3.3 + */ + public CauchyDistribution(RandomGenerator rng, double median, double scale) { + this(rng, median, scale, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a Cauchy distribution. + * + * @param rng Random number generator. + * @param median Median for this distribution. + * @param scale Scale parameter for this distribution. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code scale <= 0}. + * @since 3.1 + */ + public CauchyDistribution(RandomGenerator rng, + double median, + double scale, + double inverseCumAccuracy) { + super(rng); + if (scale <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.SCALE, scale); + } + this.scale = scale; + this.median = median; + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(double x) { + return 0.5 + (FastMath.atan((x - median) / scale) / FastMath.PI); + } + + /** + * Access the median. + * + * @return the median for this distribution. + */ + public double getMedian() { + return median; + } + + /** + * Access the scale parameter. + * + * @return the scale parameter for this distribution. + */ + public double getScale() { + return scale; + } + + /** {@inheritDoc} */ + public double density(double x) { + final double dev = x - median; + return (1 / FastMath.PI) * (scale / (dev * dev + scale * scale)); + } + + /** + * {@inheritDoc} + * + * Returns {@code Double.NEGATIVE_INFINITY} when {@code p == 0} + * and {@code Double.POSITIVE_INFINITY} when {@code p == 1}. + */ + @Override + public double inverseCumulativeProbability(double p) throws OutOfRangeException { + double ret; + if (p < 0 || p > 1) { + throw new OutOfRangeException(p, 0, 1); + } else if (p == 0) { + ret = Double.NEGATIVE_INFINITY; + } else if (p == 1) { + ret = Double.POSITIVE_INFINITY; + } else { + ret = median + scale * FastMath.tan(FastMath.PI * (p - .5)); + } + return ret; + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * The mean is always undefined no matter the parameters. + * + * @return mean (always Double.NaN) + */ + public double getNumericalMean() { + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * The variance is always undefined no matter the parameters. + * + * @return variance (always Double.NaN) + */ + public double getNumericalVariance() { + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always negative infinity no matter + * the parameters. + * + * @return lower bound of the support (always Double.NEGATIVE_INFINITY) + */ + public double getSupportLowerBound() { + return Double.NEGATIVE_INFINITY; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity no matter + * the parameters. + * + * @return upper bound of the support (always Double.POSITIVE_INFINITY) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/ChiSquaredDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/ChiSquaredDistribution.java new file mode 100644 index 0000000..21fa700 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/ChiSquaredDistribution.java @@ -0,0 +1,225 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; + +/** + * Implementation of the chi-squared distribution. + * + * @see Chi-squared distribution (Wikipedia) + * @see Chi-squared Distribution (MathWorld) + */ +public class ChiSquaredDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier */ + private static final long serialVersionUID = -8352658048349159782L; + /** Internal Gamma distribution. */ + private final GammaDistribution gamma; + /** Inverse cumulative probability accuracy */ + private final double solverAbsoluteAccuracy; + + /** + * Create a Chi-Squared distribution with the given degrees of freedom. + * + * @param degreesOfFreedom Degrees of freedom. + */ + public ChiSquaredDistribution(double degreesOfFreedom) { + this(degreesOfFreedom, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a Chi-Squared distribution with the given degrees of freedom and + * inverse cumulative probability accuracy. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param degreesOfFreedom Degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @since 2.1 + */ + public ChiSquaredDistribution(double degreesOfFreedom, + double inverseCumAccuracy) { + this(new Well19937c(), degreesOfFreedom, inverseCumAccuracy); + } + + /** + * Create a Chi-Squared distribution with the given degrees of freedom. + * + * @param rng Random number generator. + * @param degreesOfFreedom Degrees of freedom. + * @since 3.3 + */ + public ChiSquaredDistribution(RandomGenerator rng, double degreesOfFreedom) { + this(rng, degreesOfFreedom, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a Chi-Squared distribution with the given degrees of freedom and + * inverse cumulative probability accuracy. + * + * @param rng Random number generator. + * @param degreesOfFreedom Degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @since 3.1 + */ + public ChiSquaredDistribution(RandomGenerator rng, + double degreesOfFreedom, + double inverseCumAccuracy) { + super(rng); + + gamma = new GammaDistribution(degreesOfFreedom / 2, 2); + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * Access the number of degrees of freedom. + * + * @return the degrees of freedom. + */ + public double getDegreesOfFreedom() { + return gamma.getShape() * 2.0; + } + + /** {@inheritDoc} */ + public double density(double x) { + return gamma.density(x); + } + + /** {@inheritDoc} **/ + @Override + public double logDensity(double x) { + return gamma.logDensity(x); + } + + /** {@inheritDoc} */ + public double cumulativeProbability(double x) { + return gamma.cumulativeProbability(x); + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For {@code k} degrees of freedom, the mean is {@code k}. + */ + public double getNumericalMean() { + return getDegreesOfFreedom(); + } + + /** + * {@inheritDoc} + * + * @return {@code 2 * k}, where {@code k} is the number of degrees of freedom. + */ + public double getNumericalVariance() { + return 2 * getDegreesOfFreedom(); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the + * degrees of freedom. + * + * @return zero. + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity no matter the + * degrees of freedom. + * + * @return {@code Double.POSITIVE_INFINITY}. + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return true; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/ExponentialDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/ExponentialDistribution.java new file mode 100644 index 0000000..e87d76d --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/ExponentialDistribution.java @@ -0,0 +1,380 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.util.CombinatoricsUtils; +import infodynamics.utils.commonsmath3.util.FastMath; +import infodynamics.utils.commonsmath3.util.ResizableDoubleArray; + +/** + * Implementation of the exponential distribution. + * + * @see Exponential distribution (Wikipedia) + * @see Exponential distribution (MathWorld) + */ +public class ExponentialDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier */ + private static final long serialVersionUID = 2401296428283614780L; + /** + * Used when generating Exponential samples. + * Table containing the constants + * q_i = sum_{j=1}^i (ln 2)^j/j! = ln 2 + (ln 2)^2/2 + ... + (ln 2)^i/i! + * until the largest representable fraction below 1 is exceeded. + * + * Note that + * 1 = 2 - 1 = exp(ln 2) - 1 = sum_{n=1}^infty (ln 2)^n / n! + * thus q_i -> 1 as i -> +inf, + * so the higher i, the closer to one we get (the series is not alternating). + * + * By trying, n = 16 in Java is enough to reach 1.0. + */ + private static final double[] EXPONENTIAL_SA_QI; + /** The mean of this distribution. */ + private final double mean; + /** The logarithm of the mean, stored to reduce computing time. **/ + private final double logMean; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + + /** + * Initialize tables. + */ + static { + /** + * Filling EXPONENTIAL_SA_QI table. + * Note that we don't want qi = 0 in the table. + */ + final double LN2 = FastMath.log(2); + double qi = 0; + int i = 1; + + /** + * ArithmeticUtils provides factorials up to 20, so let's use that + * limit together with Precision.EPSILON to generate the following + * code (a priori, we know that there will be 16 elements, but it is + * better to not hardcode it). + */ + final ResizableDoubleArray ra = new ResizableDoubleArray(20); + + while (qi < 1) { + qi += FastMath.pow(LN2, i) / CombinatoricsUtils.factorial(i); + ra.addElement(qi); + ++i; + } + + EXPONENTIAL_SA_QI = ra.getElements(); + } + + /** + * Create an exponential distribution with the given mean. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param mean mean of this distribution. + */ + public ExponentialDistribution(double mean) { + this(mean, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create an exponential distribution with the given mean. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param mean Mean of this distribution. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code mean <= 0}. + * @since 2.1 + */ + public ExponentialDistribution(double mean, double inverseCumAccuracy) { + this(new Well19937c(), mean, inverseCumAccuracy); + } + + /** + * Creates an exponential distribution. + * + * @param rng Random number generator. + * @param mean Mean of this distribution. + * @throws NotStrictlyPositiveException if {@code mean <= 0}. + * @since 3.3 + */ + public ExponentialDistribution(RandomGenerator rng, double mean) + throws NotStrictlyPositiveException { + this(rng, mean, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates an exponential distribution. + * + * @param rng Random number generator. + * @param mean Mean of this distribution. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code mean <= 0}. + * @since 3.1 + */ + public ExponentialDistribution(RandomGenerator rng, + double mean, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (mean <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.MEAN, mean); + } + this.mean = mean; + logMean = FastMath.log(mean); + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * Access the mean. + * + * @return the mean. + */ + public double getMean() { + return mean; + } + + /** {@inheritDoc} */ + public double density(double x) { + final double logDensity = logDensity(x); + return logDensity == Double.NEGATIVE_INFINITY ? 0 : FastMath.exp(logDensity); + } + + /** {@inheritDoc} **/ + @Override + public double logDensity(double x) { + if (x < 0) { + return Double.NEGATIVE_INFINITY; + } + return -x / mean - logMean; + } + + /** + * {@inheritDoc} + * + * The implementation of this method is based on: + *

+ */ + public double cumulativeProbability(double x) { + double ret; + if (x <= 0.0) { + ret = 0.0; + } else { + ret = 1.0 - FastMath.exp(-x / mean); + } + return ret; + } + + /** + * {@inheritDoc} + * + * Returns {@code 0} when {@code p= = 0} and + * {@code Double.POSITIVE_INFINITY} when {@code p == 1}. + */ + @Override + public double inverseCumulativeProbability(double p) throws OutOfRangeException { + double ret; + + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0.0, 1.0); + } else if (p == 1.0) { + ret = Double.POSITIVE_INFINITY; + } else { + ret = -mean * FastMath.log(1.0 - p); + } + + return ret; + } + + /** + * {@inheritDoc} + * + *

Algorithm Description: this implementation uses the + * + * Inversion Method to generate exponentially distributed random values + * from uniform deviates.

+ * + * @return a random value. + * @since 2.2 + */ + @Override + public double sample() { + // Step 1: + double a = 0; + double u = random.nextDouble(); + + // Step 2 and 3: + while (u < 0.5) { + a += EXPONENTIAL_SA_QI[0]; + u *= 2; + } + + // Step 4 (now u >= 0.5): + u += u - 1; + + // Step 5: + if (u <= EXPONENTIAL_SA_QI[0]) { + return mean * (a + u); + } + + // Step 6: + int i = 0; // Should be 1, be we iterate before it in while using 0 + double u2 = random.nextDouble(); + double umin = u2; + + // Step 7 and 8: + do { + ++i; + u2 = random.nextDouble(); + + if (u2 < umin) { + umin = u2; + } + + // Step 8: + } while (u > EXPONENTIAL_SA_QI[i]); // Ensured to exit since EXPONENTIAL_SA_QI[MAX] = 1 + + return mean * (a + umin * EXPONENTIAL_SA_QI[0]); + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For mean parameter {@code k}, the mean is {@code k}. + */ + public double getNumericalMean() { + return getMean(); + } + + /** + * {@inheritDoc} + * + * For mean parameter {@code k}, the variance is {@code k^2}. + */ + public double getNumericalVariance() { + final double m = getMean(); + return m * m; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the mean parameter. + * + * @return lower bound of the support (always 0) + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity + * no matter the mean parameter. + * + * @return upper bound of the support (always Double.POSITIVE_INFINITY) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return true; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/FDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/FDistribution.java new file mode 100644 index 0000000..7ddb177 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/FDistribution.java @@ -0,0 +1,357 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Beta; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the F-distribution. + * + * @see F-distribution (Wikipedia) + * @see F-distribution (MathWorld) + */ +public class FDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier. */ + private static final long serialVersionUID = -8516354193418641566L; + /** The numerator degrees of freedom. */ + private final double numeratorDegreesOfFreedom; + /** The numerator degrees of freedom. */ + private final double denominatorDegreesOfFreedom; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + /** Cached numerical variance */ + private double numericalVariance = Double.NaN; + /** Whether or not the numerical variance has been calculated */ + private boolean numericalVarianceIsCalculated = false; + + /** + * Creates an F distribution using the given degrees of freedom. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param numeratorDegreesOfFreedom Numerator degrees of freedom. + * @param denominatorDegreesOfFreedom Denominator degrees of freedom. + * @throws NotStrictlyPositiveException if + * {@code numeratorDegreesOfFreedom <= 0} or + * {@code denominatorDegreesOfFreedom <= 0}. + */ + public FDistribution(double numeratorDegreesOfFreedom, + double denominatorDegreesOfFreedom) + throws NotStrictlyPositiveException { + this(numeratorDegreesOfFreedom, denominatorDegreesOfFreedom, + DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates an F distribution using the given degrees of freedom + * and inverse cumulative probability accuracy. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param numeratorDegreesOfFreedom Numerator degrees of freedom. + * @param denominatorDegreesOfFreedom Denominator degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates. + * @throws NotStrictlyPositiveException if + * {@code numeratorDegreesOfFreedom <= 0} or + * {@code denominatorDegreesOfFreedom <= 0}. + * @since 2.1 + */ + public FDistribution(double numeratorDegreesOfFreedom, + double denominatorDegreesOfFreedom, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + this(new Well19937c(), numeratorDegreesOfFreedom, + denominatorDegreesOfFreedom, inverseCumAccuracy); + } + + /** + * Creates an F distribution. + * + * @param rng Random number generator. + * @param numeratorDegreesOfFreedom Numerator degrees of freedom. + * @param denominatorDegreesOfFreedom Denominator degrees of freedom. + * @throws NotStrictlyPositiveException if {@code numeratorDegreesOfFreedom <= 0} or + * {@code denominatorDegreesOfFreedom <= 0}. + * @since 3.3 + */ + public FDistribution(RandomGenerator rng, + double numeratorDegreesOfFreedom, + double denominatorDegreesOfFreedom) + throws NotStrictlyPositiveException { + this(rng, numeratorDegreesOfFreedom, denominatorDegreesOfFreedom, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates an F distribution. + * + * @param rng Random number generator. + * @param numeratorDegreesOfFreedom Numerator degrees of freedom. + * @param denominatorDegreesOfFreedom Denominator degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates. + * @throws NotStrictlyPositiveException if {@code numeratorDegreesOfFreedom <= 0} or + * {@code denominatorDegreesOfFreedom <= 0}. + * @since 3.1 + */ + public FDistribution(RandomGenerator rng, + double numeratorDegreesOfFreedom, + double denominatorDegreesOfFreedom, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (numeratorDegreesOfFreedom <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.DEGREES_OF_FREEDOM, + numeratorDegreesOfFreedom); + } + if (denominatorDegreesOfFreedom <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.DEGREES_OF_FREEDOM, + denominatorDegreesOfFreedom); + } + this.numeratorDegreesOfFreedom = numeratorDegreesOfFreedom; + this.denominatorDegreesOfFreedom = denominatorDegreesOfFreedom; + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * {@inheritDoc} + * + * @since 2.1 + */ + public double density(double x) { + return FastMath.exp(logDensity(x)); + } + + /** {@inheritDoc} **/ + @Override + public double logDensity(double x) { + final double nhalf = numeratorDegreesOfFreedom / 2; + final double mhalf = denominatorDegreesOfFreedom / 2; + final double logx = FastMath.log(x); + final double logn = FastMath.log(numeratorDegreesOfFreedom); + final double logm = FastMath.log(denominatorDegreesOfFreedom); + final double lognxm = FastMath.log(numeratorDegreesOfFreedom * x + + denominatorDegreesOfFreedom); + return nhalf * logn + nhalf * logx - logx + + mhalf * logm - nhalf * lognxm - mhalf * lognxm - + Beta.logBeta(nhalf, mhalf); + } + + /** + * {@inheritDoc} + * + * The implementation of this method is based on + *

+ */ + public double cumulativeProbability(double x) { + double ret; + if (x <= 0) { + ret = 0; + } else { + double n = numeratorDegreesOfFreedom; + double m = denominatorDegreesOfFreedom; + + ret = Beta.regularizedBeta((n * x) / (m + n * x), + 0.5 * n, + 0.5 * m); + } + return ret; + } + + /** + * Access the numerator degrees of freedom. + * + * @return the numerator degrees of freedom. + */ + public double getNumeratorDegreesOfFreedom() { + return numeratorDegreesOfFreedom; + } + + /** + * Access the denominator degrees of freedom. + * + * @return the denominator degrees of freedom. + */ + public double getDenominatorDegreesOfFreedom() { + return denominatorDegreesOfFreedom; + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For denominator degrees of freedom parameter {@code b}, the mean is + *
    + *
  • if {@code b > 2} then {@code b / (b - 2)},
    • + *
  • else undefined ({@code Double.NaN}). + *
+ */ + public double getNumericalMean() { + final double denominatorDF = getDenominatorDegreesOfFreedom(); + + if (denominatorDF > 2) { + return denominatorDF / (denominatorDF - 2); + } + + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * For numerator degrees of freedom parameter {@code a} and denominator + * degrees of freedom parameter {@code b}, the variance is + *
    + *
  • + * if {@code b > 4} then + * {@code [2 * b^2 * (a + b - 2)] / [a * (b - 2)^2 * (b - 4)]}, + *
    • + *
  • else undefined ({@code Double.NaN}). + *
+ */ + public double getNumericalVariance() { + if (!numericalVarianceIsCalculated) { + numericalVariance = calculateNumericalVariance(); + numericalVarianceIsCalculated = true; + } + return numericalVariance; + } + + /** + * used by {@link #getNumericalVariance()} + * + * @return the variance of this distribution + */ + protected double calculateNumericalVariance() { + final double denominatorDF = getDenominatorDegreesOfFreedom(); + + if (denominatorDF > 4) { + final double numeratorDF = getNumeratorDegreesOfFreedom(); + final double denomDFMinusTwo = denominatorDF - 2; + + return ( 2 * (denominatorDF * denominatorDF) * (numeratorDF + denominatorDF - 2) ) / + ( (numeratorDF * (denomDFMinusTwo * denomDFMinusTwo) * (denominatorDF - 4)) ); + } + + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the parameters. + * + * @return lower bound of the support (always 0) + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity + * no matter the parameters. + * + * @return upper bound of the support (always Double.POSITIVE_INFINITY) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/GammaDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/GammaDistribution.java new file mode 100644 index 0000000..09dd772 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/GammaDistribution.java @@ -0,0 +1,542 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the Gamma distribution. + * + * @see Gamma distribution (Wikipedia) + * @see Gamma distribution (MathWorld) + */ +public class GammaDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier. */ + private static final long serialVersionUID = 20120524L; + /** The shape parameter. */ + private final double shape; + /** The scale parameter. */ + private final double scale; + /** + * The constant value of {@code shape + g + 0.5}, where {@code g} is the + * Lanczos constant {@link Gamma#LANCZOS_G}. + */ + private final double shiftedShape; + /** + * The constant value of + * {@code shape / scale * sqrt(e / (2 * pi * (shape + g + 0.5))) / L(shape)}, + * where {@code L(shape)} is the Lanczos approximation returned by + * {@link Gamma#lanczos(double)}. This prefactor is used in + * {@link #density(double)}, when no overflow occurs with the natural + * calculation. + */ + private final double densityPrefactor1; + /** + * The constant value of + * {@code log(shape / scale * sqrt(e / (2 * pi * (shape + g + 0.5))) / L(shape))}, + * where {@code L(shape)} is the Lanczos approximation returned by + * {@link Gamma#lanczos(double)}. This prefactor is used in + * {@link #logDensity(double)}, when no overflow occurs with the natural + * calculation. + */ + private final double logDensityPrefactor1; + /** + * The constant value of + * {@code shape * sqrt(e / (2 * pi * (shape + g + 0.5))) / L(shape)}, + * where {@code L(shape)} is the Lanczos approximation returned by + * {@link Gamma#lanczos(double)}. This prefactor is used in + * {@link #density(double)}, when overflow occurs with the natural + * calculation. + */ + private final double densityPrefactor2; + /** + * The constant value of + * {@code log(shape * sqrt(e / (2 * pi * (shape + g + 0.5))) / L(shape))}, + * where {@code L(shape)} is the Lanczos approximation returned by + * {@link Gamma#lanczos(double)}. This prefactor is used in + * {@link #logDensity(double)}, when overflow occurs with the natural + * calculation. + */ + private final double logDensityPrefactor2; + /** + * Lower bound on {@code y = x / scale} for the selection of the computation + * method in {@link #density(double)}. For {@code y <= minY}, the natural + * calculation overflows. + */ + private final double minY; + /** + * Upper bound on {@code log(y)} ({@code y = x / scale}) for the selection + * of the computation method in {@link #density(double)}. For + * {@code log(y) >= maxLogY}, the natural calculation overflows. + */ + private final double maxLogY; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + + /** + * Creates a new gamma distribution with specified values of the shape and + * scale parameters. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param shape the shape parameter + * @param scale the scale parameter + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + */ + public GammaDistribution(double shape, double scale) throws NotStrictlyPositiveException { + this(shape, scale, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a new gamma distribution with specified values of the shape and + * scale parameters. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param shape the shape parameter + * @param scale the scale parameter + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + * @since 2.1 + */ + public GammaDistribution(double shape, double scale, double inverseCumAccuracy) + throws NotStrictlyPositiveException { + this(new Well19937c(), shape, scale, inverseCumAccuracy); + } + + /** + * Creates a Gamma distribution. + * + * @param rng Random number generator. + * @param shape the shape parameter + * @param scale the scale parameter + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + * @since 3.3 + */ + public GammaDistribution(RandomGenerator rng, double shape, double scale) + throws NotStrictlyPositiveException { + this(rng, shape, scale, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a Gamma distribution. + * + * @param rng Random number generator. + * @param shape the shape parameter + * @param scale the scale parameter + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates (defaults to + * {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + * @since 3.1 + */ + public GammaDistribution(RandomGenerator rng, + double shape, + double scale, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (shape <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.SHAPE, shape); + } + if (scale <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.SCALE, scale); + } + + this.shape = shape; + this.scale = scale; + this.solverAbsoluteAccuracy = inverseCumAccuracy; + this.shiftedShape = shape + Gamma.LANCZOS_G + 0.5; + final double aux = FastMath.E / (2.0 * FastMath.PI * shiftedShape); + this.densityPrefactor2 = shape * FastMath.sqrt(aux) / Gamma.lanczos(shape); + this.logDensityPrefactor2 = FastMath.log(shape) + 0.5 * FastMath.log(aux) - + FastMath.log(Gamma.lanczos(shape)); + this.densityPrefactor1 = this.densityPrefactor2 / scale * + FastMath.pow(shiftedShape, -shape) * + FastMath.exp(shape + Gamma.LANCZOS_G); + this.logDensityPrefactor1 = this.logDensityPrefactor2 - FastMath.log(scale) - + FastMath.log(shiftedShape) * shape + + shape + Gamma.LANCZOS_G; + this.minY = shape + Gamma.LANCZOS_G - FastMath.log(Double.MAX_VALUE); + this.maxLogY = FastMath.log(Double.MAX_VALUE) / (shape - 1.0); + } + + /** + * Returns the shape parameter of {@code this} distribution. + * + * @return the shape parameter + * @deprecated as of version 3.1, {@link #getShape()} should be preferred. + * This method will be removed in version 4.0. + */ + @Deprecated + public double getAlpha() { + return shape; + } + + /** + * Returns the shape parameter of {@code this} distribution. + * + * @return the shape parameter + * @since 3.1 + */ + public double getShape() { + return shape; + } + + /** + * Returns the scale parameter of {@code this} distribution. + * + * @return the scale parameter + * @deprecated as of version 3.1, {@link #getScale()} should be preferred. + * This method will be removed in version 4.0. + */ + @Deprecated + public double getBeta() { + return scale; + } + + /** + * Returns the scale parameter of {@code this} distribution. + * + * @return the scale parameter + * @since 3.1 + */ + public double getScale() { + return scale; + } + + /** {@inheritDoc} */ + public double density(double x) { + /* The present method must return the value of + * + * 1 x a - x + * ---------- (-) exp(---) + * x Gamma(a) b b + * + * where a is the shape parameter, and b the scale parameter. + * Substituting the Lanczos approximation of Gamma(a) leads to the + * following expression of the density + * + * a e 1 y a + * - sqrt(------------------) ---- (-----------) exp(a - y + g), + * x 2 pi (a + g + 0.5) L(a) a + g + 0.5 + * + * where y = x / b. The above formula is the "natural" computation, which + * is implemented when no overflow is likely to occur. If overflow occurs + * with the natural computation, the following identity is used. It is + * based on the BOOST library + * http://www.boost.org/doc/libs/1_35_0/libs/math/doc/sf_and_dist/html/math_toolkit/special/sf_gamma/igamma.html + * Formula (15) needs adaptations, which are detailed below. + * + * y a + * (-----------) exp(a - y + g) + * a + g + 0.5 + * y - a - g - 0.5 y (g + 0.5) + * = exp(a log1pm(---------------) - ----------- + g), + * a + g + 0.5 a + g + 0.5 + * + * where log1pm(z) = log(1 + z) - z. Therefore, the value to be + * returned is + * + * a e 1 + * - sqrt(------------------) ---- + * x 2 pi (a + g + 0.5) L(a) + * y - a - g - 0.5 y (g + 0.5) + * * exp(a log1pm(---------------) - ----------- + g). + * a + g + 0.5 a + g + 0.5 + */ + if (x < 0) { + return 0; + } + final double y = x / scale; + if ((y <= minY) || (FastMath.log(y) >= maxLogY)) { + /* + * Overflow. + */ + final double aux1 = (y - shiftedShape) / shiftedShape; + final double aux2 = shape * (FastMath.log1p(aux1) - aux1); + final double aux3 = -y * (Gamma.LANCZOS_G + 0.5) / shiftedShape + + Gamma.LANCZOS_G + aux2; + return densityPrefactor2 / x * FastMath.exp(aux3); + } + /* + * Natural calculation. + */ + return densityPrefactor1 * FastMath.exp(-y) * FastMath.pow(y, shape - 1); + } + + /** {@inheritDoc} **/ + @Override + public double logDensity(double x) { + /* + * see the comment in {@link #density(double)} for computation details + */ + if (x < 0) { + return Double.NEGATIVE_INFINITY; + } + final double y = x / scale; + if ((y <= minY) || (FastMath.log(y) >= maxLogY)) { + /* + * Overflow. + */ + final double aux1 = (y - shiftedShape) / shiftedShape; + final double aux2 = shape * (FastMath.log1p(aux1) - aux1); + final double aux3 = -y * (Gamma.LANCZOS_G + 0.5) / shiftedShape + + Gamma.LANCZOS_G + aux2; + return logDensityPrefactor2 - FastMath.log(x) + aux3; + } + /* + * Natural calculation. + */ + return logDensityPrefactor1 - y + FastMath.log(y) * (shape - 1); + } + + /** + * {@inheritDoc} + * + * The implementation of this method is based on: + *

    + *
  • + * + * Chi-Squared Distribution, equation (9). + *
    • + *
  • Casella, G., & Berger, R. (1990). Statistical Inference. + * Belmont, CA: Duxbury Press. + *
    • + *
+ */ + public double cumulativeProbability(double x) { + double ret; + + if (x <= 0) { + ret = 0; + } else { + ret = Gamma.regularizedGammaP(shape, x / scale); + } + + return ret; + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For shape parameter {@code alpha} and scale parameter {@code beta}, the + * mean is {@code alpha * beta}. + */ + public double getNumericalMean() { + return shape * scale; + } + + /** + * {@inheritDoc} + * + * For shape parameter {@code alpha} and scale parameter {@code beta}, the + * variance is {@code alpha * beta^2}. + * + * @return {@inheritDoc} + */ + public double getNumericalVariance() { + return shape * scale * scale; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the parameters. + * + * @return lower bound of the support (always 0) + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity + * no matter the parameters. + * + * @return upper bound of the support (always Double.POSITIVE_INFINITY) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return true; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** + *

This implementation uses the following algorithms:

+ * + *

For 0 < shape < 1:
+ * Ahrens, J. H. and Dieter, U., Computer methods for + * sampling from gamma, beta, Poisson and binomial distributions. + * Computing, 12, 223-246, 1974.

+ * + *

For shape >= 1:
+ * Marsaglia and Tsang, A Simple Method for Generating + * Gamma Variables. ACM Transactions on Mathematical Software, + * Volume 26 Issue 3, September, 2000.

+ * + * @return random value sampled from the Gamma(shape, scale) distribution + */ + @Override + public double sample() { + if (shape < 1) { + // [1]: p. 228, Algorithm GS + + while (true) { + // Step 1: + final double u = random.nextDouble(); + final double bGS = 1 + shape / FastMath.E; + final double p = bGS * u; + + if (p <= 1) { + // Step 2: + + final double x = FastMath.pow(p, 1 / shape); + final double u2 = random.nextDouble(); + + if (u2 > FastMath.exp(-x)) { + // Reject + continue; + } else { + return scale * x; + } + } else { + // Step 3: + + final double x = -1 * FastMath.log((bGS - p) / shape); + final double u2 = random.nextDouble(); + + if (u2 > FastMath.pow(x, shape - 1)) { + // Reject + continue; + } else { + return scale * x; + } + } + } + } + + // Now shape >= 1 + + final double d = shape - 0.333333333333333333; + final double c = 1 / (3 * FastMath.sqrt(d)); + + while (true) { + final double x = random.nextGaussian(); + final double v = (1 + c * x) * (1 + c * x) * (1 + c * x); + + if (v <= 0) { + continue; + } + + final double x2 = x * x; + final double u = random.nextDouble(); + + // Squeeze + if (u < 1 - 0.0331 * x2 * x2) { + return scale * d * v; + } + + if (FastMath.log(u) < 0.5 * x2 + d * (1 - v + FastMath.log(v))) { + return scale * d * v; + } + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/HypergeometricDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/HypergeometricDistribution.java new file mode 100644 index 0000000..ec41c4e --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/HypergeometricDistribution.java @@ -0,0 +1,376 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotPositiveException; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the hypergeometric distribution. + * + * @see Hypergeometric distribution (Wikipedia) + * @see Hypergeometric distribution (MathWorld) + */ +public class HypergeometricDistribution extends AbstractIntegerDistribution { + /** Serializable version identifier. */ + private static final long serialVersionUID = -436928820673516179L; + /** The number of successes in the population. */ + private final int numberOfSuccesses; + /** The population size. */ + private final int populationSize; + /** The sample size. */ + private final int sampleSize; + /** Cached numerical variance */ + private double numericalVariance = Double.NaN; + /** Whether or not the numerical variance has been calculated */ + private boolean numericalVarianceIsCalculated = false; + + /** + * Construct a new hypergeometric distribution with the specified population + * size, number of successes in the population, and sample size. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param populationSize Population size. + * @param numberOfSuccesses Number of successes in the population. + * @param sampleSize Sample size. + * @throws NotPositiveException if {@code numberOfSuccesses < 0}. + * @throws NotStrictlyPositiveException if {@code populationSize <= 0}. + * @throws NumberIsTooLargeException if {@code numberOfSuccesses > populationSize}, + * or {@code sampleSize > populationSize}. + */ + public HypergeometricDistribution(int populationSize, int numberOfSuccesses, int sampleSize) + throws NotPositiveException, NotStrictlyPositiveException, NumberIsTooLargeException { + this(new Well19937c(), populationSize, numberOfSuccesses, sampleSize); + } + + /** + * Creates a new hypergeometric distribution. + * + * @param rng Random number generator. + * @param populationSize Population size. + * @param numberOfSuccesses Number of successes in the population. + * @param sampleSize Sample size. + * @throws NotPositiveException if {@code numberOfSuccesses < 0}. + * @throws NotStrictlyPositiveException if {@code populationSize <= 0}. + * @throws NumberIsTooLargeException if {@code numberOfSuccesses > populationSize}, + * or {@code sampleSize > populationSize}. + * @since 3.1 + */ + public HypergeometricDistribution(RandomGenerator rng, + int populationSize, + int numberOfSuccesses, + int sampleSize) + throws NotPositiveException, NotStrictlyPositiveException, NumberIsTooLargeException { + super(rng); + + if (populationSize <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.POPULATION_SIZE, + populationSize); + } + if (numberOfSuccesses < 0) { + throw new NotPositiveException(LocalizedFormats.NUMBER_OF_SUCCESSES, + numberOfSuccesses); + } + if (sampleSize < 0) { + throw new NotPositiveException(LocalizedFormats.NUMBER_OF_SAMPLES, + sampleSize); + } + + if (numberOfSuccesses > populationSize) { + throw new NumberIsTooLargeException(LocalizedFormats.NUMBER_OF_SUCCESS_LARGER_THAN_POPULATION_SIZE, + numberOfSuccesses, populationSize, true); + } + if (sampleSize > populationSize) { + throw new NumberIsTooLargeException(LocalizedFormats.SAMPLE_SIZE_LARGER_THAN_POPULATION_SIZE, + sampleSize, populationSize, true); + } + + this.numberOfSuccesses = numberOfSuccesses; + this.populationSize = populationSize; + this.sampleSize = sampleSize; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(int x) { + double ret; + + int[] domain = getDomain(populationSize, numberOfSuccesses, sampleSize); + if (x < domain[0]) { + ret = 0.0; + } else if (x >= domain[1]) { + ret = 1.0; + } else { + ret = innerCumulativeProbability(domain[0], x, 1); + } + + return ret; + } + + /** + * Return the domain for the given hypergeometric distribution parameters. + * + * @param n Population size. + * @param m Number of successes in the population. + * @param k Sample size. + * @return a two element array containing the lower and upper bounds of the + * hypergeometric distribution. + */ + private int[] getDomain(int n, int m, int k) { + return new int[] { getLowerDomain(n, m, k), getUpperDomain(m, k) }; + } + + /** + * Return the lowest domain value for the given hypergeometric distribution + * parameters. + * + * @param n Population size. + * @param m Number of successes in the population. + * @param k Sample size. + * @return the lowest domain value of the hypergeometric distribution. + */ + private int getLowerDomain(int n, int m, int k) { + return FastMath.max(0, m - (n - k)); + } + + /** + * Access the number of successes. + * + * @return the number of successes. + */ + public int getNumberOfSuccesses() { + return numberOfSuccesses; + } + + /** + * Access the population size. + * + * @return the population size. + */ + public int getPopulationSize() { + return populationSize; + } + + /** + * Access the sample size. + * + * @return the sample size. + */ + public int getSampleSize() { + return sampleSize; + } + + /** + * Return the highest domain value for the given hypergeometric distribution + * parameters. + * + * @param m Number of successes in the population. + * @param k Sample size. + * @return the highest domain value of the hypergeometric distribution. + */ + private int getUpperDomain(int m, int k) { + return FastMath.min(k, m); + } + + /** {@inheritDoc} */ + public double probability(int x) { + final double logProbability = logProbability(x); + return logProbability == Double.NEGATIVE_INFINITY ? 0 : FastMath.exp(logProbability); + } + + /** {@inheritDoc} */ + @Override + public double logProbability(int x) { + double ret; + + int[] domain = getDomain(populationSize, numberOfSuccesses, sampleSize); + if (x < domain[0] || x > domain[1]) { + ret = Double.NEGATIVE_INFINITY; + } else { + double p = (double) sampleSize / (double) populationSize; + double q = (double) (populationSize - sampleSize) / (double) populationSize; + double p1 = SaddlePointExpansion.logBinomialProbability(x, + numberOfSuccesses, p, q); + double p2 = + SaddlePointExpansion.logBinomialProbability(sampleSize - x, + populationSize - numberOfSuccesses, p, q); + double p3 = + SaddlePointExpansion.logBinomialProbability(sampleSize, populationSize, p, q); + ret = p1 + p2 - p3; + } + + return ret; + } + + /** + * For this distribution, {@code X}, this method returns {@code P(X >= x)}. + * + * @param x Value at which the CDF is evaluated. + * @return the upper tail CDF for this distribution. + * @since 1.1 + */ + public double upperCumulativeProbability(int x) { + double ret; + + final int[] domain = getDomain(populationSize, numberOfSuccesses, sampleSize); + if (x <= domain[0]) { + ret = 1.0; + } else if (x > domain[1]) { + ret = 0.0; + } else { + ret = innerCumulativeProbability(domain[1], x, -1); + } + + return ret; + } + + /** + * For this distribution, {@code X}, this method returns + * {@code P(x0 <= X <= x1)}. + * This probability is computed by summing the point probabilities for the + * values {@code x0, x0 + 1, x0 + 2, ..., x1}, in the order directed by + * {@code dx}. + * + * @param x0 Inclusive lower bound. + * @param x1 Inclusive upper bound. + * @param dx Direction of summation (1 indicates summing from x0 to x1, and + * 0 indicates summing from x1 to x0). + * @return {@code P(x0 <= X <= x1)}. + */ + private double innerCumulativeProbability(int x0, int x1, int dx) { + double ret = probability(x0); + while (x0 != x1) { + x0 += dx; + ret += probability(x0); + } + return ret; + } + + /** + * {@inheritDoc} + * + * For population size {@code N}, number of successes {@code m}, and sample + * size {@code n}, the mean is {@code n * m / N}. + */ + public double getNumericalMean() { + return getSampleSize() * (getNumberOfSuccesses() / (double) getPopulationSize()); + } + + /** + * {@inheritDoc} + * + * For population size {@code N}, number of successes {@code m}, and sample + * size {@code n}, the variance is + * {@code [n * m * (N - n) * (N - m)] / [N^2 * (N - 1)]}. + */ + public double getNumericalVariance() { + if (!numericalVarianceIsCalculated) { + numericalVariance = calculateNumericalVariance(); + numericalVarianceIsCalculated = true; + } + return numericalVariance; + } + + /** + * Used by {@link #getNumericalVariance()}. + * + * @return the variance of this distribution + */ + protected double calculateNumericalVariance() { + final double N = getPopulationSize(); + final double m = getNumberOfSuccesses(); + final double n = getSampleSize(); + return (n * m * (N - n) * (N - m)) / (N * N * (N - 1)); + } + + /** + * {@inheritDoc} + * + * For population size {@code N}, number of successes {@code m}, and sample + * size {@code n}, the lower bound of the support is + * {@code max(0, n + m - N)}. + * + * @return lower bound of the support + */ + public int getSupportLowerBound() { + return FastMath.max(0, + getSampleSize() + getNumberOfSuccesses() - getPopulationSize()); + } + + /** + * {@inheritDoc} + * + * For number of successes {@code m} and sample size {@code n}, the upper + * bound of the support is {@code min(m, n)}. + * + * @return upper bound of the support + */ + public int getSupportUpperBound() { + return FastMath.min(getNumberOfSuccesses(), getSampleSize()); + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/IntegerDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/IntegerDistribution.java new file mode 100644 index 0000000..5194aa7 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/IntegerDistribution.java @@ -0,0 +1,184 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; + +/** + * Interface for distributions on the integers. + * + */ +public interface IntegerDistribution { + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(X = x)}. In other + * words, this method represents the probability mass function (PMF) + * for the distribution. + * + * @param x the point at which the PMF is evaluated + * @return the value of the probability mass function at {@code x} + */ + double probability(int x); + + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(X <= x)}. In other + * words, this method represents the (cumulative) distribution function + * (CDF) for this distribution. + * + * @param x the point at which the CDF is evaluated + * @return the probability that a random variable with this + * distribution takes a value less than or equal to {@code x} + */ + double cumulativeProbability(int x); + + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(x0 < X <= x1)}. + * + * @param x0 the exclusive lower bound + * @param x1 the inclusive upper bound + * @return the probability that a random variable with this distribution + * will take a value between {@code x0} and {@code x1}, + * excluding the lower and including the upper endpoint + * @throws NumberIsTooLargeException if {@code x0 > x1} + */ + double cumulativeProbability(int x0, int x1) throws NumberIsTooLargeException; + + /** + * Computes the quantile function of this distribution. + * For a random variable {@code X} distributed according to this distribution, + * the returned value is + *

    + *
  • inf{x in Z | P(X<=x) >= p} for {@code 0 < p <= 1},
    • + *
  • inf{x in Z | P(X<=x) > 0} for {@code p = 0}.
    • + *
+ * If the result exceeds the range of the data type {@code int}, + * then {@code Integer.MIN_VALUE} or {@code Integer.MAX_VALUE} is returned. + * + * @param p the cumulative probability + * @return the smallest {@code p}-quantile of this distribution + * (largest 0-quantile for {@code p = 0}) + * @throws OutOfRangeException if {@code p < 0} or {@code p > 1} + */ + int inverseCumulativeProbability(double p) throws OutOfRangeException; + + /** + * Use this method to get the numerical value of the mean of this + * distribution. + * + * @return the mean or {@code Double.NaN} if it is not defined + */ + double getNumericalMean(); + + /** + * Use this method to get the numerical value of the variance of this + * distribution. + * + * @return the variance (possibly {@code Double.POSITIVE_INFINITY} or + * {@code Double.NaN} if it is not defined) + */ + double getNumericalVariance(); + + /** + * Access the lower bound of the support. This method must return the same + * value as {@code inverseCumulativeProbability(0)}. In other words, this + * method must return + *

inf {x in Z | P(X <= x) > 0}.

+ * + * @return lower bound of the support ({@code Integer.MIN_VALUE} + * for negative infinity) + */ + int getSupportLowerBound(); + + /** + * Access the upper bound of the support. This method must return the same + * value as {@code inverseCumulativeProbability(1)}. In other words, this + * method must return + *

inf {x in R | P(X <= x) = 1}.

+ * + * @return upper bound of the support ({@code Integer.MAX_VALUE} + * for positive infinity) + */ + int getSupportUpperBound(); + + /** + * Use this method to get information about whether the support is + * connected, i.e. whether all integers between the lower and upper bound of + * the support are included in the support. + * + * @return whether the support is connected or not + */ + boolean isSupportConnected(); + + /** + * Reseed the random generator used to generate samples. + * + * @param seed the new seed + * @since 3.0 + */ + void reseedRandomGenerator(long seed); + + /** + * Generate a random value sampled from this distribution. + * + * @return a random value + * @since 3.0 + */ + int sample(); + + /** + * Generate a random sample from the distribution. + * + * @param sampleSize the number of random values to generate + * @return an array representing the random sample + * @throws infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException + * if {@code sampleSize} is not positive + * @since 3.0 + */ + int[] sample(int sampleSize); +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/NormalDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/NormalDistribution.java new file mode 100644 index 0000000..460a813 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/NormalDistribution.java @@ -0,0 +1,340 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Erf; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the normal (gaussian) distribution. + * + * @see Normal distribution (Wikipedia) + * @see Normal distribution (MathWorld) + */ +public class NormalDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier. */ + private static final long serialVersionUID = 8589540077390120676L; + /** √(2) */ + private static final double SQRT2 = FastMath.sqrt(2.0); + /** Mean of this distribution. */ + private final double mean; + /** Standard deviation of this distribution. */ + private final double standardDeviation; + /** The value of {@code log(sd) + 0.5*log(2*pi)} stored for faster computation. */ + private final double logStandardDeviationPlusHalfLog2Pi; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + + /** + * Create a normal distribution with mean equal to zero and standard + * deviation equal to one. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + */ + public NormalDistribution() { + this(0, 1); + } + + /** + * Create a normal distribution using the given mean and standard deviation. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + */ + public NormalDistribution(double mean, double sd) + throws NotStrictlyPositiveException { + this(mean, sd, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a normal distribution using the given mean, standard deviation and + * inverse cumulative distribution accuracy. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @param inverseCumAccuracy Inverse cumulative probability accuracy. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + * @since 2.1 + */ + public NormalDistribution(double mean, double sd, double inverseCumAccuracy) + throws NotStrictlyPositiveException { + this(new Well19937c(), mean, sd, inverseCumAccuracy); + } + + /** + * Creates a normal distribution. + * + * @param rng Random number generator. + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + * @since 3.3 + */ + public NormalDistribution(RandomGenerator rng, double mean, double sd) + throws NotStrictlyPositiveException { + this(rng, mean, sd, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a normal distribution. + * + * @param rng Random number generator. + * @param mean Mean for this distribution. + * @param sd Standard deviation for this distribution. + * @param inverseCumAccuracy Inverse cumulative probability accuracy. + * @throws NotStrictlyPositiveException if {@code sd <= 0}. + * @since 3.1 + */ + public NormalDistribution(RandomGenerator rng, + double mean, + double sd, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (sd <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.STANDARD_DEVIATION, sd); + } + + this.mean = mean; + standardDeviation = sd; + logStandardDeviationPlusHalfLog2Pi = FastMath.log(sd) + 0.5 * FastMath.log(2 * FastMath.PI); + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * Access the mean. + * + * @return the mean for this distribution. + */ + public double getMean() { + return mean; + } + + /** + * Access the standard deviation. + * + * @return the standard deviation for this distribution. + */ + public double getStandardDeviation() { + return standardDeviation; + } + + /** {@inheritDoc} */ + public double density(double x) { + return FastMath.exp(logDensity(x)); + } + + /** {@inheritDoc} */ + @Override + public double logDensity(double x) { + final double x0 = x - mean; + final double x1 = x0 / standardDeviation; + return -0.5 * x1 * x1 - logStandardDeviationPlusHalfLog2Pi; + } + + /** + * {@inheritDoc} + * + * If {@code x} is more than 40 standard deviations from the mean, 0 or 1 + * is returned, as in these cases the actual value is within + * {@code Double.MIN_VALUE} of 0 or 1. + */ + public double cumulativeProbability(double x) { + final double dev = x - mean; + if (FastMath.abs(dev) > 40 * standardDeviation) { + return dev < 0 ? 0.0d : 1.0d; + } + return 0.5 * Erf.erfc(-dev / (standardDeviation * SQRT2)); + } + + /** {@inheritDoc} + * @since 3.2 + */ + @Override + public double inverseCumulativeProbability(final double p) throws OutOfRangeException { + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0, 1); + } + return mean + standardDeviation * SQRT2 * Erf.erfInv(2 * p - 1); + } + + /** + * {@inheritDoc} + * + * @deprecated See {@link RealDistribution#cumulativeProbability(double,double)} + */ + @Override@Deprecated + public double cumulativeProbability(double x0, double x1) + throws NumberIsTooLargeException { + return probability(x0, x1); + } + + /** {@inheritDoc} */ + @Override + public double probability(double x0, + double x1) + throws NumberIsTooLargeException { + if (x0 > x1) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_ENDPOINT_ABOVE_UPPER_ENDPOINT, + x0, x1, true); + } + final double denom = standardDeviation * SQRT2; + final double v0 = (x0 - mean) / denom; + final double v1 = (x1 - mean) / denom; + return 0.5 * Erf.erf(v0, v1); + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For mean parameter {@code mu}, the mean is {@code mu}. + */ + public double getNumericalMean() { + return getMean(); + } + + /** + * {@inheritDoc} + * + * For standard deviation parameter {@code s}, the variance is {@code s^2}. + */ + public double getNumericalVariance() { + final double s = getStandardDeviation(); + return s * s; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always negative infinity + * no matter the parameters. + * + * @return lower bound of the support (always + * {@code Double.NEGATIVE_INFINITY}) + */ + public double getSupportLowerBound() { + return Double.NEGATIVE_INFINITY; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity + * no matter the parameters. + * + * @return upper bound of the support (always + * {@code Double.POSITIVE_INFINITY}) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** {@inheritDoc} */ + @Override + public double sample() { + return standardDeviation * random.nextGaussian() + mean; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/PascalDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/PascalDistribution.java new file mode 100644 index 0000000..0db3003 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/PascalDistribution.java @@ -0,0 +1,277 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Beta; +import infodynamics.utils.commonsmath3.util.CombinatoricsUtils; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + *

+ * Implementation of the Pascal distribution. The Pascal distribution is a + * special case of the Negative Binomial distribution where the number of + * successes parameter is an integer. + *

+ *

+ * There are various ways to express the probability mass and distribution + * functions for the Pascal distribution. The present implementation represents + * the distribution of the number of failures before {@code r} successes occur. + * This is the convention adopted in e.g. + * MathWorld, + * but not in + * Wikipedia. + *

+ *

+ * For a random variable {@code X} whose values are distributed according to this + * distribution, the probability mass function is given by
+ * {@code P(X = k) = C(k + r - 1, r - 1) * p^r * (1 - p)^k,}
+ * where {@code r} is the number of successes, {@code p} is the probability of + * success, and {@code X} is the total number of failures. {@code C(n, k)} is + * the binomial coefficient ({@code n} choose {@code k}). The mean and variance + * of {@code X} are
+ * {@code E(X) = (1 - p) * r / p, var(X) = (1 - p) * r / p^2.}
+ * Finally, the cumulative distribution function is given by
+ * {@code P(X <= k) = I(p, r, k + 1)}, + * where I is the regularized incomplete Beta function. + *

+ * + * @see + * Negative binomial distribution (Wikipedia) + * @see + * Negative binomial distribution (MathWorld) + * @since 1.2 (changed to concrete class in 3.0) + */ +public class PascalDistribution extends AbstractIntegerDistribution { + /** Serializable version identifier. */ + private static final long serialVersionUID = 6751309484392813623L; + /** The number of successes. */ + private final int numberOfSuccesses; + /** The probability of success. */ + private final double probabilityOfSuccess; + /** The value of {@code log(p)}, where {@code p} is the probability of success, + * stored for faster computation. */ + private final double logProbabilityOfSuccess; + /** The value of {@code log(1-p)}, where {@code p} is the probability of success, + * stored for faster computation. */ + private final double log1mProbabilityOfSuccess; + + /** + * Create a Pascal distribution with the given number of successes and + * probability of success. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param r Number of successes. + * @param p Probability of success. + * @throws NotStrictlyPositiveException if the number of successes is not positive + * @throws OutOfRangeException if the probability of success is not in the + * range {@code [0, 1]}. + */ + public PascalDistribution(int r, double p) + throws NotStrictlyPositiveException, OutOfRangeException { + this(new Well19937c(), r, p); + } + + /** + * Create a Pascal distribution with the given number of successes and + * probability of success. + * + * @param rng Random number generator. + * @param r Number of successes. + * @param p Probability of success. + * @throws NotStrictlyPositiveException if the number of successes is not positive + * @throws OutOfRangeException if the probability of success is not in the + * range {@code [0, 1]}. + * @since 3.1 + */ + public PascalDistribution(RandomGenerator rng, + int r, + double p) + throws NotStrictlyPositiveException, OutOfRangeException { + super(rng); + + if (r <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.NUMBER_OF_SUCCESSES, + r); + } + if (p < 0 || p > 1) { + throw new OutOfRangeException(p, 0, 1); + } + + numberOfSuccesses = r; + probabilityOfSuccess = p; + logProbabilityOfSuccess = FastMath.log(p); + log1mProbabilityOfSuccess = FastMath.log1p(-p); + } + + /** + * Access the number of successes for this distribution. + * + * @return the number of successes. + */ + public int getNumberOfSuccesses() { + return numberOfSuccesses; + } + + /** + * Access the probability of success for this distribution. + * + * @return the probability of success. + */ + public double getProbabilityOfSuccess() { + return probabilityOfSuccess; + } + + /** {@inheritDoc} */ + public double probability(int x) { + double ret; + if (x < 0) { + ret = 0.0; + } else { + ret = CombinatoricsUtils.binomialCoefficientDouble(x + + numberOfSuccesses - 1, numberOfSuccesses - 1) * + FastMath.pow(probabilityOfSuccess, numberOfSuccesses) * + FastMath.pow(1.0 - probabilityOfSuccess, x); + } + return ret; + } + + /** {@inheritDoc} */ + @Override + public double logProbability(int x) { + double ret; + if (x < 0) { + ret = Double.NEGATIVE_INFINITY; + } else { + ret = CombinatoricsUtils.binomialCoefficientLog(x + + numberOfSuccesses - 1, numberOfSuccesses - 1) + + logProbabilityOfSuccess * numberOfSuccesses + + log1mProbabilityOfSuccess * x; + } + return ret; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(int x) { + double ret; + if (x < 0) { + ret = 0.0; + } else { + ret = Beta.regularizedBeta(probabilityOfSuccess, + numberOfSuccesses, x + 1.0); + } + return ret; + } + + /** + * {@inheritDoc} + * + * For number of successes {@code r} and probability of success {@code p}, + * the mean is {@code r * (1 - p) / p}. + */ + public double getNumericalMean() { + final double p = getProbabilityOfSuccess(); + final double r = getNumberOfSuccesses(); + return (r * (1 - p)) / p; + } + + /** + * {@inheritDoc} + * + * For number of successes {@code r} and probability of success {@code p}, + * the variance is {@code r * (1 - p) / p^2}. + */ + public double getNumericalVariance() { + final double p = getProbabilityOfSuccess(); + final double r = getNumberOfSuccesses(); + return r * (1 - p) / (p * p); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the parameters. + * + * @return lower bound of the support (always 0) + */ + public int getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity no matter the + * parameters. Positive infinity is symbolized by {@code Integer.MAX_VALUE}. + * + * @return upper bound of the support (always {@code Integer.MAX_VALUE} + * for positive infinity) + */ + public int getSupportUpperBound() { + return Integer.MAX_VALUE; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/PoissonDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/PoissonDistribution.java new file mode 100644 index 0000000..45602e9 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/PoissonDistribution.java @@ -0,0 +1,424 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.CombinatoricsUtils; +import infodynamics.utils.commonsmath3.util.FastMath; +import infodynamics.utils.commonsmath3.util.MathUtils; + +/** + * Implementation of the Poisson distribution. + * + * @see Poisson distribution (Wikipedia) + * @see Poisson distribution (MathWorld) + */ +public class PoissonDistribution extends AbstractIntegerDistribution { + /** + * Default maximum number of iterations for cumulative probability calculations. + * @since 2.1 + */ + public static final int DEFAULT_MAX_ITERATIONS = 10000000; + /** + * Default convergence criterion. + * @since 2.1 + */ + public static final double DEFAULT_EPSILON = 1e-12; + /** Serializable version identifier. */ + private static final long serialVersionUID = -3349935121172596109L; + /** Distribution used to compute normal approximation. */ + private final NormalDistribution normal; + /** Distribution needed for the {@link #sample()} method. */ + private final ExponentialDistribution exponential; + /** Mean of the distribution. */ + private final double mean; + + /** + * Maximum number of iterations for cumulative probability. Cumulative + * probabilities are estimated using either Lanczos series approximation + * of {@link Gamma#regularizedGammaP(double, double, double, int)} + * or continued fraction approximation of + * {@link Gamma#regularizedGammaQ(double, double, double, int)}. + */ + private final int maxIterations; + + /** Convergence criterion for cumulative probability. */ + private final double epsilon; + + /** + * Creates a new Poisson distribution with specified mean. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param p the Poisson mean + * @throws NotStrictlyPositiveException if {@code p <= 0}. + */ + public PoissonDistribution(double p) throws NotStrictlyPositiveException { + this(p, DEFAULT_EPSILON, DEFAULT_MAX_ITERATIONS); + } + + /** + * Creates a new Poisson distribution with specified mean, convergence + * criterion and maximum number of iterations. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param p Poisson mean. + * @param epsilon Convergence criterion for cumulative probabilities. + * @param maxIterations the maximum number of iterations for cumulative + * probabilities. + * @throws NotStrictlyPositiveException if {@code p <= 0}. + * @since 2.1 + */ + public PoissonDistribution(double p, double epsilon, int maxIterations) + throws NotStrictlyPositiveException { + this(new Well19937c(), p, epsilon, maxIterations); + } + + /** + * Creates a new Poisson distribution with specified mean, convergence + * criterion and maximum number of iterations. + * + * @param rng Random number generator. + * @param p Poisson mean. + * @param epsilon Convergence criterion for cumulative probabilities. + * @param maxIterations the maximum number of iterations for cumulative + * probabilities. + * @throws NotStrictlyPositiveException if {@code p <= 0}. + * @since 3.1 + */ + public PoissonDistribution(RandomGenerator rng, + double p, + double epsilon, + int maxIterations) + throws NotStrictlyPositiveException { + super(rng); + + if (p <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.MEAN, p); + } + mean = p; + this.epsilon = epsilon; + this.maxIterations = maxIterations; + + // Use the same RNG instance as the parent class. + normal = new NormalDistribution(rng, p, FastMath.sqrt(p), + NormalDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + exponential = new ExponentialDistribution(rng, 1, + ExponentialDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a new Poisson distribution with the specified mean and + * convergence criterion. + * + * @param p Poisson mean. + * @param epsilon Convergence criterion for cumulative probabilities. + * @throws NotStrictlyPositiveException if {@code p <= 0}. + * @since 2.1 + */ + public PoissonDistribution(double p, double epsilon) + throws NotStrictlyPositiveException { + this(p, epsilon, DEFAULT_MAX_ITERATIONS); + } + + /** + * Creates a new Poisson distribution with the specified mean and maximum + * number of iterations. + * + * @param p Poisson mean. + * @param maxIterations Maximum number of iterations for cumulative + * probabilities. + * @since 2.1 + */ + public PoissonDistribution(double p, int maxIterations) { + this(p, DEFAULT_EPSILON, maxIterations); + } + + /** + * Get the mean for the distribution. + * + * @return the mean for the distribution. + */ + public double getMean() { + return mean; + } + + /** {@inheritDoc} */ + public double probability(int x) { + final double logProbability = logProbability(x); + return logProbability == Double.NEGATIVE_INFINITY ? 0 : FastMath.exp(logProbability); + } + + /** {@inheritDoc} */ + @Override + public double logProbability(int x) { + double ret; + if (x < 0 || x == Integer.MAX_VALUE) { + ret = Double.NEGATIVE_INFINITY; + } else if (x == 0) { + ret = -mean; + } else { + ret = -SaddlePointExpansion.getStirlingError(x) - + SaddlePointExpansion.getDeviancePart(x, mean) - + 0.5 * FastMath.log(MathUtils.TWO_PI) - 0.5 * FastMath.log(x); + } + return ret; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(int x) { + if (x < 0) { + return 0; + } + if (x == Integer.MAX_VALUE) { + return 1; + } + return Gamma.regularizedGammaQ((double) x + 1, mean, epsilon, + maxIterations); + } + + /** + * Calculates the Poisson distribution function using a normal + * approximation. The {@code N(mean, sqrt(mean))} distribution is used + * to approximate the Poisson distribution. The computation uses + * "half-correction" (evaluating the normal distribution function at + * {@code x + 0.5}). + * + * @param x Upper bound, inclusive. + * @return the distribution function value calculated using a normal + * approximation. + */ + public double normalApproximateProbability(int x) { + // calculate the probability using half-correction + return normal.cumulativeProbability(x + 0.5); + } + + /** + * {@inheritDoc} + * + * For mean parameter {@code p}, the mean is {@code p}. + */ + public double getNumericalMean() { + return getMean(); + } + + /** + * {@inheritDoc} + * + * For mean parameter {@code p}, the variance is {@code p}. + */ + public double getNumericalVariance() { + return getMean(); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the mean parameter. + * + * @return lower bound of the support (always 0) + */ + public int getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is positive infinity, + * regardless of the parameter values. There is no integer infinity, + * so this method returns {@code Integer.MAX_VALUE}. + * + * @return upper bound of the support (always {@code Integer.MAX_VALUE} for + * positive infinity) + */ + public int getSupportUpperBound() { + return Integer.MAX_VALUE; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: + *

    + *
  • For small means, uses simulation of a Poisson process + * using Uniform deviates, as described + * here. + * The Poisson process (and hence value returned) is bounded by 1000 * mean. + *
    • + *
  • For large means, uses the rejection algorithm described in + *
    + * Devroye, Luc. (1981).The Computer Generation of Poisson Random Variables
    + * Computing vol. 26 pp. 197-207.
    + *
    + *
    • + *
+ *

+ * + * @return a random value. + * @since 2.2 + */ + @Override + public int sample() { + return (int) FastMath.min(nextPoisson(mean), Integer.MAX_VALUE); + } + + /** + * @param meanPoisson Mean of the Poisson distribution. + * @return the next sample. + */ + private long nextPoisson(double meanPoisson) { + final double pivot = 40.0d; + if (meanPoisson < pivot) { + double p = FastMath.exp(-meanPoisson); + long n = 0; + double r = 1.0d; + double rnd = 1.0d; + + while (n < 1000 * meanPoisson) { + rnd = random.nextDouble(); + r *= rnd; + if (r >= p) { + n++; + } else { + return n; + } + } + return n; + } else { + final double lambda = FastMath.floor(meanPoisson); + final double lambdaFractional = meanPoisson - lambda; + final double logLambda = FastMath.log(lambda); + final double logLambdaFactorial = CombinatoricsUtils.factorialLog((int) lambda); + final long y2 = lambdaFractional < Double.MIN_VALUE ? 0 : nextPoisson(lambdaFractional); + final double delta = FastMath.sqrt(lambda * FastMath.log(32 * lambda / FastMath.PI + 1)); + final double halfDelta = delta / 2; + final double twolpd = 2 * lambda + delta; + final double a1 = FastMath.sqrt(FastMath.PI * twolpd) * FastMath.exp(1 / (8 * lambda)); + final double a2 = (twolpd / delta) * FastMath.exp(-delta * (1 + delta) / twolpd); + final double aSum = a1 + a2 + 1; + final double p1 = a1 / aSum; + final double p2 = a2 / aSum; + final double c1 = 1 / (8 * lambda); + + double x = 0; + double y = 0; + double v = 0; + int a = 0; + double t = 0; + double qr = 0; + double qa = 0; + for (;;) { + final double u = random.nextDouble(); + if (u <= p1) { + final double n = random.nextGaussian(); + x = n * FastMath.sqrt(lambda + halfDelta) - 0.5d; + if (x > delta || x < -lambda) { + continue; + } + y = x < 0 ? FastMath.floor(x) : FastMath.ceil(x); + final double e = exponential.sample(); + v = -e - (n * n / 2) + c1; + } else { + if (u > p1 + p2) { + y = lambda; + break; + } else { + x = delta + (twolpd / delta) * exponential.sample(); + y = FastMath.ceil(x); + v = -exponential.sample() - delta * (x + 1) / twolpd; + } + } + a = x < 0 ? 1 : 0; + t = y * (y + 1) / (2 * lambda); + if (v < -t && a == 0) { + y = lambda + y; + break; + } + qr = t * ((2 * y + 1) / (6 * lambda) - 1); + qa = qr - (t * t) / (3 * (lambda + a * (y + 1))); + if (v < qa) { + y = lambda + y; + break; + } + if (v > qr) { + continue; + } + if (v < y * logLambda - CombinatoricsUtils.factorialLog((int) (y + lambda)) + logLambdaFactorial) { + y = lambda + y; + break; + } + } + return y2 + (long) y; + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/RealDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/RealDistribution.java new file mode 100644 index 0000000..b78bc0c --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/RealDistribution.java @@ -0,0 +1,225 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; + +/** + * Base interface for distributions on the reals. + * + * @since 3.0 + */ +public interface RealDistribution { + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(X = x)}. In other + * words, this method represents the probability mass function (PMF) + * for the distribution. + * + * @param x the point at which the PMF is evaluated + * @return the value of the probability mass function at point {@code x} + */ + double probability(double x); + + /** + * Returns the probability density function (PDF) of this distribution + * evaluated at the specified point {@code x}. In general, the PDF is + * the derivative of the {@link #cumulativeProbability(double) CDF}. + * If the derivative does not exist at {@code x}, then an appropriate + * replacement should be returned, e.g. {@code Double.POSITIVE_INFINITY}, + * {@code Double.NaN}, or the limit inferior or limit superior of the + * difference quotient. + * + * @param x the point at which the PDF is evaluated + * @return the value of the probability density function at point {@code x} + */ + double density(double x); + + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(X <= x)}. In other + * words, this method represents the (cumulative) distribution function + * (CDF) for this distribution. + * + * @param x the point at which the CDF is evaluated + * @return the probability that a random variable with this + * distribution takes a value less than or equal to {@code x} + */ + double cumulativeProbability(double x); + + /** + * For a random variable {@code X} whose values are distributed according + * to this distribution, this method returns {@code P(x0 < X <= x1)}. + * + * @param x0 the exclusive lower bound + * @param x1 the inclusive upper bound + * @return the probability that a random variable with this distribution + * takes a value between {@code x0} and {@code x1}, + * excluding the lower and including the upper endpoint + * @throws NumberIsTooLargeException if {@code x0 > x1} + * + * @deprecated As of 3.1. In 4.0, this method will be renamed + * {@code probability(double x0, double x1)}. + */ + @Deprecated + double cumulativeProbability(double x0, double x1) throws NumberIsTooLargeException; + + /** + * Computes the quantile function of this distribution. For a random + * variable {@code X} distributed according to this distribution, the + * returned value is + *
    + *
  • inf{x in R | P(X<=x) >= p} for {@code 0 < p <= 1},
    • + *
  • inf{x in R | P(X<=x) > 0} for {@code p = 0}.
    • + *
+ * + * @param p the cumulative probability + * @return the smallest {@code p}-quantile of this distribution + * (largest 0-quantile for {@code p = 0}) + * @throws OutOfRangeException if {@code p < 0} or {@code p > 1} + */ + double inverseCumulativeProbability(double p) throws OutOfRangeException; + + /** + * Use this method to get the numerical value of the mean of this + * distribution. + * + * @return the mean or {@code Double.NaN} if it is not defined + */ + double getNumericalMean(); + + /** + * Use this method to get the numerical value of the variance of this + * distribution. + * + * @return the variance (possibly {@code Double.POSITIVE_INFINITY} as + * for certain cases in {@link TDistribution}) or {@code Double.NaN} if it + * is not defined + */ + double getNumericalVariance(); + + /** + * Access the lower bound of the support. This method must return the same + * value as {@code inverseCumulativeProbability(0)}. In other words, this + * method must return + *

inf {x in R | P(X <= x) > 0}.

+ * + * @return lower bound of the support (might be + * {@code Double.NEGATIVE_INFINITY}) + */ + double getSupportLowerBound(); + + /** + * Access the upper bound of the support. This method must return the same + * value as {@code inverseCumulativeProbability(1)}. In other words, this + * method must return + *

inf {x in R | P(X <= x) = 1}.

+ * + * @return upper bound of the support (might be + * {@code Double.POSITIVE_INFINITY}) + */ + double getSupportUpperBound(); + + /** + * Whether or not the lower bound of support is in the domain of the density + * function. Returns true iff {@code getSupporLowerBound()} is finite and + * {@code density(getSupportLowerBound())} returns a non-NaN, non-infinite + * value. + * + * @return true if the lower bound of support is finite and the density + * function returns a non-NaN, non-infinite value there + * @deprecated to be removed in 4.0 + */ + @Deprecated + boolean isSupportLowerBoundInclusive(); + + /** + * Whether or not the upper bound of support is in the domain of the density + * function. Returns true iff {@code getSupportUpperBound()} is finite and + * {@code density(getSupportUpperBound())} returns a non-NaN, non-infinite + * value. + * + * @return true if the upper bound of support is finite and the density + * function returns a non-NaN, non-infinite value there + * @deprecated to be removed in 4.0 + */ + @Deprecated + boolean isSupportUpperBoundInclusive(); + + /** + * Use this method to get information about whether the support is connected, + * i.e. whether all values between the lower and upper bound of the support + * are included in the support. + * + * @return whether the support is connected or not + */ + boolean isSupportConnected(); + + /** + * Reseed the random generator used to generate samples. + * + * @param seed the new seed + */ + void reseedRandomGenerator(long seed); + + /** + * Generate a random value sampled from this distribution. + * + * @return a random value. + */ + double sample(); + + /** + * Generate a random sample from the distribution. + * + * @param sampleSize the number of random values to generate + * @return an array representing the random sample + * @throws infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException + * if {@code sampleSize} is not positive + */ + double[] sample(int sampleSize); +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/SaddlePointExpansion.java b/java/source/infodynamics/utils/commonsmath3/distribution/SaddlePointExpansion.java new file mode 100644 index 0000000..3036915 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/SaddlePointExpansion.java @@ -0,0 +1,229 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.FastMath; +import infodynamics.utils.commonsmath3.util.MathUtils; + +/** + *

+ * Utility class used by various distributions to accurately compute their + * respective probability mass functions. The implementation for this class is + * based on the Catherine Loader's dbinom routines. + *

+ *

+ * This class is not intended to be called directly. + *

+ *

+ * References: + *

    + *
  1. Catherine Loader (2000). "Fast and Accurate Computation of Binomial + * Probabilities.". + * http://www.herine.net/stat/papers/dbinom.pdf
    2. + *
+ *

+ * + * @since 2.1 + */ +final class SaddlePointExpansion { + + /** 1/2 * log(2 π). */ + private static final double HALF_LOG_2_PI = 0.5 * FastMath.log(MathUtils.TWO_PI); + + /** exact Stirling expansion error for certain values. */ + private static final double[] EXACT_STIRLING_ERRORS = { 0.0, /* 0.0 */ + 0.1534264097200273452913848, /* 0.5 */ + 0.0810614667953272582196702, /* 1.0 */ + 0.0548141210519176538961390, /* 1.5 */ + 0.0413406959554092940938221, /* 2.0 */ + 0.03316287351993628748511048, /* 2.5 */ + 0.02767792568499833914878929, /* 3.0 */ + 0.02374616365629749597132920, /* 3.5 */ + 0.02079067210376509311152277, /* 4.0 */ + 0.01848845053267318523077934, /* 4.5 */ + 0.01664469118982119216319487, /* 5.0 */ + 0.01513497322191737887351255, /* 5.5 */ + 0.01387612882307074799874573, /* 6.0 */ + 0.01281046524292022692424986, /* 6.5 */ + 0.01189670994589177009505572, /* 7.0 */ + 0.01110455975820691732662991, /* 7.5 */ + 0.010411265261972096497478567, /* 8.0 */ + 0.009799416126158803298389475, /* 8.5 */ + 0.009255462182712732917728637, /* 9.0 */ + 0.008768700134139385462952823, /* 9.5 */ + 0.008330563433362871256469318, /* 10.0 */ + 0.007934114564314020547248100, /* 10.5 */ + 0.007573675487951840794972024, /* 11.0 */ + 0.007244554301320383179543912, /* 11.5 */ + 0.006942840107209529865664152, /* 12.0 */ + 0.006665247032707682442354394, /* 12.5 */ + 0.006408994188004207068439631, /* 13.0 */ + 0.006171712263039457647532867, /* 13.5 */ + 0.005951370112758847735624416, /* 14.0 */ + 0.005746216513010115682023589, /* 14.5 */ + 0.005554733551962801371038690 /* 15.0 */ + }; + + /** + * Default constructor. + */ + private SaddlePointExpansion() { + super(); + } + + /** + * Compute the error of Stirling's series at the given value. + *

+ * References: + *

    + *
  1. Eric W. Weisstein. "Stirling's Series." From MathWorld--A Wolfram Web + * Resource. + * http://mathworld.wolfram.com/StirlingsSeries.html
    2. + *
+ *

+ * + * @param z the value. + * @return the Striling's series error. + */ + static double getStirlingError(double z) { + double ret; + if (z < 15.0) { + double z2 = 2.0 * z; + if (FastMath.floor(z2) == z2) { + ret = EXACT_STIRLING_ERRORS[(int) z2]; + } else { + ret = Gamma.logGamma(z + 1.0) - (z + 0.5) * FastMath.log(z) + + z - HALF_LOG_2_PI; + } + } else { + double z2 = z * z; + ret = (0.083333333333333333333 - + (0.00277777777777777777778 - + (0.00079365079365079365079365 - + (0.000595238095238095238095238 - + 0.0008417508417508417508417508 / + z2) / z2) / z2) / z2) / z; + } + return ret; + } + + /** + * A part of the deviance portion of the saddle point approximation. + *

+ * References: + *

    + *
  1. Catherine Loader (2000). "Fast and Accurate Computation of Binomial + * Probabilities.". + * http://www.herine.net/stat/papers/dbinom.pdf
    2. + *
+ *

+ * + * @param x the x value. + * @param mu the average. + * @return a part of the deviance. + */ + static double getDeviancePart(double x, double mu) { + double ret; + if (FastMath.abs(x - mu) < 0.1 * (x + mu)) { + double d = x - mu; + double v = d / (x + mu); + double s1 = v * d; + double s = Double.NaN; + double ej = 2.0 * x * v; + v *= v; + int j = 1; + while (s1 != s) { + s = s1; + ej *= v; + s1 = s + ej / ((j * 2) + 1); + ++j; + } + ret = s1; + } else { + ret = x * FastMath.log(x / mu) + mu - x; + } + return ret; + } + + /** + * Compute the logarithm of the PMF for a binomial distribution + * using the saddle point expansion. + * + * @param x the value at which the probability is evaluated. + * @param n the number of trials. + * @param p the probability of success. + * @param q the probability of failure (1 - p). + * @return log(p(x)). + */ + static double logBinomialProbability(int x, int n, double p, double q) { + double ret; + if (x == 0) { + if (p < 0.1) { + ret = -getDeviancePart(n, n * q) - n * p; + } else { + ret = n * FastMath.log(q); + } + } else if (x == n) { + if (q < 0.1) { + ret = -getDeviancePart(n, n * p) - n * q; + } else { + ret = n * FastMath.log(p); + } + } else { + ret = getStirlingError(n) - getStirlingError(x) - + getStirlingError(n - x) - getDeviancePart(x, n * p) - + getDeviancePart(n - x, n * q); + double f = (MathUtils.TWO_PI * x * (n - x)) / n; + ret = -0.5 * FastMath.log(f) + ret; + } + return ret; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/TDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/TDistribution.java new file mode 100644 index 0000000..f0d905a --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/TDistribution.java @@ -0,0 +1,301 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Beta; +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of Student's t-distribution. + * + * @see "Student's t-distribution (Wikipedia)" + * @see "Student's t-distribution (MathWorld)" + */ +public class TDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier */ + private static final long serialVersionUID = -5852615386664158222L; + /** The degrees of freedom. */ + private final double degreesOfFreedom; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + /** Static computation factor based on degreesOfFreedom. */ + private final double factor; + + /** + * Create a t distribution using the given degrees of freedom. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param degreesOfFreedom Degrees of freedom. + * @throws NotStrictlyPositiveException if {@code degreesOfFreedom <= 0} + */ + public TDistribution(double degreesOfFreedom) + throws NotStrictlyPositiveException { + this(degreesOfFreedom, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a t distribution using the given degrees of freedom and the + * specified inverse cumulative probability absolute accuracy. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param degreesOfFreedom Degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code degreesOfFreedom <= 0} + * @since 2.1 + */ + public TDistribution(double degreesOfFreedom, double inverseCumAccuracy) + throws NotStrictlyPositiveException { + this(new Well19937c(), degreesOfFreedom, inverseCumAccuracy); + } + + /** + * Creates a t distribution. + * + * @param rng Random number generator. + * @param degreesOfFreedom Degrees of freedom. + * @throws NotStrictlyPositiveException if {@code degreesOfFreedom <= 0} + * @since 3.3 + */ + public TDistribution(RandomGenerator rng, double degreesOfFreedom) + throws NotStrictlyPositiveException { + this(rng, degreesOfFreedom, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a t distribution. + * + * @param rng Random number generator. + * @param degreesOfFreedom Degrees of freedom. + * @param inverseCumAccuracy the maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code degreesOfFreedom <= 0} + * @since 3.1 + */ + public TDistribution(RandomGenerator rng, + double degreesOfFreedom, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (degreesOfFreedom <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.DEGREES_OF_FREEDOM, + degreesOfFreedom); + } + this.degreesOfFreedom = degreesOfFreedom; + solverAbsoluteAccuracy = inverseCumAccuracy; + + final double n = degreesOfFreedom; + final double nPlus1Over2 = (n + 1) / 2; + factor = Gamma.logGamma(nPlus1Over2) - + 0.5 * (FastMath.log(FastMath.PI) + FastMath.log(n)) - + Gamma.logGamma(n / 2); + } + + /** + * Access the degrees of freedom. + * + * @return the degrees of freedom. + */ + public double getDegreesOfFreedom() { + return degreesOfFreedom; + } + + /** {@inheritDoc} */ + public double density(double x) { + return FastMath.exp(logDensity(x)); + } + + /** {@inheritDoc} */ + @Override + public double logDensity(double x) { + final double n = degreesOfFreedom; + final double nPlus1Over2 = (n + 1) / 2; + return factor - nPlus1Over2 * FastMath.log(1 + x * x / n); + } + + /** {@inheritDoc} */ + public double cumulativeProbability(double x) { + double ret; + if (x == 0) { + ret = 0.5; + } else { + double t = + Beta.regularizedBeta( + degreesOfFreedom / (degreesOfFreedom + (x * x)), + 0.5 * degreesOfFreedom, + 0.5); + if (x < 0.0) { + ret = 0.5 * t; + } else { + ret = 1.0 - 0.5 * t; + } + } + + return ret; + } + + /** {@inheritDoc} */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * For degrees of freedom parameter {@code df}, the mean is + *

    + *
  • if {@code df > 1} then {@code 0},
    • + *
  • else undefined ({@code Double.NaN}).
    • + *
+ */ + public double getNumericalMean() { + final double df = getDegreesOfFreedom(); + + if (df > 1) { + return 0; + } + + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * For degrees of freedom parameter {@code df}, the variance is + *
    + *
  • if {@code df > 2} then {@code df / (df - 2)},
    • + *
  • if {@code 1 < df <= 2} then positive infinity + * ({@code Double.POSITIVE_INFINITY}),
    • + *
  • else undefined ({@code Double.NaN}).
    • + *
+ */ + public double getNumericalVariance() { + final double df = getDegreesOfFreedom(); + + if (df > 2) { + return df / (df - 2); + } + + if (df > 1 && df <= 2) { + return Double.POSITIVE_INFINITY; + } + + return Double.NaN; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always negative infinity no matter the + * parameters. + * + * @return lower bound of the support (always + * {@code Double.NEGATIVE_INFINITY}) + */ + public double getSupportLowerBound() { + return Double.NEGATIVE_INFINITY; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity no matter the + * parameters. + * + * @return upper bound of the support (always + * {@code Double.POSITIVE_INFINITY}) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return false; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/UniformIntegerDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/UniformIntegerDistribution.java new file mode 100644 index 0000000..4f4177c --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/UniformIntegerDistribution.java @@ -0,0 +1,210 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; + +/** + * Implementation of the uniform integer distribution. + * + * @see Uniform distribution (discrete), at Wikipedia + * + * @since 3.0 + */ +public class UniformIntegerDistribution extends AbstractIntegerDistribution { + /** Serializable version identifier. */ + private static final long serialVersionUID = 20120109L; + /** Lower bound (inclusive) of this distribution. */ + private final int lower; + /** Upper bound (inclusive) of this distribution. */ + private final int upper; + + /** + * Creates a new uniform integer distribution using the given lower and + * upper bounds (both inclusive). + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param lower Lower bound (inclusive) of this distribution. + * @param upper Upper bound (inclusive) of this distribution. + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + public UniformIntegerDistribution(int lower, int upper) + throws NumberIsTooLargeException { + this(new Well19937c(), lower, upper); + } + + /** + * Creates a new uniform integer distribution using the given lower and + * upper bounds (both inclusive). + * + * @param rng Random number generator. + * @param lower Lower bound (inclusive) of this distribution. + * @param upper Upper bound (inclusive) of this distribution. + * @throws NumberIsTooLargeException if {@code lower > upper}. + * @since 3.1 + */ + public UniformIntegerDistribution(RandomGenerator rng, + int lower, + int upper) + throws NumberIsTooLargeException { + super(rng); + + if (lower > upper) { + throw new NumberIsTooLargeException( + LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, + lower, upper, true); + } + this.lower = lower; + this.upper = upper; + } + + /** {@inheritDoc} */ + public double probability(int x) { + if (x < lower || x > upper) { + return 0; + } + return 1.0 / (upper - lower + 1); + } + + /** {@inheritDoc} */ + public double cumulativeProbability(int x) { + if (x < lower) { + return 0; + } + if (x > upper) { + return 1; + } + return (x - lower + 1.0) / (upper - lower + 1.0); + } + + /** + * {@inheritDoc} + * + * For lower bound {@code lower} and upper bound {@code upper}, the mean is + * {@code 0.5 * (lower + upper)}. + */ + public double getNumericalMean() { + return 0.5 * (lower + upper); + } + + /** + * {@inheritDoc} + * + * For lower bound {@code lower} and upper bound {@code upper}, and + * {@code n = upper - lower + 1}, the variance is {@code (n^2 - 1) / 12}. + */ + public double getNumericalVariance() { + double n = upper - lower + 1; + return (n * n - 1) / 12.0; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is equal to the lower bound parameter + * of the distribution. + * + * @return lower bound of the support + */ + public int getSupportLowerBound() { + return lower; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is equal to the upper bound parameter + * of the distribution. + * + * @return upper bound of the support + */ + public int getSupportUpperBound() { + return upper; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** {@inheritDoc} */ + @Override + public int sample() { + final int max = (upper - lower) + 1; + if (max <= 0) { + // The range is too wide to fit in a positive int (larger + // than 2^31); as it covers more than half the integer range, + // we use a simple rejection method. + while (true) { + final int r = random.nextInt(); + if (r >= lower && + r <= upper) { + return r; + } + } + } else { + // We can shift the range and directly generate a positive int. + return lower + random.nextInt(max); + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/WeibullDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/WeibullDistribution.java new file mode 100644 index 0000000..a1fba9a --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/WeibullDistribution.java @@ -0,0 +1,382 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.special.Gamma; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the Weibull distribution. This implementation uses the + * two parameter form of the distribution defined by + * + * Weibull Distribution, equations (1) and (2). + * + * @see Weibull distribution (Wikipedia) + * @see Weibull distribution (MathWorld) + * @since 1.1 (changed to concrete class in 3.0) + */ +public class WeibullDistribution extends AbstractRealDistribution { + /** + * Default inverse cumulative probability accuracy. + * @since 2.1 + */ + public static final double DEFAULT_INVERSE_ABSOLUTE_ACCURACY = 1e-9; + /** Serializable version identifier. */ + private static final long serialVersionUID = 8589540077390120676L; + /** The shape parameter. */ + private final double shape; + /** The scale parameter. */ + private final double scale; + /** Inverse cumulative probability accuracy. */ + private final double solverAbsoluteAccuracy; + /** Cached numerical mean */ + private double numericalMean = Double.NaN; + /** Whether or not the numerical mean has been calculated */ + private boolean numericalMeanIsCalculated = false; + /** Cached numerical variance */ + private double numericalVariance = Double.NaN; + /** Whether or not the numerical variance has been calculated */ + private boolean numericalVarianceIsCalculated = false; + + /** + * Create a Weibull distribution with the given shape and scale and a + * location equal to zero. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param alpha Shape parameter. + * @param beta Scale parameter. + * @throws NotStrictlyPositiveException if {@code alpha <= 0} or + * {@code beta <= 0}. + */ + public WeibullDistribution(double alpha, double beta) + throws NotStrictlyPositiveException { + this(alpha, beta, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Create a Weibull distribution with the given shape, scale and inverse + * cumulative probability accuracy and a location equal to zero. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param alpha Shape parameter. + * @param beta Scale parameter. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code alpha <= 0} or + * {@code beta <= 0}. + * @since 2.1 + */ + public WeibullDistribution(double alpha, double beta, + double inverseCumAccuracy) { + this(new Well19937c(), alpha, beta, inverseCumAccuracy); + } + + /** + * Creates a Weibull distribution. + * + * @param rng Random number generator. + * @param alpha Shape parameter. + * @param beta Scale parameter. + * @throws NotStrictlyPositiveException if {@code alpha <= 0} or {@code beta <= 0}. + * @since 3.3 + */ + public WeibullDistribution(RandomGenerator rng, double alpha, double beta) + throws NotStrictlyPositiveException { + this(rng, alpha, beta, DEFAULT_INVERSE_ABSOLUTE_ACCURACY); + } + + /** + * Creates a Weibull distribution. + * + * @param rng Random number generator. + * @param alpha Shape parameter. + * @param beta Scale parameter. + * @param inverseCumAccuracy Maximum absolute error in inverse + * cumulative probability estimates + * (defaults to {@link #DEFAULT_INVERSE_ABSOLUTE_ACCURACY}). + * @throws NotStrictlyPositiveException if {@code alpha <= 0} or {@code beta <= 0}. + * @since 3.1 + */ + public WeibullDistribution(RandomGenerator rng, + double alpha, + double beta, + double inverseCumAccuracy) + throws NotStrictlyPositiveException { + super(rng); + + if (alpha <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.SHAPE, + alpha); + } + if (beta <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.SCALE, + beta); + } + scale = beta; + shape = alpha; + solverAbsoluteAccuracy = inverseCumAccuracy; + } + + /** + * Access the shape parameter, {@code alpha}. + * + * @return the shape parameter, {@code alpha}. + */ + public double getShape() { + return shape; + } + + /** + * Access the scale parameter, {@code beta}. + * + * @return the scale parameter, {@code beta}. + */ + public double getScale() { + return scale; + } + + /** {@inheritDoc} */ + public double density(double x) { + if (x < 0) { + return 0; + } + + final double xscale = x / scale; + final double xscalepow = FastMath.pow(xscale, shape - 1); + + /* + * FastMath.pow(x / scale, shape) = + * FastMath.pow(xscale, shape) = + * FastMath.pow(xscale, shape - 1) * xscale + */ + final double xscalepowshape = xscalepow * xscale; + + return (shape / scale) * xscalepow * FastMath.exp(-xscalepowshape); + } + + /** {@inheritDoc} */ + @Override + public double logDensity(double x) { + if (x < 0) { + return Double.NEGATIVE_INFINITY; + } + + final double xscale = x / scale; + final double logxscalepow = FastMath.log(xscale) * (shape - 1); + + /* + * FastMath.pow(x / scale, shape) = + * FastMath.pow(xscale, shape) = + * FastMath.pow(xscale, shape - 1) * xscale + */ + final double xscalepowshape = FastMath.exp(logxscalepow) * xscale; + + return FastMath.log(shape / scale) + logxscalepow - xscalepowshape; + } + + /** {@inheritDoc} */ + public double cumulativeProbability(double x) { + double ret; + if (x <= 0.0) { + ret = 0.0; + } else { + ret = 1.0 - FastMath.exp(-FastMath.pow(x / scale, shape)); + } + return ret; + } + + /** + * {@inheritDoc} + * + * Returns {@code 0} when {@code p == 0} and + * {@code Double.POSITIVE_INFINITY} when {@code p == 1}. + */ + @Override + public double inverseCumulativeProbability(double p) { + double ret; + if (p < 0.0 || p > 1.0) { + throw new OutOfRangeException(p, 0.0, 1.0); + } else if (p == 0) { + ret = 0.0; + } else if (p == 1) { + ret = Double.POSITIVE_INFINITY; + } else { + ret = scale * FastMath.pow(-FastMath.log1p(-p), 1.0 / shape); + } + return ret; + } + + /** + * Return the absolute accuracy setting of the solver used to estimate + * inverse cumulative probabilities. + * + * @return the solver absolute accuracy. + * @since 2.1 + */ + @Override + protected double getSolverAbsoluteAccuracy() { + return solverAbsoluteAccuracy; + } + + /** + * {@inheritDoc} + * + * The mean is {@code scale * Gamma(1 + (1 / shape))}, where {@code Gamma()} + * is the Gamma-function. + */ + public double getNumericalMean() { + if (!numericalMeanIsCalculated) { + numericalMean = calculateNumericalMean(); + numericalMeanIsCalculated = true; + } + return numericalMean; + } + + /** + * used by {@link #getNumericalMean()} + * + * @return the mean of this distribution + */ + protected double calculateNumericalMean() { + final double sh = getShape(); + final double sc = getScale(); + + return sc * FastMath.exp(Gamma.logGamma(1 + (1 / sh))); + } + + /** + * {@inheritDoc} + * + * The variance is {@code scale^2 * Gamma(1 + (2 / shape)) - mean^2} + * where {@code Gamma()} is the Gamma-function. + */ + public double getNumericalVariance() { + if (!numericalVarianceIsCalculated) { + numericalVariance = calculateNumericalVariance(); + numericalVarianceIsCalculated = true; + } + return numericalVariance; + } + + /** + * used by {@link #getNumericalVariance()} + * + * @return the variance of this distribution + */ + protected double calculateNumericalVariance() { + final double sh = getShape(); + final double sc = getScale(); + final double mn = getNumericalMean(); + + return (sc * sc) * FastMath.exp(Gamma.logGamma(1 + (2 / sh))) - + (mn * mn); + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 0 no matter the parameters. + * + * @return lower bound of the support (always 0) + */ + public double getSupportLowerBound() { + return 0; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is always positive infinity + * no matter the parameters. + * + * @return upper bound of the support (always + * {@code Double.POSITIVE_INFINITY}) + */ + public double getSupportUpperBound() { + return Double.POSITIVE_INFINITY; + } + + /** {@inheritDoc} */ + public boolean isSupportLowerBoundInclusive() { + return true; + } + + /** {@inheritDoc} */ + public boolean isSupportUpperBoundInclusive() { + return false; + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } +} + diff --git a/java/source/infodynamics/utils/commonsmath3/distribution/ZipfDistribution.java b/java/source/infodynamics/utils/commonsmath3/distribution/ZipfDistribution.java new file mode 100644 index 0000000..66b8ddf --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/distribution/ZipfDistribution.java @@ -0,0 +1,517 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.distribution; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.random.RandomGenerator; +import infodynamics.utils.commonsmath3.random.Well19937c; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + * Implementation of the Zipf distribution. + *

+ * Parameters: + * For a random variable {@code X} whose values are distributed according to this + * distribution, the probability mass function is given by + * 

+ *   P(X = k) = H(N,s) * 1 / k^s    for {@code k = 1,2,...,N}.
+ *
+ * {@code H(N,s)} is the normalizing constant + * which corresponds to the generalized harmonic number of order N of s. + *

+ *

    + *
  • {@code N} is the number of elements
    • + *
  • {@code s} is the exponent
    • + *
+ * @see Zipf's law (Wikipedia) + * @see Generalized harmonic numbers + */ +public class ZipfDistribution extends AbstractIntegerDistribution { + /** Serializable version identifier. */ + private static final long serialVersionUID = -140627372283420404L; + /** Number of elements. */ + private final int numberOfElements; + /** Exponent parameter of the distribution. */ + private final double exponent; + /** Cached numerical mean */ + private double numericalMean = Double.NaN; + /** Whether or not the numerical mean has been calculated */ + private boolean numericalMeanIsCalculated = false; + /** Cached numerical variance */ + private double numericalVariance = Double.NaN; + /** Whether or not the numerical variance has been calculated */ + private boolean numericalVarianceIsCalculated = false; + /** The sampler to be used for the sample() method */ + private transient ZipfRejectionInversionSampler sampler; + + /** + * Create a new Zipf distribution with the given number of elements and + * exponent. + *

+ * Note: this constructor will implicitly create an instance of + * {@link Well19937c} as random generator to be used for sampling only (see + * {@link #sample()} and {@link #sample(int)}). In case no sampling is + * needed for the created distribution, it is advised to pass {@code null} + * as random generator via the appropriate constructors to avoid the + * additional initialisation overhead. + * + * @param numberOfElements Number of elements. + * @param exponent Exponent. + * @exception NotStrictlyPositiveException if {@code numberOfElements <= 0} + * or {@code exponent <= 0}. + */ + public ZipfDistribution(final int numberOfElements, final double exponent) { + this(new Well19937c(), numberOfElements, exponent); + } + + /** + * Creates a Zipf distribution. + * + * @param rng Random number generator. + * @param numberOfElements Number of elements. + * @param exponent Exponent. + * @exception NotStrictlyPositiveException if {@code numberOfElements <= 0} + * or {@code exponent <= 0}. + * @since 3.1 + */ + public ZipfDistribution(RandomGenerator rng, + int numberOfElements, + double exponent) + throws NotStrictlyPositiveException { + super(rng); + + if (numberOfElements <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.DIMENSION, + numberOfElements); + } + if (exponent <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.EXPONENT, + exponent); + } + + this.numberOfElements = numberOfElements; + this.exponent = exponent; + } + + /** + * Get the number of elements (e.g. corpus size) for the distribution. + * + * @return the number of elements + */ + public int getNumberOfElements() { + return numberOfElements; + } + + /** + * Get the exponent characterizing the distribution. + * + * @return the exponent + */ + public double getExponent() { + return exponent; + } + + /** {@inheritDoc} */ + public double probability(final int x) { + if (x <= 0 || x > numberOfElements) { + return 0.0; + } + + return (1.0 / FastMath.pow(x, exponent)) / generalizedHarmonic(numberOfElements, exponent); + } + + /** {@inheritDoc} */ + @Override + public double logProbability(int x) { + if (x <= 0 || x > numberOfElements) { + return Double.NEGATIVE_INFINITY; + } + + return -FastMath.log(x) * exponent - FastMath.log(generalizedHarmonic(numberOfElements, exponent)); + } + + /** {@inheritDoc} */ + public double cumulativeProbability(final int x) { + if (x <= 0) { + return 0.0; + } else if (x >= numberOfElements) { + return 1.0; + } + + return generalizedHarmonic(x, exponent) / generalizedHarmonic(numberOfElements, exponent); + } + + /** + * {@inheritDoc} + * + * For number of elements {@code N} and exponent {@code s}, the mean is + * {@code Hs1 / Hs}, where + *

    + *
  • {@code Hs1 = generalizedHarmonic(N, s - 1)},
    • + *
  • {@code Hs = generalizedHarmonic(N, s)}.
    • + *
+ */ + public double getNumericalMean() { + if (!numericalMeanIsCalculated) { + numericalMean = calculateNumericalMean(); + numericalMeanIsCalculated = true; + } + return numericalMean; + } + + /** + * Used by {@link #getNumericalMean()}. + * + * @return the mean of this distribution + */ + protected double calculateNumericalMean() { + final int N = getNumberOfElements(); + final double s = getExponent(); + + final double Hs1 = generalizedHarmonic(N, s - 1); + final double Hs = generalizedHarmonic(N, s); + + return Hs1 / Hs; + } + + /** + * {@inheritDoc} + * + * For number of elements {@code N} and exponent {@code s}, the mean is + * {@code (Hs2 / Hs) - (Hs1^2 / Hs^2)}, where + *
    + *
  • {@code Hs2 = generalizedHarmonic(N, s - 2)},
    • + *
  • {@code Hs1 = generalizedHarmonic(N, s - 1)},
    • + *
  • {@code Hs = generalizedHarmonic(N, s)}.
    • + *
+ */ + public double getNumericalVariance() { + if (!numericalVarianceIsCalculated) { + numericalVariance = calculateNumericalVariance(); + numericalVarianceIsCalculated = true; + } + return numericalVariance; + } + + /** + * Used by {@link #getNumericalVariance()}. + * + * @return the variance of this distribution + */ + protected double calculateNumericalVariance() { + final int N = getNumberOfElements(); + final double s = getExponent(); + + final double Hs2 = generalizedHarmonic(N, s - 2); + final double Hs1 = generalizedHarmonic(N, s - 1); + final double Hs = generalizedHarmonic(N, s); + + return (Hs2 / Hs) - ((Hs1 * Hs1) / (Hs * Hs)); + } + + /** + * Calculates the Nth generalized harmonic number. See + * Harmonic + * Series. + * + * @param n Term in the series to calculate (must be larger than 1) + * @param m Exponent (special case {@code m = 1} is the harmonic series). + * @return the nth generalized harmonic number. + */ + private double generalizedHarmonic(final int n, final double m) { + double value = 0; + for (int k = n; k > 0; --k) { + value += 1.0 / FastMath.pow(k, m); + } + return value; + } + + /** + * {@inheritDoc} + * + * The lower bound of the support is always 1 no matter the parameters. + * + * @return lower bound of the support (always 1) + */ + public int getSupportLowerBound() { + return 1; + } + + /** + * {@inheritDoc} + * + * The upper bound of the support is the number of elements. + * + * @return upper bound of the support + */ + public int getSupportUpperBound() { + return getNumberOfElements(); + } + + /** + * {@inheritDoc} + * + * The support of this distribution is connected. + * + * @return {@code true} + */ + public boolean isSupportConnected() { + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public int sample() { + if (sampler == null) { + sampler = new ZipfRejectionInversionSampler(numberOfElements, exponent); + } + return sampler.sample(random); + } + + /** + * Utility class implementing a rejection inversion sampling method for a discrete, + * bounded Zipf distribution that is based on the method described in + *

+ * Wolfgang Hörmann and Gerhard Derflinger + * "Rejection-inversion to generate variates from monotone discrete distributions." + * ACM Transactions on Modeling and Computer Simulation (TOMACS) 6.3 (1996): 169-184. + *

+ * The paper describes an algorithm for exponents larger than 1 (Algorithm ZRI). + * The original method uses {@code H(x) := (v + x)^(1 - q) / (1 - q)} + * as the integral of the hat function. This function is undefined for + * q = 1, which is the reason for the limitation of the exponent. + * If instead the integral function + * {@code H(x) := ((v + x)^(1 - q) - 1) / (1 - q)} is used, + * for which a meaningful limit exists for q = 1, + * the method works for all positive exponents. + *

+ * The following implementation uses v := 0 and generates integral numbers + * in the range [1, numberOfElements]. This is different to the original method + * where v is defined to be positive and numbers are taken from [0, i_max]. + * This explains why the implementation looks slightly different. + * + * @since 3.6 + */ + static final class ZipfRejectionInversionSampler { + + /** Exponent parameter of the distribution. */ + private final double exponent; + /** Number of elements. */ + private final int numberOfElements; + /** Constant equal to {@code hIntegral(1.5) - 1}. */ + private final double hIntegralX1; + /** Constant equal to {@code hIntegral(numberOfElements + 0.5)}. */ + private final double hIntegralNumberOfElements; + /** Constant equal to {@code 2 - hIntegralInverse(hIntegral(2.5) - h(2)}. */ + private final double s; + + /** Simple constructor. + * @param numberOfElements number of elements + * @param exponent exponent parameter of the distribution + */ + ZipfRejectionInversionSampler(final int numberOfElements, final double exponent) { + this.exponent = exponent; + this.numberOfElements = numberOfElements; + this.hIntegralX1 = hIntegral(1.5) - 1d; + this.hIntegralNumberOfElements = hIntegral(numberOfElements + 0.5); + this.s = 2d - hIntegralInverse(hIntegral(2.5) - h(2)); + } + + /** Generate one integral number in the range [1, numberOfElements]. + * @param random random generator to use + * @return generated integral number in the range [1, numberOfElements] + */ + int sample(final RandomGenerator random) { + while(true) { + + final double u = hIntegralNumberOfElements + random.nextDouble() * (hIntegralX1 - hIntegralNumberOfElements); + // u is uniformly distributed in (hIntegralX1, hIntegralNumberOfElements] + + double x = hIntegralInverse(u); + + int k = (int)(x + 0.5); + + // Limit k to the range [1, numberOfElements] + // (k could be outside due to numerical inaccuracies) + if (k < 1) { + k = 1; + } + else if (k > numberOfElements) { + k = numberOfElements; + } + + // Here, the distribution of k is given by: + // + // P(k = 1) = C * (hIntegral(1.5) - hIntegralX1) = C + // P(k = m) = C * (hIntegral(m + 1/2) - hIntegral(m - 1/2)) for m >= 2 + // + // where C := 1 / (hIntegralNumberOfElements - hIntegralX1) + + if (k - x <= s || u >= hIntegral(k + 0.5) - h(k)) { + + // Case k = 1: + // + // The right inequality is always true, because replacing k by 1 gives + // u >= hIntegral(1.5) - h(1) = hIntegralX1 and u is taken from + // (hIntegralX1, hIntegralNumberOfElements]. + // + // Therefore, the acceptance rate for k = 1 is P(accepted | k = 1) = 1 + // and the probability that 1 is returned as random value is + // P(k = 1 and accepted) = P(accepted | k = 1) * P(k = 1) = C = C / 1^exponent + // + // Case k >= 2: + // + // The left inequality (k - x <= s) is just a short cut + // to avoid the more expensive evaluation of the right inequality + // (u >= hIntegral(k + 0.5) - h(k)) in many cases. + // + // If the left inequality is true, the right inequality is also true: + // Theorem 2 in the paper is valid for all positive exponents, because + // the requirements h'(x) = -exponent/x^(exponent + 1) < 0 and + // (-1/hInverse'(x))'' = (1+1/exponent) * x^(1/exponent-1) >= 0 + // are both fulfilled. + // Therefore, f(x) := x - hIntegralInverse(hIntegral(x + 0.5) - h(x)) + // is a non-decreasing function. If k - x <= s holds, + // k - x <= s + f(k) - f(2) is obviously also true which is equivalent to + // -x <= -hIntegralInverse(hIntegral(k + 0.5) - h(k)), + // -hIntegralInverse(u) <= -hIntegralInverse(hIntegral(k + 0.5) - h(k)), + // and finally u >= hIntegral(k + 0.5) - h(k). + // + // Hence, the right inequality determines the acceptance rate: + // P(accepted | k = m) = h(m) / (hIntegrated(m+1/2) - hIntegrated(m-1/2)) + // The probability that m is returned is given by + // P(k = m and accepted) = P(accepted | k = m) * P(k = m) = C * h(m) = C / m^exponent. + // + // In both cases the probabilities are proportional to the probability mass function + // of the Zipf distribution. + + return k; + } + } + } + + /** + * {@code H(x) :=} + *

    + *
  • {@code (x^(1-exponent) - 1)/(1 - exponent)}, if {@code exponent != 1}
    • + *
  • {@code log(x)}, if {@code exponent == 1}
    • + *
+ * H(x) is an integral function of h(x), + * the derivative of H(x) is h(x). + * + * @param x free parameter + * @return {@code H(x)} + */ + private double hIntegral(final double x) { + final double logX = FastMath.log(x); + return helper2((1d-exponent)*logX)*logX; + } + + /** + * {@code h(x) := 1/x^exponent} + * + * @param x free parameter + * @return h(x) + */ + private double h(final double x) { + return FastMath.exp(-exponent * FastMath.log(x)); + } + + /** + * The inverse function of H(x). + * + * @param x free parameter + * @return y for which {@code H(y) = x} + */ + private double hIntegralInverse(final double x) { + double t = x*(1d-exponent); + if (t < -1d) { + // Limit value to the range [-1, +inf). + // t could be smaller than -1 in some rare cases due to numerical errors. + t = -1; + } + return FastMath.exp(helper1(t)*x); + } + + /** + * Helper function that calculates {@code log(1+x)/x}. + *

+ * A Taylor series expansion is used, if x is close to 0. + * + * @param x a value larger than or equal to -1 + * @return {@code log(1+x)/x} + */ + static double helper1(final double x) { + if (FastMath.abs(x)>1e-8) { + return FastMath.log1p(x)/x; + } + else { + return 1.-x*((1./2.)-x*((1./3.)-x*(1./4.))); + } + } + + /** + * Helper function to calculate {@code (exp(x)-1)/x}. + *

+ * A Taylor series expansion is used, if x is close to 0. + * + * @param x free parameter + * @return {@code (exp(x)-1)/x} if x is non-zero, or 1 if x=0 + */ + static double helper2(final double x) { + if (FastMath.abs(x)>1e-8) { + return FastMath.expm1(x)/x; + } + else { + return 1.+x*(1./2.)*(1.+x*(1./3.)*(1.+x*(1./4.))); + } + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/ConvergenceException.java b/java/source/infodynamics/utils/commonsmath3/exception/ConvergenceException.java old mode 100755 new mode 100644 index 7df3555..50737e0 --- a/java/source/infodynamics/utils/commonsmath3/exception/ConvergenceException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/ConvergenceException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,11 +14,11 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see . - */ +*/ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -53,7 +53,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; * numerical result failed to converge to a finite value. * * @since 2.2 - * @version $Id$ */ public class ConvergenceException extends MathIllegalStateException { /** Serializable version Id. */ diff --git a/java/source/infodynamics/utils/commonsmath3/exception/DimensionMismatchException.java b/java/source/infodynamics/utils/commonsmath3/exception/DimensionMismatchException.java old mode 100755 new mode 100644 index 07c2836..40b4d8c --- a/java/source/infodynamics/utils/commonsmath3/exception/DimensionMismatchException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/DimensionMismatchException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -52,7 +52,6 @@ import infodynamics.utils.commonsmath3.exception.util.Localizable; * Exception to be thrown when two dimensions differ. * * @since 2.2 - * @version $Id$ */ public class DimensionMismatchException extends MathIllegalNumberException { /** Serializable version Id. */ diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathArithmeticException.java b/java/source/infodynamics/utils/commonsmath3/exception/MathArithmeticException.java old mode 100755 new mode 100644 index 6246ad7..e2ae33a --- a/java/source/infodynamics/utils/commonsmath3/exception/MathArithmeticException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathArithmeticException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -57,7 +57,6 @@ import infodynamics.utils.commonsmath3.exception.util.ExceptionContextProvider; * message. * * @since 3.0 - * @version $Id$ */ public class MathArithmeticException extends ArithmeticException implements ExceptionContextProvider { diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalArgumentException.java b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalArgumentException.java old mode 100755 new mode 100644 index 2f6a27f..76d329d --- a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalArgumentException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalArgumentException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -56,7 +56,6 @@ import infodynamics.utils.commonsmath3.exception.util.ExceptionContextProvider; * of the standard {@link IllegalArgumentException}. * * @since 2.2 - * @version $Id$ */ public class MathIllegalArgumentException extends IllegalArgumentException implements ExceptionContextProvider { diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalNumberException.java b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalNumberException.java old mode 100755 new mode 100644 index f70e2e4..6597ef2 --- a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalNumberException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalNumberException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -54,7 +54,6 @@ import infodynamics.utils.commonsmath3.exception.util.Localizable; * precondition is violated by a number argument. * * @since 2.2 - * @version $Id$ */ public class MathIllegalNumberException extends MathIllegalArgumentException { diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalStateException.java b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalStateException.java old mode 100755 new mode 100644 index cb5f505..4f2cbc6 --- a/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalStateException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathIllegalStateException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -51,11 +51,11 @@ import infodynamics.utils.commonsmath3.exception.util.ExceptionContext; import infodynamics.utils.commonsmath3.exception.util.ExceptionContextProvider; /** - * Base class for all exceptions that signal a mismatch between the - * current state and the user's expectations. + * Base class for all exceptions that signal that the process + * throwing the exception is in a state that does not comply with + * the set of states that it is designed to be in. * * @since 2.2 - * @version $Id$ */ public class MathIllegalStateException extends IllegalStateException implements ExceptionContextProvider { diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathInternalError.java b/java/source/infodynamics/utils/commonsmath3/exception/MathInternalError.java new file mode 100644 index 0000000..de99627 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathInternalError.java @@ -0,0 +1,86 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception triggered when something that shouldn't happen does happen. + * + * @since 2.2 + */ +public class MathInternalError extends MathIllegalStateException { + /** Serializable version Id. */ + private static final long serialVersionUID = -6276776513966934846L; + /** URL for reporting problems. */ + private static final String REPORT_URL = "https://issues.apache.org/jira/browse/MATH"; + + /** + * Simple constructor. + */ + public MathInternalError() { + getContext().addMessage(LocalizedFormats.INTERNAL_ERROR, REPORT_URL); + } + + /** + * Simple constructor. + * @param cause root cause + */ + public MathInternalError(final Throwable cause) { + super(cause, LocalizedFormats.INTERNAL_ERROR, REPORT_URL); + } + + /** + * Constructor accepting a localized message. + * + * @param pattern Message pattern explaining the cause of the error. + * @param args Arguments. + */ + public MathInternalError(Localizable pattern, Object ... args) { + super(pattern, args); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MathUnsupportedOperationException.java b/java/source/infodynamics/utils/commonsmath3/exception/MathUnsupportedOperationException.java new file mode 100644 index 0000000..b39940b --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/MathUnsupportedOperationException.java @@ -0,0 +1,101 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.exception.util.ExceptionContext; +import infodynamics.utils.commonsmath3.exception.util.ExceptionContextProvider; + +/** + * Base class for all unsupported features. + * It is used for all the exceptions that have the semantics of the standard + * {@link UnsupportedOperationException}, but must also provide a localized + * message. + * + * @since 2.2 + */ +public class MathUnsupportedOperationException extends UnsupportedOperationException + implements ExceptionContextProvider { + /** Serializable version Id. */ + private static final long serialVersionUID = -6024911025449780478L; + /** Context. */ + private final ExceptionContext context; + + /** + * Default constructor. + */ + public MathUnsupportedOperationException() { + this(LocalizedFormats.UNSUPPORTED_OPERATION); + } + /** + * @param pattern Message pattern providing the specific context of + * the error. + * @param args Arguments. + */ + public MathUnsupportedOperationException(Localizable pattern, + Object ... args) { + context = new ExceptionContext(this); + context.addMessage(pattern, args); + } + + /** {@inheritDoc} */ + public ExceptionContext getContext() { + return context; + } + + /** {@inheritDoc} */ + @Override + public String getMessage() { + return context.getMessage(); + } + + /** {@inheritDoc} */ + @Override + public String getLocalizedMessage() { + return context.getLocalizedMessage(); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/MaxCountExceededException.java b/java/source/infodynamics/utils/commonsmath3/exception/MaxCountExceededException.java old mode 100755 new mode 100644 index a676f55..32bc48a --- a/java/source/infodynamics/utils/commonsmath3/exception/MaxCountExceededException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/MaxCountExceededException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -52,7 +52,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; * Exception to be thrown when some counter maximum value is exceeded. * * @since 3.0 - * @version $Id$ */ public class MaxCountExceededException extends MathIllegalStateException { /** Serializable version Id. */ diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NoBracketingException.java b/java/source/infodynamics/utils/commonsmath3/exception/NoBracketingException.java new file mode 100644 index 0000000..64d088b --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NoBracketingException.java @@ -0,0 +1,135 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when function values have the same sign at both + * ends of an interval. + * + * @since 3.0 + */ +public class NoBracketingException extends MathIllegalArgumentException { + /** Serializable version Id. */ + private static final long serialVersionUID = -3629324471511904459L; + /** Lower end of the interval. */ + private final double lo; + /** Higher end of the interval. */ + private final double hi; + /** Value at lower end of the interval. */ + private final double fLo; + /** Value at higher end of the interval. */ + private final double fHi; + + /** + * Construct the exception. + * + * @param lo Lower end of the interval. + * @param hi Higher end of the interval. + * @param fLo Value at lower end of the interval. + * @param fHi Value at higher end of the interval. + */ + public NoBracketingException(double lo, double hi, + double fLo, double fHi) { + this(LocalizedFormats.SAME_SIGN_AT_ENDPOINTS, lo, hi, fLo, fHi); + } + + /** + * Construct the exception with a specific context. + * + * @param specific Contextual information on what caused the exception. + * @param lo Lower end of the interval. + * @param hi Higher end of the interval. + * @param fLo Value at lower end of the interval. + * @param fHi Value at higher end of the interval. + * @param args Additional arguments. + */ + public NoBracketingException(Localizable specific, + double lo, double hi, + double fLo, double fHi, + Object ... args) { + super(specific, Double.valueOf(lo), Double.valueOf(hi), Double.valueOf(fLo), Double.valueOf(fHi), args); + this.lo = lo; + this.hi = hi; + this.fLo = fLo; + this.fHi = fHi; + } + + /** + * Get the lower end of the interval. + * + * @return the lower end. + */ + public double getLo() { + return lo; + } + /** + * Get the higher end of the interval. + * + * @return the higher end. + */ + public double getHi() { + return hi; + } + /** + * Get the value at the lower end of the interval. + * + * @return the value at the lower end. + */ + public double getFLo() { + return fLo; + } + /** + * Get the value at the higher end of the interval. + * + * @return the value at the higher end. + */ + public double getFHi() { + return fHi; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NoDataException.java b/java/source/infodynamics/utils/commonsmath3/exception/NoDataException.java new file mode 100644 index 0000000..03115b3 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NoDataException.java @@ -0,0 +1,75 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when the required data is missing. + * + * @since 2.2 + */ +public class NoDataException extends MathIllegalArgumentException { + + /** Serializable version Id. */ + private static final long serialVersionUID = -3629324471511904459L; + + /** + * Construct the exception. + */ + public NoDataException() { + this(LocalizedFormats.NO_DATA); + } + /** + * Construct the exception with a specific context. + * + * @param specific Contextual information on what caused the exception. + */ + public NoDataException(Localizable specific) { + super(specific); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NonMonotonicSequenceException.java b/java/source/infodynamics/utils/commonsmath3/exception/NonMonotonicSequenceException.java new file mode 100644 index 0000000..625f4a6 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NonMonotonicSequenceException.java @@ -0,0 +1,149 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.util.MathArrays; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when the a sequence of values is not monotonically + * increasing or decreasing. + * + * @since 2.2 (name changed to "NonMonotonicSequenceException" in 3.0) + */ +public class NonMonotonicSequenceException extends MathIllegalNumberException { + /** Serializable version Id. */ + private static final long serialVersionUID = 3596849179428944575L; + /** + * Direction (positive for increasing, negative for decreasing). + */ + private final MathArrays.OrderDirection direction; + /** + * Whether the sequence must be strictly increasing or decreasing. + */ + private final boolean strict; + /** + * Index of the wrong value. + */ + private final int index; + /** + * Previous value. + */ + private final Number previous; + + /** + * Construct the exception. + * This constructor uses default values assuming that the sequence should + * have been strictly increasing. + * + * @param wrong Value that did not match the requirements. + * @param previous Previous value in the sequence. + * @param index Index of the value that did not match the requirements. + */ + public NonMonotonicSequenceException(Number wrong, + Number previous, + int index) { + this(wrong, previous, index, MathArrays.OrderDirection.INCREASING, true); + } + + /** + * Construct the exception. + * + * @param wrong Value that did not match the requirements. + * @param previous Previous value in the sequence. + * @param index Index of the value that did not match the requirements. + * @param direction Strictly positive for a sequence required to be + * increasing, negative (or zero) for a decreasing sequence. + * @param strict Whether the sequence must be strictly increasing or + * decreasing. + */ + public NonMonotonicSequenceException(Number wrong, + Number previous, + int index, + MathArrays.OrderDirection direction, + boolean strict) { + super(direction == MathArrays.OrderDirection.INCREASING ? + (strict ? + LocalizedFormats.NOT_STRICTLY_INCREASING_SEQUENCE : + LocalizedFormats.NOT_INCREASING_SEQUENCE) : + (strict ? + LocalizedFormats.NOT_STRICTLY_DECREASING_SEQUENCE : + LocalizedFormats.NOT_DECREASING_SEQUENCE), + wrong, previous, Integer.valueOf(index), Integer.valueOf(index - 1)); + + this.direction = direction; + this.strict = strict; + this.index = index; + this.previous = previous; + } + + /** + * @return the order direction. + **/ + public MathArrays.OrderDirection getDirection() { + return direction; + } + /** + * @return {@code true} is the sequence should be strictly monotonic. + **/ + public boolean getStrict() { + return strict; + } + /** + * Get the index of the wrong value. + * + * @return the current index. + */ + public int getIndex() { + return index; + } + /** + * @return the previous value. + */ + public Number getPrevious() { + return previous; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NotANumberException.java b/java/source/infodynamics/utils/commonsmath3/exception/NotANumberException.java new file mode 100644 index 0000000..ce913ee --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NotANumberException.java @@ -0,0 +1,66 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when a number is not a number. + * + * @since 3.1 + */ +public class NotANumberException extends MathIllegalNumberException { + /** Serializable version Id. */ + private static final long serialVersionUID = 20120906L; + + /** + * Construct the exception. + */ + public NotANumberException() { + super(LocalizedFormats.NAN_NOT_ALLOWED, Double.valueOf(Double.NaN)); + } + +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NotFiniteNumberException.java b/java/source/infodynamics/utils/commonsmath3/exception/NotFiniteNumberException.java new file mode 100644 index 0000000..7b744ff --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NotFiniteNumberException.java @@ -0,0 +1,83 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when a number is not finite. + * + * @since 3.0 + */ +public class NotFiniteNumberException extends MathIllegalNumberException { + /** Serializable version Id. */ + private static final long serialVersionUID = -6100997100383932834L; + + /** + * Construct the exception. + * + * @param wrong Value that is infinite or NaN. + * @param args Optional arguments. + */ + public NotFiniteNumberException(Number wrong, + Object ... args) { + this(LocalizedFormats.NOT_FINITE_NUMBER, wrong, args); + } + + /** + * Construct the exception with a specific context. + * + * @param specific Specific context pattern. + * @param wrong Value that is infinite or NaN. + * @param args Optional arguments. + */ + public NotFiniteNumberException(Localizable specific, + Number wrong, + Object ... args) { + super(specific, wrong, args); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NotPositiveException.java b/java/source/infodynamics/utils/commonsmath3/exception/NotPositiveException.java new file mode 100644 index 0000000..7bce917 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NotPositiveException.java @@ -0,0 +1,77 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; + +/** + * Exception to be thrown when the argument is negative. + * + * @since 2.2 + */ +public class NotPositiveException extends NumberIsTooSmallException { + /** Serializable version Id. */ + private static final long serialVersionUID = -2250556892093726375L; + + /** + * Construct the exception. + * + * @param value Argument. + */ + public NotPositiveException(Number value) { + super(value, INTEGER_ZERO, true); + } + /** + * Construct the exception with a specific context. + * + * @param specific Specific context where the error occurred. + * @param value Argument. + */ + public NotPositiveException(Localizable specific, + Number value) { + super(specific, value, INTEGER_ZERO, true); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NotStrictlyPositiveException.java b/java/source/infodynamics/utils/commonsmath3/exception/NotStrictlyPositiveException.java new file mode 100644 index 0000000..8769d17 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NotStrictlyPositiveException.java @@ -0,0 +1,78 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; + +/** + * Exception to be thrown when the argument is not greater than 0. + * + * @since 2.2 + */ +public class NotStrictlyPositiveException extends NumberIsTooSmallException { + + /** Serializable version Id. */ + private static final long serialVersionUID = -7824848630829852237L; + + /** + * Construct the exception. + * + * @param value Argument. + */ + public NotStrictlyPositiveException(Number value) { + super(value, INTEGER_ZERO, false); + } + /** + * Construct the exception with a specific context. + * + * @param specific Specific context where the error occurred. + * @param value Argument. + */ + public NotStrictlyPositiveException(Localizable specific, + Number value) { + super(specific, value, INTEGER_ZERO, false); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NullArgumentException.java b/java/source/infodynamics/utils/commonsmath3/exception/NullArgumentException.java new file mode 100644 index 0000000..6da2f04 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/NullArgumentException.java @@ -0,0 +1,80 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * All conditions checks that fail due to a {@code null} argument must throw + * this exception. + * This class is meant to signal a precondition violation ("null is an illegal + * argument") and so does not extend the standard {@code NullPointerException}. + * Propagation of {@code NullPointerException} from within Commons-Math is + * construed to be a bug. + * + * @since 2.2 + */ +public class NullArgumentException extends MathIllegalArgumentException { + /** Serializable version Id. */ + private static final long serialVersionUID = -6024911025449780478L; + + /** + * Default constructor. + */ + public NullArgumentException() { + this(LocalizedFormats.NULL_NOT_ALLOWED); + } + /** + * @param pattern Message pattern providing the specific context of + * the error. + * @param arguments Values for replacing the placeholders in {@code pattern}. + */ + public NullArgumentException(Localizable pattern, + Object ... arguments) { + super(pattern, arguments); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooLargeException.java b/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooLargeException.java old mode 100755 new mode 100644 index aade84a..de737f3 --- a/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooLargeException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooLargeException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -52,7 +52,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; * Exception to be thrown when a number is too large. * * @since 2.2 - * @version $Id$ */ public class NumberIsTooLargeException extends MathIllegalNumberException { /** Serializable version Id. */ diff --git a/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooSmallException.java b/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooSmallException.java old mode 100755 new mode 100644 index 8a9a409..bda973e --- a/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooSmallException.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/NumberIsTooSmallException.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -52,7 +52,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; * Exception to be thrown when a number is too small. * * @since 2.2 - * @version $Id$ */ public class NumberIsTooSmallException extends MathIllegalNumberException { /** Serializable version Id. */ diff --git a/java/source/infodynamics/utils/commonsmath3/exception/OutOfRangeException.java b/java/source/infodynamics/utils/commonsmath3/exception/OutOfRangeException.java new file mode 100644 index 0000000..668531d --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/OutOfRangeException.java @@ -0,0 +1,107 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.exception.util.Localizable; + +/** + * Exception to be thrown when some argument is out of range. + * + * @since 2.2 + */ +public class OutOfRangeException extends MathIllegalNumberException { + /** Serializable version Id. */ + private static final long serialVersionUID = 111601815794403609L; + /** Lower bound. */ + private final Number lo; + /** Higher bound. */ + private final Number hi; + + /** + * Construct an exception from the mismatched dimensions. + * + * @param wrong Requested value. + * @param lo Lower bound. + * @param hi Higher bound. + */ + public OutOfRangeException(Number wrong, + Number lo, + Number hi) { + this(LocalizedFormats.OUT_OF_RANGE_SIMPLE, wrong, lo, hi); + } + + /** + * Construct an exception from the mismatched dimensions with a + * specific context information. + * + * @param specific Context information. + * @param wrong Requested value. + * @param lo Lower bound. + * @param hi Higher bound. + */ + public OutOfRangeException(Localizable specific, + Number wrong, + Number lo, + Number hi) { + super(specific, wrong, lo, hi); + this.lo = lo; + this.hi = hi; + } + + /** + * @return the lower bound. + */ + public Number getLo() { + return lo; + } + /** + * @return the higher bound. + */ + public Number getHi() { + return hi; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/TooManyEvaluationsException.java b/java/source/infodynamics/utils/commonsmath3/exception/TooManyEvaluationsException.java new file mode 100644 index 0000000..b7fdeca --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/TooManyEvaluationsException.java @@ -0,0 +1,68 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when the maximal number of evaluations is exceeded. + * + * @since 3.0 + */ +public class TooManyEvaluationsException extends MaxCountExceededException { + /** Serializable version Id. */ + private static final long serialVersionUID = 4330003017885151975L; + + /** + * Construct the exception. + * + * @param max Maximum number of evaluations. + */ + public TooManyEvaluationsException(Number max) { + super(max); + getContext().addMessage(LocalizedFormats.EVALUATIONS); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/ZeroException.java b/java/source/infodynamics/utils/commonsmath3/exception/ZeroException.java new file mode 100644 index 0000000..9d1eba5 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/exception/ZeroException.java @@ -0,0 +1,77 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.exception; + +import infodynamics.utils.commonsmath3.exception.util.Localizable; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; + +/** + * Exception to be thrown when zero is provided where it is not allowed. + * + * @since 2.2 + */ +public class ZeroException extends MathIllegalNumberException { + + /** Serializable version identifier */ + private static final long serialVersionUID = -1960874856936000015L; + + /** + * Construct the exception. + */ + public ZeroException() { + this(LocalizedFormats.ZERO_NOT_ALLOWED); + } + + /** + * Construct the exception with a specific context. + * + * @param specific Specific context pattern. + * @param arguments Arguments. + */ + public ZeroException(Localizable specific, Object ... arguments) { + super(specific, INTEGER_ZERO, arguments); + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/exception/util/ArgUtils.java b/java/source/infodynamics/utils/commonsmath3/exception/util/ArgUtils.java old mode 100755 new mode 100644 index 497586a..7d82bdc --- a/java/source/infodynamics/utils/commonsmath3/exception/util/ArgUtils.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/util/ArgUtils.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -52,7 +52,6 @@ import java.util.ArrayList; * Utility class for transforming the list of arguments passed to * constructors of exceptions. * - * @version $Id$ */ public class ArgUtils { /** diff --git a/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContext.java b/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContext.java old mode 100755 new mode 100644 index 0a1e55c..ffd1bb5 --- a/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContext.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContext.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -63,7 +63,6 @@ import java.util.Locale; * All Commons Math exceptions delegate the interface's methods to this class. * * @since 3.0 - * @version $Id$ */ public class ExceptionContext implements Serializable { /** Serializable version Id. */ @@ -313,12 +312,12 @@ public class ExceptionContext implements Serializable { private void serializeContext(ObjectOutputStream out) throws IOException { // Step 1. - final int len = context.keySet().size(); + final int len = context.size(); out.writeInt(len); - for (String key : context.keySet()) { + for (Map.Entry entry : context.entrySet()) { // Step 2. - out.writeObject(key); - final Object value = context.get(key); + out.writeObject(entry.getKey()); + final Object value = entry.getValue(); if (value instanceof Serializable) { // Step 3a. out.writeObject(value); diff --git a/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContextProvider.java b/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContextProvider.java old mode 100755 new mode 100644 index a7dad1c..f9b1162 --- a/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContextProvider.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/util/ExceptionContextProvider.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -49,7 +49,6 @@ package infodynamics.utils.commonsmath3.exception.util; * Interface for accessing the context data structure stored in Commons Math * exceptions. * - * @version $Id$ */ public interface ExceptionContextProvider { /** diff --git a/java/source/infodynamics/utils/commonsmath3/exception/util/Localizable.java b/java/source/infodynamics/utils/commonsmath3/exception/util/Localizable.java old mode 100755 new mode 100644 index 8aa9475..fa0bc08 --- a/java/source/infodynamics/utils/commonsmath3/exception/util/Localizable.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/util/Localizable.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -51,7 +51,6 @@ import java.util.Locale; /** * Interface for localizable strings. * - * @version $Id$ * @since 2.2 */ public interface Localizable extends Serializable { diff --git a/java/source/infodynamics/utils/commonsmath3/exception/util/LocalizedFormats.java b/java/source/infodynamics/utils/commonsmath3/exception/util/LocalizedFormats.java old mode 100755 new mode 100644 index 338b07d..0e371b9 --- a/java/source/infodynamics/utils/commonsmath3/exception/util/LocalizedFormats.java +++ b/java/source/infodynamics/utils/commonsmath3/exception/util/LocalizedFormats.java @@ -1,6 +1,6 @@ /* * Java Information Dynamics Toolkit (JIDT) - * Copyright (C) 2012, Joseph T. Lizier + * Copyright (C) 2017, Joseph T. Lizier * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -18,7 +18,7 @@ /* * This class was originally distributed as part of the Apache Commons - * Math3 library, under the Apache License Version 2.0, which is + * Math3 library (3.6.1), under the Apache License Version 2.0, which is * copied below. This Apache 2 software is now included as a derivative * work in the GPLv3 licensed JIDT project, as per: * http://www.apache.org/licenses/GPL-compatibility.html @@ -63,7 +63,6 @@ import java.util.ResourceBundle; * translation is missing. *

* @since 2.2 - * @version $Id$ */ public enum LocalizedFormats implements Localizable { @@ -78,6 +77,8 @@ public enum LocalizedFormats implements Localizable { AT_LEAST_ONE_COLUMN("matrix must have at least one column"), AT_LEAST_ONE_ROW("matrix must have at least one row"), BANDWIDTH("bandwidth ({0})"), + BESSEL_FUNCTION_BAD_ARGUMENT("Bessel function of order {0} cannot be computed for x = {1}"), + BESSEL_FUNCTION_FAILED_CONVERGENCE("Bessel function of order {0} failed to converge for x = {1}"), BINOMIAL_INVALID_PARAMETERS_ORDER("must have n >= k for binomial coefficient (n, k), got k = {0}, n = {1}"), BINOMIAL_NEGATIVE_PARAMETER("must have n >= 0 for binomial coefficient (n, k), got n = {0}"), CANNOT_CLEAR_STATISTIC_CONSTRUCTED_FROM_EXTERNAL_MOMENTS("statistics constructed from external moments cannot be cleared"), @@ -98,6 +99,7 @@ public enum LocalizedFormats implements Localizable { CANNOT_TRANSFORM_TO_DOUBLE("Conversion Exception in Transformation: {0}"), CARDAN_ANGLES_SINGULARITY("Cardan angles singularity"), CLASS_DOESNT_IMPLEMENT_COMPARABLE("class ({0}) does not implement Comparable"), + CLOSE_VERTICES("too close vertices near point ({0}, {1}, {2})"), CLOSEST_ORTHOGONAL_MATRIX_HAS_NEGATIVE_DETERMINANT("the closest orthogonal matrix has a negative determinant {0}"), COLUMN_INDEX_OUT_OF_RANGE("column index {0} out of allowed range [{1}, {2}]"), COLUMN_INDEX("column index ({0})"), /* keep */ @@ -119,6 +121,7 @@ public enum LocalizedFormats implements Localizable { DISCRETE_CUMULATIVE_PROBABILITY_RETURNED_NAN("Discrete cumulative probability function returned NaN for argument {0}"), DISTRIBUTION_NOT_LOADED("distribution not loaded"), DUPLICATED_ABSCISSA_DIVISION_BY_ZERO("duplicated abscissa {0} causes division by zero"), + EDGE_CONNECTED_TO_ONE_FACET("edge joining points ({0}, {1}, {2}) and ({3}, {4}, {5}) is connected to one facet only"), ELITISM_RATE("elitism rate ({0})"), EMPTY_CLUSTER_IN_K_MEANS("empty cluster in k-means"), EMPTY_INTERPOLATION_SAMPLE("sample for interpolation is empty"), @@ -131,6 +134,7 @@ public enum LocalizedFormats implements Localizable { EULER_ANGLES_SINGULARITY("Euler angles singularity"), EVALUATION("evaluation"), /* keep */ EXPANSION_FACTOR_SMALLER_THAN_ONE("expansion factor smaller than one ({0})"), + FACET_ORIENTATION_MISMATCH("facets orientation mismatch around edge joining points ({0}, {1}, {2}) and ({3}, {4}, {5})"), FACTORIAL_NEGATIVE_PARAMETER("must have n >= 0 for n!, got n = {0}"), FAILED_BRACKETING("number of iterations={4}, maximum iterations={5}, initial={6}, lower bound={7}, upper bound={8}, final a value={0}, final b value={1}, f(a)={2}, f(b)={3}"), FAILED_FRACTION_CONVERSION("Unable to convert {0} to fraction after {1} iterations"), @@ -172,6 +176,7 @@ public enum LocalizedFormats implements Localizable { INVALID_BINARY_CHROMOSOME("binary mutation works on BinaryChromosome only"), INVALID_BRACKETING_PARAMETERS("invalid bracketing parameters: lower bound={0}, initial={1}, upper bound={2}"), INVALID_FIXED_LENGTH_CHROMOSOME("one-point crossover only works with fixed-length chromosomes"), + INVALID_IMPLEMENTATION("required functionality is missing in {0}"), INVALID_INTERVAL_INITIAL_VALUE_PARAMETERS("invalid interval, initial value parameters: lower={0}, initial={1}, upper={2}"), INVALID_ITERATIONS_LIMITS("invalid iteration limits: min={0}, max={1}"), INVALID_MAX_ITERATIONS("bad value for maximum iterations number: {0}"), @@ -188,6 +193,7 @@ public enum LocalizedFormats implements Localizable { LOWER_BOUND_NOT_BELOW_UPPER_BOUND("lower bound ({0}) must be strictly less than upper bound ({1})"), /* keep */ LOWER_ENDPOINT_ABOVE_UPPER_ENDPOINT("lower endpoint ({0}) must be less than or equal to upper endpoint ({1})"), MAP_MODIFIED_WHILE_ITERATING("map has been modified while iterating"), + MULTISTEP_STARTER_STOPPED_EARLY("multistep integrator starter stopped early, maybe too large step size"), EVALUATIONS("evaluations"), /* keep */ MAX_COUNT_EXCEEDED("maximal count ({0}) exceeded"), /* keep */ MAX_ITERATIONS_EXCEEDED("maximal number of iterations ({0}) exceeded"), @@ -206,6 +212,7 @@ public enum LocalizedFormats implements Localizable { NUMBER_OF_INTERPOLATION_POINTS("number of interpolation points ({0})"), /* keep */ NUMBER_OF_TRIALS("number of trials ({0})"), NOT_CONVEX("vertices do not form a convex hull in CCW winding"), + NOT_CONVEX_HYPERPLANES("hyperplanes do not define a convex region"), ROBUSTNESS_ITERATIONS("number of robustness iterations ({0})"), START_POSITION("start position ({0})"), /* keep */ NON_CONVERGENT_CONTINUED_FRACTION("Continued fraction convergents failed to converge (in less than {0} iterations) for value {1}"), @@ -312,6 +319,7 @@ public enum LocalizedFormats implements Localizable { OUT_OF_BOUND_SIGNIFICANCE_LEVEL("out of bounds significance level {0}, must be between {1} and {2}"), SIGNIFICANCE_LEVEL("significance level ({0})"), /* keep */ OUT_OF_ORDER_ABSCISSA_ARRAY("the abscissae array must be sorted in a strictly increasing order, but the {0}-th element is {1} whereas {2}-th is {3}"), + OUT_OF_PLANE("point ({0}, {1}, {2}) is out of plane"), OUT_OF_RANGE_ROOT_OF_UNITY_INDEX("out of range root of unity index {0} (must be in [{1};{2}])"), OUT_OF_RANGE("out of range"), /* keep */ OUT_OF_RANGE_SIMPLE("{0} out of [{1}, {2}] range"), /* keep */ @@ -322,6 +330,7 @@ public enum LocalizedFormats implements Localizable { OVERFLOW_IN_FRACTION("overflow in fraction {0}/{1}, cannot negate"), OVERFLOW_IN_ADDITION("overflow in addition: {0} + {1}"), OVERFLOW_IN_SUBTRACTION("overflow in subtraction: {0} - {1}"), + OVERFLOW_IN_MULTIPLICATION("overflow in multiplication: {0} * {1}"), PERCENTILE_IMPLEMENTATION_CANNOT_ACCESS_METHOD("cannot access {0} method in percentile implementation {1}"), PERCENTILE_IMPLEMENTATION_UNSUPPORTED_METHOD("percentile implementation {0} does not support {1}"), PERMUTATION_EXCEEDS_N("permutation size ({0}) exceeds permuation domain ({1})"), /* keep */ @@ -401,7 +410,7 @@ public enum LocalizedFormats implements Localizable { * @param sourceFormat source English format to use when no * localized version is available */ - private LocalizedFormats(final String sourceFormat) { + LocalizedFormats(final String sourceFormat) { this.sourceFormat = sourceFormat; } diff --git a/java/source/infodynamics/utils/commonsmath3/random/AbstractWell.java b/java/source/infodynamics/utils/commonsmath3/random/AbstractWell.java new file mode 100644 index 0000000..708f493 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/AbstractWell.java @@ -0,0 +1,216 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.random; + +import java.io.Serializable; + +import infodynamics.utils.commonsmath3.util.FastMath; + + +/** This abstract class implements the WELL class of pseudo-random number generator + * from François Panneton, Pierre L'Ecuyer and Makoto Matsumoto. + + *

This generator is described in a paper by François Panneton, + * Pierre L'Ecuyer and Makoto Matsumoto Improved + * Long-Period Generators Based on Linear Recurrences Modulo 2 ACM + * Transactions on Mathematical Software, 32, 1 (2006). The errata for the paper + * are in wellrng-errata.txt.

+ + * @see WELL Random number generator + * @since 2.2 + + */ +public abstract class AbstractWell extends BitsStreamGenerator implements Serializable { + + /** Serializable version identifier. */ + private static final long serialVersionUID = -817701723016583596L; + + /** Current index in the bytes pool. */ + protected int index; + + /** Bytes pool. */ + protected final int[] v; + + /** Index indirection table giving for each index its predecessor taking table size into account. */ + protected final int[] iRm1; + + /** Index indirection table giving for each index its second predecessor taking table size into account. */ + protected final int[] iRm2; + + /** Index indirection table giving for each index the value index + m1 taking table size into account. */ + protected final int[] i1; + + /** Index indirection table giving for each index the value index + m2 taking table size into account. */ + protected final int[] i2; + + /** Index indirection table giving for each index the value index + m3 taking table size into account. */ + protected final int[] i3; + + /** Creates a new random number generator. + *

The instance is initialized using the current time plus the + * system identity hash code of this instance as the seed.

+ * @param k number of bits in the pool (not necessarily a multiple of 32) + * @param m1 first parameter of the algorithm + * @param m2 second parameter of the algorithm + * @param m3 third parameter of the algorithm + */ + protected AbstractWell(final int k, final int m1, final int m2, final int m3) { + this(k, m1, m2, m3, null); + } + + /** Creates a new random number generator using a single int seed. + * @param k number of bits in the pool (not necessarily a multiple of 32) + * @param m1 first parameter of the algorithm + * @param m2 second parameter of the algorithm + * @param m3 third parameter of the algorithm + * @param seed the initial seed (32 bits integer) + */ + protected AbstractWell(final int k, final int m1, final int m2, final int m3, final int seed) { + this(k, m1, m2, m3, new int[] { seed }); + } + + /** Creates a new random number generator using an int array seed. + * @param k number of bits in the pool (not necessarily a multiple of 32) + * @param m1 first parameter of the algorithm + * @param m2 second parameter of the algorithm + * @param m3 third parameter of the algorithm + * @param seed the initial seed (32 bits integers array), if null + * the seed of the generator will be related to the current time + */ + protected AbstractWell(final int k, final int m1, final int m2, final int m3, final int[] seed) { + + // the bits pool contains k bits, k = r w - p where r is the number + // of w bits blocks, w is the block size (always 32 in the original paper) + // and p is the number of unused bits in the last block + final int w = 32; + final int r = (k + w - 1) / w; + this.v = new int[r]; + this.index = 0; + + // precompute indirection index tables. These tables are used for optimizing access + // they allow saving computations like "(j + r - 2) % r" with costly modulo operations + iRm1 = new int[r]; + iRm2 = new int[r]; + i1 = new int[r]; + i2 = new int[r]; + i3 = new int[r]; + for (int j = 0; j < r; ++j) { + iRm1[j] = (j + r - 1) % r; + iRm2[j] = (j + r - 2) % r; + i1[j] = (j + m1) % r; + i2[j] = (j + m2) % r; + i3[j] = (j + m3) % r; + } + + // initialize the pool content + setSeed(seed); + + } + + /** Creates a new random number generator using a single long seed. + * @param k number of bits in the pool (not necessarily a multiple of 32) + * @param m1 first parameter of the algorithm + * @param m2 second parameter of the algorithm + * @param m3 third parameter of the algorithm + * @param seed the initial seed (64 bits integer) + */ + protected AbstractWell(final int k, final int m1, final int m2, final int m3, final long seed) { + this(k, m1, m2, m3, new int[] { (int) (seed >>> 32), (int) (seed & 0xffffffffl) }); + } + + /** Reinitialize the generator as if just built with the given int seed. + *

The state of the generator is exactly the same as a new + * generator built with the same seed.

+ * @param seed the initial seed (32 bits integer) + */ + @Override + public void setSeed(final int seed) { + setSeed(new int[] { seed }); + } + + /** Reinitialize the generator as if just built with the given int array seed. + *

The state of the generator is exactly the same as a new + * generator built with the same seed.

+ * @param seed the initial seed (32 bits integers array). If null + * the seed of the generator will be the system time plus the system identity + * hash code of the instance. + */ + @Override + public void setSeed(final int[] seed) { + if (seed == null) { + setSeed(System.currentTimeMillis() + System.identityHashCode(this)); + return; + } + + System.arraycopy(seed, 0, v, 0, FastMath.min(seed.length, v.length)); + + if (seed.length < v.length) { + for (int i = seed.length; i < v.length; ++i) { + final long l = v[i - seed.length]; + v[i] = (int) ((1812433253l * (l ^ (l >> 30)) + i) & 0xffffffffL); + } + } + + index = 0; + clear(); // Clear normal deviate cache + } + + /** Reinitialize the generator as if just built with the given long seed. + *

The state of the generator is exactly the same as a new + * generator built with the same seed.

+ * @param seed the initial seed (64 bits integer) + */ + @Override + public void setSeed(final long seed) { + setSeed(new int[] { (int) (seed >>> 32), (int) (seed & 0xffffffffl) }); + } + + /** {@inheritDoc} */ + @Override + protected abstract int next(final int bits); + +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/BitsStreamGenerator.java b/java/source/infodynamics/utils/commonsmath3/random/BitsStreamGenerator.java new file mode 100644 index 0000000..dd13d95 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/BitsStreamGenerator.java @@ -0,0 +1,299 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.random; + +import java.io.Serializable; + +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** Base class for random number generators that generates bits streams. + * + * @since 2.0 + */ +public abstract class BitsStreamGenerator + implements RandomGenerator, + Serializable { + /** Serializable version identifier */ + private static final long serialVersionUID = 20130104L; + /** Next gaussian. */ + private double nextGaussian; + + /** + * Creates a new random number generator. + */ + public BitsStreamGenerator() { + nextGaussian = Double.NaN; + } + + /** {@inheritDoc} */ + public abstract void setSeed(int seed); + + /** {@inheritDoc} */ + public abstract void setSeed(int[] seed); + + /** {@inheritDoc} */ + public abstract void setSeed(long seed); + + /** Generate next pseudorandom number. + *

This method is the core generation algorithm. It is used by all the + * public generation methods for the various primitive types {@link + * #nextBoolean()}, {@link #nextBytes(byte[])}, {@link #nextDouble()}, + * {@link #nextFloat()}, {@link #nextGaussian()}, {@link #nextInt()}, + * {@link #next(int)} and {@link #nextLong()}.

+ * @param bits number of random bits to produce + * @return random bits generated + */ + protected abstract int next(int bits); + + /** {@inheritDoc} */ + public boolean nextBoolean() { + return next(1) != 0; + } + + /** {@inheritDoc} */ + public double nextDouble() { + final long high = ((long) next(26)) << 26; + final int low = next(26); + return (high | low) * 0x1.0p-52d; + } + + /** {@inheritDoc} */ + public float nextFloat() { + return next(23) * 0x1.0p-23f; + } + + /** {@inheritDoc} */ + public double nextGaussian() { + + final double random; + if (Double.isNaN(nextGaussian)) { + // generate a new pair of gaussian numbers + final double x = nextDouble(); + final double y = nextDouble(); + final double alpha = 2 * FastMath.PI * x; + final double r = FastMath.sqrt(-2 * FastMath.log(y)); + random = r * FastMath.cos(alpha); + nextGaussian = r * FastMath.sin(alpha); + } else { + // use the second element of the pair already generated + random = nextGaussian; + nextGaussian = Double.NaN; + } + + return random; + + } + + /** {@inheritDoc} */ + public int nextInt() { + return next(32); + } + + /** + * {@inheritDoc} + *

This default implementation is copied from Apache Harmony + * java.util.Random (r929253).

+ * + *

Implementation notes:

    + *
  • If n is a power of 2, this method returns + * {@code (int) ((n * (long) next(31)) >> 31)}.
    • + * + *
  • If n is not a power of 2, what is returned is {@code next(31) % n} + * with {@code next(31)} values rejected (i.e. regenerated) until a + * value that is larger than the remainder of {@code Integer.MAX_VALUE / n} + * is generated. Rejection of this initial segment is necessary to ensure + * a uniform distribution.

+ */ + public int nextInt(int n) throws IllegalArgumentException { + if (n > 0) { + if ((n & -n) == n) { + return (int) ((n * (long) next(31)) >> 31); + } + int bits; + int val; + do { + bits = next(31); + val = bits % n; + } while (bits - val + (n - 1) < 0); + return val; + } + throw new NotStrictlyPositiveException(n); + } + + /** {@inheritDoc} */ + public long nextLong() { + final long high = ((long) next(32)) << 32; + final long low = ((long) next(32)) & 0xffffffffL; + return high | low; + } + + /** + * Returns a pseudorandom, uniformly distributed {@code long} value + * between 0 (inclusive) and the specified value (exclusive), drawn from + * this random number generator's sequence. + * + * @param n the bound on the random number to be returned. Must be + * positive. + * @return a pseudorandom, uniformly distributed {@code long} + * value between 0 (inclusive) and n (exclusive). + * @throws IllegalArgumentException if n is not positive. + */ + public long nextLong(long n) throws IllegalArgumentException { + if (n > 0) { + long bits; + long val; + do { + bits = ((long) next(31)) << 32; + bits |= ((long) next(32)) & 0xffffffffL; + val = bits % n; + } while (bits - val + (n - 1) < 0); + return val; + } + throw new NotStrictlyPositiveException(n); + } + + /** + * Clears the cache used by the default implementation of + * {@link #nextGaussian}. + */ + public void clear() { + nextGaussian = Double.NaN; + } + + /** + * Generates random bytes and places them into a user-supplied array. + * + *

+ * The array is filled with bytes extracted from random integers. + * This implies that the number of random bytes generated may be larger than + * the length of the byte array. + *

+ * + * @param bytes Array in which to put the generated bytes. Cannot be {@code null}. + */ + public void nextBytes(byte[] bytes) { + nextBytesFill(bytes, 0, bytes.length); + } + + /** + * Generates random bytes and places them into a user-supplied array. + * + *

+ * The array is filled with bytes extracted from random integers. + * This implies that the number of random bytes generated may be larger than + * the length of the byte array. + *

+ * + * @param bytes Array in which to put the generated bytes. Cannot be {@code null}. + * @param start Index at which to start inserting the generated bytes. + * @param len Number of bytes to insert. + * @throws OutOfRangeException if {@code start < 0} or {@code start >= bytes.length}. + * @throws OutOfRangeException if {@code len < 0} or {@code len > bytes.length - start}. + */ + public void nextBytes(byte[] bytes, + int start, + int len) { + if (start < 0 || + start >= bytes.length) { + throw new OutOfRangeException(start, 0, bytes.length); + } + if (len < 0 || + len > bytes.length - start) { + throw new OutOfRangeException(len, 0, bytes.length - start); + } + + nextBytesFill(bytes, start, len); + } + + /** + * Generates random bytes and places them into a user-supplied array. + * + *

+ * The array is filled with bytes extracted from random integers. + * This implies that the number of random bytes generated may be larger than + * the length of the byte array. + *

+ * + * @param bytes Array in which to put the generated bytes. Cannot be {@code null}. + * @param start Index at which to start inserting the generated bytes. + * @param len Number of bytes to insert. + */ + private void nextBytesFill(byte[] bytes, + int start, + int len) { + int index = start; // Index of first insertion. + + // Index of first insertion plus multiple 4 part of length (i.e. length + // with two least significant bits unset). + final int indexLoopLimit = index + (len & 0x7ffffffc); + + // Start filling in the byte array, 4 bytes at a time. + while (index < indexLoopLimit) { + final int random = next(32); + bytes[index++] = (byte) random; + bytes[index++] = (byte) (random >>> 8); + bytes[index++] = (byte) (random >>> 16); + bytes[index++] = (byte) (random >>> 24); + } + + final int indexLimit = start + len; // Index of last insertion + 1. + + // Fill in the remaining bytes. + if (index < indexLimit) { + int random = next(32); + while (true) { + bytes[index++] = (byte) random; + if (index < indexLimit) { + random >>>= 8; + } else { + break; + } + } + } + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/RandomData.java b/java/source/infodynamics/utils/commonsmath3/random/RandomData.java new file mode 100644 index 0000000..6d96f3f --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/RandomData.java @@ -0,0 +1,293 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.random; +import java.util.Collection; + +import infodynamics.utils.commonsmath3.exception.NotANumberException; +import infodynamics.utils.commonsmath3.exception.NotFiniteNumberException; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; + +/** + * Random data generation utilities. + * @deprecated to be removed in 4.0. Use {@link RandomDataGenerator} directly + */ +@Deprecated +public interface RandomData { + /** + * Generates a random string of hex characters of length {@code len}. + *

+ * The generated string will be random, but not cryptographically + * secure. To generate cryptographically secure strings, use + * {@link #nextSecureHexString(int)}. + *

+ * + * @param len the length of the string to be generated + * @return a random string of hex characters of length {@code len} + * @throws NotStrictlyPositiveException + * if {@code len <= 0} + */ + String nextHexString(int len) throws NotStrictlyPositiveException; + + /** + * Generates a uniformly distributed random integer between {@code lower} + * and {@code upper} (endpoints included). + *

+ * The generated integer will be random, but not cryptographically secure. + * To generate cryptographically secure integer sequences, use + * {@link #nextSecureInt(int, int)}. + *

+ * + * @param lower lower bound for generated integer + * @param upper upper bound for generated integer + * @return a random integer greater than or equal to {@code lower} + * and less than or equal to {@code upper} + * @throws NumberIsTooLargeException if {@code lower >= upper} + */ + int nextInt(int lower, int upper) throws NumberIsTooLargeException; + + /** + * Generates a uniformly distributed random long integer between + * {@code lower} and {@code upper} (endpoints included). + *

+ * The generated long integer values will be random, but not + * cryptographically secure. To generate cryptographically secure sequences + * of longs, use {@link #nextSecureLong(long, long)}. + *

+ * + * @param lower lower bound for generated long integer + * @param upper upper bound for generated long integer + * @return a random long integer greater than or equal to {@code lower} and + * less than or equal to {@code upper} + * @throws NumberIsTooLargeException if {@code lower >= upper} + */ + long nextLong(long lower, long upper) throws NumberIsTooLargeException; + + /** + * Generates a random string of hex characters from a secure random + * sequence. + *

+ * If cryptographic security is not required, use + * {@link #nextHexString(int)}. + *

+ * + * @param len the length of the string to be generated + * @return a random string of hex characters of length {@code len} + * @throws NotStrictlyPositiveException if {@code len <= 0} + */ + String nextSecureHexString(int len) throws NotStrictlyPositiveException; + + /** + * Generates a uniformly distributed random integer between {@code lower} + * and {@code upper} (endpoints included) from a secure random sequence. + *

+ * Sequences of integers generated using this method will be + * cryptographically secure. If cryptographic security is not required, + * {@link #nextInt(int, int)} should be used instead of this method.

+ *

+ * Definition: + * + * Secure Random Sequence

+ * + * @param lower lower bound for generated integer + * @param upper upper bound for generated integer + * @return a random integer greater than or equal to {@code lower} and less + * than or equal to {@code upper}. + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + int nextSecureInt(int lower, int upper) throws NumberIsTooLargeException; + + /** + * Generates a uniformly distributed random long integer between + * {@code lower} and {@code upper} (endpoints included) from a secure random + * sequence. + *

+ * Sequences of long values generated using this method will be + * cryptographically secure. If cryptographic security is not required, + * {@link #nextLong(long, long)} should be used instead of this method.

+ *

+ * Definition: + * + * Secure Random Sequence

+ * + * @param lower lower bound for generated integer + * @param upper upper bound for generated integer + * @return a random long integer greater than or equal to {@code lower} and + * less than or equal to {@code upper}. + * @throws NumberIsTooLargeException if {@code lower >= upper}. + */ + long nextSecureLong(long lower, long upper) throws NumberIsTooLargeException; + + /** + * Generates a random value from the Poisson distribution with the given + * mean. + *

+ * Definition: + * + * Poisson Distribution

+ * + * @param mean the mean of the Poisson distribution + * @return a random value following the specified Poisson distribution + * @throws NotStrictlyPositiveException if {@code mean <= 0}. + */ + long nextPoisson(double mean) throws NotStrictlyPositiveException; + + /** + * Generates a random value from the Normal (or Gaussian) distribution with + * specified mean and standard deviation. + *

+ * Definition: + * + * Normal Distribution

+ * + * @param mu the mean of the distribution + * @param sigma the standard deviation of the distribution + * @return a random value following the specified Gaussian distribution + * @throws NotStrictlyPositiveException if {@code sigma <= 0}. + */ + double nextGaussian(double mu, double sigma) throws NotStrictlyPositiveException; + + /** + * Generates a random value from the exponential distribution + * with specified mean. + *

+ * Definition: + * + * Exponential Distribution

+ * + * @param mean the mean of the distribution + * @return a random value following the specified exponential distribution + * @throws NotStrictlyPositiveException if {@code mean <= 0}. + */ + double nextExponential(double mean) throws NotStrictlyPositiveException; + + /** + * Generates a uniformly distributed random value from the open interval + * {@code (lower, upper)} (i.e., endpoints excluded). + *

+ * Definition: + * + * Uniform Distribution {@code lower} and {@code upper - lower} are the + * + * location and scale parameters, respectively.

+ * + * @param lower the exclusive lower bound of the support + * @param upper the exclusive upper bound of the support + * @return a uniformly distributed random value between lower and upper + * (exclusive) + * @throws NumberIsTooLargeException if {@code lower >= upper} + * @throws NotFiniteNumberException if one of the bounds is infinite + * @throws NotANumberException if one of the bounds is NaN + */ + double nextUniform(double lower, double upper) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException; + + /** + * Generates a uniformly distributed random value from the interval + * {@code (lower, upper)} or the interval {@code [lower, upper)}. The lower + * bound is thus optionally included, while the upper bound is always + * excluded. + *

+ * Definition: + * + * Uniform Distribution {@code lower} and {@code upper - lower} are the + * + * location and scale parameters, respectively.

+ * + * @param lower the lower bound of the support + * @param upper the exclusive upper bound of the support + * @param lowerInclusive {@code true} if the lower bound is inclusive + * @return uniformly distributed random value in the {@code (lower, upper)} + * interval, if {@code lowerInclusive} is {@code false}, or in the + * {@code [lower, upper)} interval, if {@code lowerInclusive} is + * {@code true} + * @throws NumberIsTooLargeException if {@code lower >= upper} + * @throws NotFiniteNumberException if one of the bounds is infinite + * @throws NotANumberException if one of the bounds is NaN + */ + double nextUniform(double lower, double upper, boolean lowerInclusive) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException; + + /** + * Generates an integer array of length {@code k} whose entries are selected + * randomly, without repetition, from the integers {@code 0, ..., n - 1} + * (inclusive). + *

+ * Generated arrays represent permutations of {@code n} taken {@code k} at a + * time.

+ * + * @param n the domain of the permutation + * @param k the size of the permutation + * @return a random {@code k}-permutation of {@code n}, as an array of + * integers + * @throws NumberIsTooLargeException if {@code k > n}. + * @throws NotStrictlyPositiveException if {@code k <= 0}. + */ + int[] nextPermutation(int n, int k) + throws NumberIsTooLargeException, NotStrictlyPositiveException; + + /** + * Returns an array of {@code k} objects selected randomly from the + * Collection {@code c}. + *

+ * Sampling from {@code c} is without replacement; but if {@code c} contains + * identical objects, the sample may include repeats. If all elements of + * {@code c} are distinct, the resulting object array represents a + * + * Simple Random Sample of size {@code k} from the elements of + * {@code c}.

+ * + * @param c the collection to be sampled + * @param k the size of the sample + * @return a random sample of {@code k} elements from {@code c} + * @throws NumberIsTooLargeException if {@code k > c.size()}. + * @throws NotStrictlyPositiveException if {@code k <= 0}. + */ + Object[] nextSample(Collection c, int k) + throws NumberIsTooLargeException, NotStrictlyPositiveException; + +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/RandomDataGenerator.java b/java/source/infodynamics/utils/commonsmath3/random/RandomDataGenerator.java new file mode 100644 index 0000000..e09a7c3 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/RandomDataGenerator.java @@ -0,0 +1,811 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.random; + +import java.io.Serializable; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.util.Collection; + +import infodynamics.utils.commonsmath3.distribution.BetaDistribution; +import infodynamics.utils.commonsmath3.distribution.BinomialDistribution; +import infodynamics.utils.commonsmath3.distribution.CauchyDistribution; +import infodynamics.utils.commonsmath3.distribution.ChiSquaredDistribution; +import infodynamics.utils.commonsmath3.distribution.ExponentialDistribution; +import infodynamics.utils.commonsmath3.distribution.FDistribution; +import infodynamics.utils.commonsmath3.distribution.GammaDistribution; +import infodynamics.utils.commonsmath3.distribution.HypergeometricDistribution; +import infodynamics.utils.commonsmath3.distribution.PascalDistribution; +import infodynamics.utils.commonsmath3.distribution.PoissonDistribution; +import infodynamics.utils.commonsmath3.distribution.TDistribution; +import infodynamics.utils.commonsmath3.distribution.WeibullDistribution; +import infodynamics.utils.commonsmath3.distribution.ZipfDistribution; +import infodynamics.utils.commonsmath3.distribution.UniformIntegerDistribution; +import infodynamics.utils.commonsmath3.exception.MathInternalError; +import infodynamics.utils.commonsmath3.exception.NotANumberException; +import infodynamics.utils.commonsmath3.exception.NotFiniteNumberException; +import infodynamics.utils.commonsmath3.exception.NotPositiveException; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats; +import infodynamics.utils.commonsmath3.util.MathArrays; + +/** + * Implements the {@link RandomData} interface using a {@link RandomGenerator} + * instance to generate non-secure data and a {@link java.security.SecureRandom} + * instance to provide data for the nextSecureXxx methods. If no + * RandomGenerator is provided in the constructor, the default is + * to use a {@link Well19937c} generator. To plug in a different + * implementation, either implement RandomGenerator directly or + * extend {@link AbstractRandomGenerator}. + *

+ * Supports reseeding the underlying pseudo-random number generator (PRNG). The + * SecurityProvider and Algorithm used by the + * SecureRandom instance can also be reset. + *

+ *

+ * For details on the default PRNGs, see {@link java.util.Random} and + * {@link java.security.SecureRandom}. + *

+ *

+ * Usage Notes: + *

    + *
  • + * Instance variables are used to maintain RandomGenerator and + * SecureRandom instances used in data generation. Therefore, to + * generate a random sequence of values or strings, you should use just + * one RandomDataImpl instance repeatedly.
    • + *
  • + * The "secure" methods are *much* slower. These should be used only when a + * cryptographically secure random sequence is required. A secure random + * sequence is a sequence of pseudo-random values which, in addition to being + * well-dispersed (so no subsequence of values is an any more likely than other + * subsequence of the the same length), also has the additional property that + * knowledge of values generated up to any point in the sequence does not make + * it any easier to predict subsequent values.
    • + *
  • + * When a new RandomDataImpl is created, the underlying random + * number generators are not initialized. If you do not + * explicitly seed the default non-secure generator, it is seeded with the + * current time in milliseconds plus the system identity hash code on first use. + * The same holds for the secure generator. If you provide a RandomGenerator + * to the constructor, however, this generator is not reseeded by the constructor + * nor is it reseeded on first use.
    • + *
  • + * The reSeed and reSeedSecure methods delegate to the + * corresponding methods on the underlying RandomGenerator and + * SecureRandom instances. Therefore, reSeed(long) + * fully resets the initial state of the non-secure random number generator (so + * that reseeding with a specific value always results in the same subsequent + * random sequence); whereas reSeedSecure(long) does not + * reinitialize the secure random number generator (so secure sequences started + * with calls to reseedSecure(long) won't be identical).
    • + *
  • + * This implementation is not synchronized. The underlying RandomGenerator + * or SecureRandom instances are not protected by synchronization and + * are not guaranteed to be thread-safe. Therefore, if an instance of this class + * is concurrently utilized by multiple threads, it is the responsibility of + * client code to synchronize access to seeding and data generation methods. + *
    • + *
+ *

+ * @since 3.1 + */ +public class RandomDataGenerator implements RandomData, Serializable { + + /** Serializable version identifier */ + private static final long serialVersionUID = -626730818244969716L; + + /** underlying random number generator */ + private RandomGenerator rand = null; + + /** underlying secure random number generator */ + private RandomGenerator secRand = null; + + /** + * Construct a RandomDataGenerator, using a default random generator as the source + * of randomness. + * + *

The default generator is a {@link Well19937c} seeded + * with {@code System.currentTimeMillis() + System.identityHashCode(this))}. + * The generator is initialized and seeded on first use.

+ */ + public RandomDataGenerator() { + } + + /** + * Construct a RandomDataGenerator using the supplied {@link RandomGenerator} as + * the source of (non-secure) random data. + * + * @param rand the source of (non-secure) random data + * (may be null, resulting in the default generator) + */ + public RandomDataGenerator(RandomGenerator rand) { + this.rand = rand; + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: hex strings are generated using a + * 2-step process. + *

    + *
  1. {@code len / 2 + 1} binary bytes are generated using the underlying + * Random
    2. + *
  2. Each binary byte is translated into 2 hex digits
    3. + *
+ *

+ * + * @param len the desired string length. + * @return the random string. + * @throws NotStrictlyPositiveException if {@code len <= 0}. + */ + public String nextHexString(int len) throws NotStrictlyPositiveException { + if (len <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.LENGTH, len); + } + + // Get a random number generator + RandomGenerator ran = getRandomGenerator(); + + // Initialize output buffer + StringBuilder outBuffer = new StringBuilder(); + + // Get int(len/2)+1 random bytes + byte[] randomBytes = new byte[(len / 2) + 1]; + ran.nextBytes(randomBytes); + + // Convert each byte to 2 hex digits + for (int i = 0; i < randomBytes.length; i++) { + Integer c = Integer.valueOf(randomBytes[i]); + + /* + * Add 128 to byte value to make interval 0-255 before doing hex + * conversion. This guarantees <= 2 hex digits from toHexString() + * toHexString would otherwise add 2^32 to negative arguments. + */ + String hex = Integer.toHexString(c.intValue() + 128); + + // Make sure we add 2 hex digits for each byte + if (hex.length() == 1) { + hex = "0" + hex; + } + outBuffer.append(hex); + } + return outBuffer.toString().substring(0, len); + } + + /** {@inheritDoc} */ + public int nextInt(final int lower, final int upper) throws NumberIsTooLargeException { + return new UniformIntegerDistribution(getRandomGenerator(), lower, upper).sample(); + } + + /** {@inheritDoc} */ + public long nextLong(final long lower, final long upper) throws NumberIsTooLargeException { + if (lower >= upper) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, + lower, upper, false); + } + final long max = (upper - lower) + 1; + if (max <= 0) { + // the range is too wide to fit in a positive long (larger than 2^63); as it covers + // more than half the long range, we use directly a simple rejection method + final RandomGenerator rng = getRandomGenerator(); + while (true) { + final long r = rng.nextLong(); + if (r >= lower && r <= upper) { + return r; + } + } + } else if (max < Integer.MAX_VALUE){ + // we can shift the range and generate directly a positive int + return lower + getRandomGenerator().nextInt((int) max); + } else { + // we can shift the range and generate directly a positive long + return lower + nextLong(getRandomGenerator(), max); + } + } + + /** + * Returns a pseudorandom, uniformly distributed {@code long} value + * between 0 (inclusive) and the specified value (exclusive), drawn from + * this random number generator's sequence. + * + * @param rng random generator to use + * @param n the bound on the random number to be returned. Must be + * positive. + * @return a pseudorandom, uniformly distributed {@code long} + * value between 0 (inclusive) and n (exclusive). + * @throws IllegalArgumentException if n is not positive. + */ + private static long nextLong(final RandomGenerator rng, final long n) throws IllegalArgumentException { + if (n > 0) { + final byte[] byteArray = new byte[8]; + long bits; + long val; + do { + rng.nextBytes(byteArray); + bits = 0; + for (final byte b : byteArray) { + bits = (bits << 8) | (((long) b) & 0xffL); + } + bits &= 0x7fffffffffffffffL; + val = bits % n; + } while (bits - val + (n - 1) < 0); + return val; + } + throw new NotStrictlyPositiveException(n); + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: hex strings are generated in + * 40-byte segments using a 3-step process. + *

    + *
  1. + * 20 random bytes are generated using the underlying + * SecureRandom.
    2. + *
  2. + * SHA-1 hash is applied to yield a 20-byte binary digest.
    3. + *
  3. + * Each byte of the binary digest is converted to 2 hex digits.
    4. + *
+ *

+ * @throws NotStrictlyPositiveException if {@code len <= 0} + */ + public String nextSecureHexString(int len) throws NotStrictlyPositiveException { + if (len <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.LENGTH, len); + } + + // Get SecureRandom and setup Digest provider + final RandomGenerator secRan = getSecRan(); + MessageDigest alg = null; + try { + alg = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException ex) { + // this should never happen + throw new MathInternalError(ex); + } + alg.reset(); + + // Compute number of iterations required (40 bytes each) + int numIter = (len / 40) + 1; + + StringBuilder outBuffer = new StringBuilder(); + for (int iter = 1; iter < numIter + 1; iter++) { + byte[] randomBytes = new byte[40]; + secRan.nextBytes(randomBytes); + alg.update(randomBytes); + + // Compute hash -- will create 20-byte binary hash + byte[] hash = alg.digest(); + + // Loop over the hash, converting each byte to 2 hex digits + for (int i = 0; i < hash.length; i++) { + Integer c = Integer.valueOf(hash[i]); + + /* + * Add 128 to byte value to make interval 0-255 This guarantees + * <= 2 hex digits from toHexString() toHexString would + * otherwise add 2^32 to negative arguments + */ + String hex = Integer.toHexString(c.intValue() + 128); + + // Keep strings uniform length -- guarantees 40 bytes + if (hex.length() == 1) { + hex = "0" + hex; + } + outBuffer.append(hex); + } + } + return outBuffer.toString().substring(0, len); + } + + /** {@inheritDoc} */ + public int nextSecureInt(final int lower, final int upper) throws NumberIsTooLargeException { + return new UniformIntegerDistribution(getSecRan(), lower, upper).sample(); + } + + /** {@inheritDoc} */ + public long nextSecureLong(final long lower, final long upper) throws NumberIsTooLargeException { + if (lower >= upper) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, + lower, upper, false); + } + final RandomGenerator rng = getSecRan(); + final long max = (upper - lower) + 1; + if (max <= 0) { + // the range is too wide to fit in a positive long (larger than 2^63); as it covers + // more than half the long range, we use directly a simple rejection method + while (true) { + final long r = rng.nextLong(); + if (r >= lower && r <= upper) { + return r; + } + } + } else if (max < Integer.MAX_VALUE){ + // we can shift the range and generate directly a positive int + return lower + rng.nextInt((int) max); + } else { + // we can shift the range and generate directly a positive long + return lower + nextLong(rng, max); + } + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: + *

  • For small means, uses simulation of a Poisson process + * using Uniform deviates, as described + * here. + * The Poisson process (and hence value returned) is bounded by 1000 * mean.
    • + * + *
  • For large means, uses the rejection algorithm described in
    + * Devroye, Luc. (1981).The Computer Generation of Poisson Random Variables + * Computing vol. 26 pp. 197-207.

+ * @throws NotStrictlyPositiveException if {@code len <= 0} + */ + public long nextPoisson(double mean) throws NotStrictlyPositiveException { + return new PoissonDistribution(getRandomGenerator(), mean, + PoissonDistribution.DEFAULT_EPSILON, + PoissonDistribution.DEFAULT_MAX_ITERATIONS).sample(); + } + + /** {@inheritDoc} */ + public double nextGaussian(double mu, double sigma) throws NotStrictlyPositiveException { + if (sigma <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.STANDARD_DEVIATION, sigma); + } + return sigma * getRandomGenerator().nextGaussian() + mu; + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: Uses the Algorithm SA (Ahrens) + * from p. 876 in: + * [1]: Ahrens, J. H. and Dieter, U. (1972). Computer methods for + * sampling from the exponential and normal distributions. + * Communications of the ACM, 15, 873-882. + *

+ */ + public double nextExponential(double mean) throws NotStrictlyPositiveException { + return new ExponentialDistribution(getRandomGenerator(), mean, + ExponentialDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + *

Generates a random value from the + * {@link infodynamics.utils.commonsmath3.distribution.GammaDistribution Gamma Distribution}.

+ * + *

This implementation uses the following algorithms:

+ * + *

For 0 < shape < 1:
+ * Ahrens, J. H. and Dieter, U., Computer methods for + * sampling from gamma, beta, Poisson and binomial distributions. + * Computing, 12, 223-246, 1974.

+ * + *

For shape >= 1:
+ * Marsaglia and Tsang, A Simple Method for Generating + * Gamma Variables. ACM Transactions on Mathematical Software, + * Volume 26 Issue 3, September, 2000.

+ * + * @param shape the median of the Gamma distribution + * @param scale the scale parameter of the Gamma distribution + * @return random value sampled from the Gamma(shape, scale) distribution + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + */ + public double nextGamma(double shape, double scale) throws NotStrictlyPositiveException { + return new GammaDistribution(getRandomGenerator(),shape, scale, + GammaDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link HypergeometricDistribution Hypergeometric Distribution}. + * + * @param populationSize the population size of the Hypergeometric distribution + * @param numberOfSuccesses number of successes in the population of the Hypergeometric distribution + * @param sampleSize the sample size of the Hypergeometric distribution + * @return random value sampled from the Hypergeometric(numberOfSuccesses, sampleSize) distribution + * @throws NumberIsTooLargeException if {@code numberOfSuccesses > populationSize}, + * or {@code sampleSize > populationSize}. + * @throws NotStrictlyPositiveException if {@code populationSize <= 0}. + * @throws NotPositiveException if {@code numberOfSuccesses < 0}. + */ + public int nextHypergeometric(int populationSize, int numberOfSuccesses, int sampleSize) throws NotPositiveException, NotStrictlyPositiveException, NumberIsTooLargeException { + return new HypergeometricDistribution(getRandomGenerator(),populationSize, + numberOfSuccesses, sampleSize).sample(); + } + + /** + * Generates a random value from the {@link PascalDistribution Pascal Distribution}. + * + * @param r the number of successes of the Pascal distribution + * @param p the probability of success of the Pascal distribution + * @return random value sampled from the Pascal(r, p) distribution + * @throws NotStrictlyPositiveException if the number of successes is not positive + * @throws OutOfRangeException if the probability of success is not in the + * range {@code [0, 1]}. + */ + public int nextPascal(int r, double p) throws NotStrictlyPositiveException, OutOfRangeException { + return new PascalDistribution(getRandomGenerator(), r, p).sample(); + } + + /** + * Generates a random value from the {@link TDistribution T Distribution}. + * + * @param df the degrees of freedom of the T distribution + * @return random value from the T(df) distribution + * @throws NotStrictlyPositiveException if {@code df <= 0} + */ + public double nextT(double df) throws NotStrictlyPositiveException { + return new TDistribution(getRandomGenerator(), df, + TDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link WeibullDistribution Weibull Distribution}. + * + * @param shape the shape parameter of the Weibull distribution + * @param scale the scale parameter of the Weibull distribution + * @return random value sampled from the Weibull(shape, size) distribution + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + */ + public double nextWeibull(double shape, double scale) throws NotStrictlyPositiveException { + return new WeibullDistribution(getRandomGenerator(), shape, scale, + WeibullDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link ZipfDistribution Zipf Distribution}. + * + * @param numberOfElements the number of elements of the ZipfDistribution + * @param exponent the exponent of the ZipfDistribution + * @return random value sampled from the Zipf(numberOfElements, exponent) distribution + * @exception NotStrictlyPositiveException if {@code numberOfElements <= 0} + * or {@code exponent <= 0}. + */ + public int nextZipf(int numberOfElements, double exponent) throws NotStrictlyPositiveException { + return new ZipfDistribution(getRandomGenerator(), numberOfElements, exponent).sample(); + } + + /** + * Generates a random value from the {@link BetaDistribution Beta Distribution}. + * + * @param alpha first distribution shape parameter + * @param beta second distribution shape parameter + * @return random value sampled from the beta(alpha, beta) distribution + */ + public double nextBeta(double alpha, double beta) { + return new BetaDistribution(getRandomGenerator(), alpha, beta, + BetaDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link BinomialDistribution Binomial Distribution}. + * + * @param numberOfTrials number of trials of the Binomial distribution + * @param probabilityOfSuccess probability of success of the Binomial distribution + * @return random value sampled from the Binomial(numberOfTrials, probabilityOfSuccess) distribution + */ + public int nextBinomial(int numberOfTrials, double probabilityOfSuccess) { + return new BinomialDistribution(getRandomGenerator(), numberOfTrials, probabilityOfSuccess).sample(); + } + + /** + * Generates a random value from the {@link CauchyDistribution Cauchy Distribution}. + * + * @param median the median of the Cauchy distribution + * @param scale the scale parameter of the Cauchy distribution + * @return random value sampled from the Cauchy(median, scale) distribution + */ + public double nextCauchy(double median, double scale) { + return new CauchyDistribution(getRandomGenerator(), median, scale, + CauchyDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link ChiSquaredDistribution ChiSquare Distribution}. + * + * @param df the degrees of freedom of the ChiSquare distribution + * @return random value sampled from the ChiSquare(df) distribution + */ + public double nextChiSquare(double df) { + return new ChiSquaredDistribution(getRandomGenerator(), df, + ChiSquaredDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * Generates a random value from the {@link FDistribution F Distribution}. + * + * @param numeratorDf the numerator degrees of freedom of the F distribution + * @param denominatorDf the denominator degrees of freedom of the F distribution + * @return random value sampled from the F(numeratorDf, denominatorDf) distribution + * @throws NotStrictlyPositiveException if + * {@code numeratorDf <= 0} or {@code denominatorDf <= 0}. + */ + public double nextF(double numeratorDf, double denominatorDf) throws NotStrictlyPositiveException { + return new FDistribution(getRandomGenerator(), numeratorDf, denominatorDf, + FDistribution.DEFAULT_INVERSE_ABSOLUTE_ACCURACY).sample(); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: scales the output of + * Random.nextDouble(), but rejects 0 values (i.e., will generate another + * random double if Random.nextDouble() returns 0). This is necessary to + * provide a symmetric output interval (both endpoints excluded). + *

+ * @throws NumberIsTooLargeException if {@code lower >= upper} + * @throws NotFiniteNumberException if one of the bounds is infinite + * @throws NotANumberException if one of the bounds is NaN + */ + public double nextUniform(double lower, double upper) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException { + return nextUniform(lower, upper, false); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: if the lower bound is excluded, + * scales the output of Random.nextDouble(), but rejects 0 values (i.e., + * will generate another random double if Random.nextDouble() returns 0). + * This is necessary to provide a symmetric output interval (both + * endpoints excluded). + *

+ * + * @throws NumberIsTooLargeException if {@code lower >= upper} + * @throws NotFiniteNumberException if one of the bounds is infinite + * @throws NotANumberException if one of the bounds is NaN + */ + public double nextUniform(double lower, double upper, boolean lowerInclusive) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException { + + if (lower >= upper) { + throw new NumberIsTooLargeException(LocalizedFormats.LOWER_BOUND_NOT_BELOW_UPPER_BOUND, + lower, upper, false); + } + + if (Double.isInfinite(lower)) { + throw new NotFiniteNumberException(LocalizedFormats.INFINITE_BOUND, lower); + } + if (Double.isInfinite(upper)) { + throw new NotFiniteNumberException(LocalizedFormats.INFINITE_BOUND, upper); + } + + if (Double.isNaN(lower) || Double.isNaN(upper)) { + throw new NotANumberException(); + } + + final RandomGenerator generator = getRandomGenerator(); + + // ensure nextDouble() isn't 0.0 + double u = generator.nextDouble(); + while (!lowerInclusive && u <= 0.0) { + u = generator.nextDouble(); + } + + return u * upper + (1.0 - u) * lower; + } + + /** + * {@inheritDoc} + * + * This method calls {@link MathArrays#shuffle(int[],RandomGenerator) + * MathArrays.shuffle} in order to create a random shuffle of the set + * of natural numbers {@code { 0, 1, ..., n - 1 }}. + * + * @throws NumberIsTooLargeException if {@code k > n}. + * @throws NotStrictlyPositiveException if {@code k <= 0}. + */ + public int[] nextPermutation(int n, int k) + throws NumberIsTooLargeException, NotStrictlyPositiveException { + if (k > n) { + throw new NumberIsTooLargeException(LocalizedFormats.PERMUTATION_EXCEEDS_N, + k, n, true); + } + if (k <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.PERMUTATION_SIZE, + k); + } + + int[] index = MathArrays.natural(n); + MathArrays.shuffle(index, getRandomGenerator()); + + // Return a new array containing the first "k" entries of "index". + return MathArrays.copyOf(index, k); + } + + /** + * {@inheritDoc} + * + * This method calls {@link #nextPermutation(int,int) nextPermutation(c.size(), k)} + * in order to sample the collection. + */ + public Object[] nextSample(Collection c, int k) throws NumberIsTooLargeException, NotStrictlyPositiveException { + + int len = c.size(); + if (k > len) { + throw new NumberIsTooLargeException(LocalizedFormats.SAMPLE_SIZE_EXCEEDS_COLLECTION_SIZE, + k, len, true); + } + if (k <= 0) { + throw new NotStrictlyPositiveException(LocalizedFormats.NUMBER_OF_SAMPLES, k); + } + + Object[] objects = c.toArray(); + int[] index = nextPermutation(len, k); + Object[] result = new Object[k]; + for (int i = 0; i < k; i++) { + result[i] = objects[index[i]]; + } + return result; + } + + + + /** + * Reseeds the random number generator with the supplied seed. + *

+ * Will create and initialize if null. + *

+ * + * @param seed the seed value to use + */ + public void reSeed(long seed) { + getRandomGenerator().setSeed(seed); + } + + /** + * Reseeds the secure random number generator with the current time in + * milliseconds. + *

+ * Will create and initialize if null. + *

+ */ + public void reSeedSecure() { + getSecRan().setSeed(System.currentTimeMillis()); + } + + /** + * Reseeds the secure random number generator with the supplied seed. + *

+ * Will create and initialize if null. + *

+ * + * @param seed the seed value to use + */ + public void reSeedSecure(long seed) { + getSecRan().setSeed(seed); + } + + /** + * Reseeds the random number generator with + * {@code System.currentTimeMillis() + System.identityHashCode(this))}. + */ + public void reSeed() { + getRandomGenerator().setSeed(System.currentTimeMillis() + System.identityHashCode(this)); + } + + /** + * Sets the PRNG algorithm for the underlying SecureRandom instance using + * the Security Provider API. The Security Provider API is defined in + * Java Cryptography Architecture API Specification & Reference. + *

+ * USAGE NOTE: This method carries significant + * overhead and may take several seconds to execute. + *

+ * + * @param algorithm the name of the PRNG algorithm + * @param provider the name of the provider + * @throws NoSuchAlgorithmException if the specified algorithm is not available + * @throws NoSuchProviderException if the specified provider is not installed + */ + public void setSecureAlgorithm(String algorithm, String provider) + throws NoSuchAlgorithmException, NoSuchProviderException { + secRand = RandomGeneratorFactory.createRandomGenerator(SecureRandom.getInstance(algorithm, provider)); + } + + /** + * Returns the RandomGenerator used to generate non-secure random data. + *

+ * Creates and initializes a default generator if null. Uses a {@link Well19937c} + * generator with {@code System.currentTimeMillis() + System.identityHashCode(this))} + * as the default seed. + *

+ * + * @return the Random used to generate random data + * @since 3.2 + */ + public RandomGenerator getRandomGenerator() { + if (rand == null) { + initRan(); + } + return rand; + } + + /** + * Sets the default generator to a {@link Well19937c} generator seeded with + * {@code System.currentTimeMillis() + System.identityHashCode(this))}. + */ + private void initRan() { + rand = new Well19937c(System.currentTimeMillis() + System.identityHashCode(this)); + } + + /** + * Returns the SecureRandom used to generate secure random data. + *

+ * Creates and initializes if null. Uses + * {@code System.currentTimeMillis() + System.identityHashCode(this)} as the default seed. + *

+ * + * @return the SecureRandom used to generate secure random data, wrapped in a + * {@link RandomGenerator}. + */ + private RandomGenerator getSecRan() { + if (secRand == null) { + secRand = RandomGeneratorFactory.createRandomGenerator(new SecureRandom()); + secRand.setSeed(System.currentTimeMillis() + System.identityHashCode(this)); + } + return secRand; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/RandomDataImpl.java b/java/source/infodynamics/utils/commonsmath3/random/RandomDataImpl.java new file mode 100644 index 0000000..a3f9139 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/RandomDataImpl.java @@ -0,0 +1,614 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package infodynamics.utils.commonsmath3.random; + +import java.io.Serializable; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.util.Collection; + +import infodynamics.utils.commonsmath3.distribution.IntegerDistribution; +import infodynamics.utils.commonsmath3.distribution.RealDistribution; +import infodynamics.utils.commonsmath3.exception.NotANumberException; +import infodynamics.utils.commonsmath3.exception.NotFiniteNumberException; +import infodynamics.utils.commonsmath3.exception.NotPositiveException; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; +import infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException; +import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; + +/** + * Generates random deviates and other random data using a {@link RandomGenerator} + * instance to generate non-secure data and a {@link java.security.SecureRandom} + * instance to provide data for the nextSecureXxx methods. If no + * RandomGenerator is provided in the constructor, the default is + * to use a {@link Well19937c} generator. To plug in a different + * implementation, either implement RandomGenerator directly or + * extend {@link AbstractRandomGenerator}. + *

+ * Supports reseeding the underlying pseudo-random number generator (PRNG). The + * SecurityProvider and Algorithm used by the + * SecureRandom instance can also be reset. + *

+ *

+ * For details on the default PRNGs, see {@link java.util.Random} and + * {@link java.security.SecureRandom}. + *

+ *

+ * Usage Notes: + *

    + *
  • + * Instance variables are used to maintain RandomGenerator and + * SecureRandom instances used in data generation. Therefore, to + * generate a random sequence of values or strings, you should use just + * one RandomDataGenerator instance repeatedly.
    • + *
  • + * The "secure" methods are *much* slower. These should be used only when a + * cryptographically secure random sequence is required. A secure random + * sequence is a sequence of pseudo-random values which, in addition to being + * well-dispersed (so no subsequence of values is an any more likely than other + * subsequence of the the same length), also has the additional property that + * knowledge of values generated up to any point in the sequence does not make + * it any easier to predict subsequent values.
    • + *
  • + * When a new RandomDataGenerator is created, the underlying random + * number generators are not initialized. If you do not + * explicitly seed the default non-secure generator, it is seeded with the + * current time in milliseconds plus the system identity hash code on first use. + * The same holds for the secure generator. If you provide a RandomGenerator + * to the constructor, however, this generator is not reseeded by the constructor + * nor is it reseeded on first use.
    • + *
  • + * The reSeed and reSeedSecure methods delegate to the + * corresponding methods on the underlying RandomGenerator and + * SecureRandom instances. Therefore, reSeed(long) + * fully resets the initial state of the non-secure random number generator (so + * that reseeding with a specific value always results in the same subsequent + * random sequence); whereas reSeedSecure(long) does not + * reinitialize the secure random number generator (so secure sequences started + * with calls to reseedSecure(long) won't be identical).
    • + *
  • + * This implementation is not synchronized. The underlying RandomGenerator + * or SecureRandom instances are not protected by synchronization and + * are not guaranteed to be thread-safe. Therefore, if an instance of this class + * is concurrently utilized by multiple threads, it is the responsibility of + * client code to synchronize access to seeding and data generation methods. + *
    • + *
+ *

+ * @deprecated to be removed in 4.0. Use {@link RandomDataGenerator} instead + */ +@Deprecated +public class RandomDataImpl implements RandomData, Serializable { + + /** Serializable version identifier */ + private static final long serialVersionUID = -626730818244969716L; + + /** RandomDataGenerator delegate */ + private final RandomDataGenerator delegate; + + /** + * Construct a RandomDataImpl, using a default random generator as the source + * of randomness. + * + *

The default generator is a {@link Well19937c} seeded + * with {@code System.currentTimeMillis() + System.identityHashCode(this))}. + * The generator is initialized and seeded on first use.

+ */ + public RandomDataImpl() { + delegate = new RandomDataGenerator(); + } + + /** + * Construct a RandomDataImpl using the supplied {@link RandomGenerator} as + * the source of (non-secure) random data. + * + * @param rand the source of (non-secure) random data + * (may be null, resulting in the default generator) + * @since 1.1 + */ + public RandomDataImpl(RandomGenerator rand) { + delegate = new RandomDataGenerator(rand); + } + + /** + * @return the delegate object. + * @deprecated To be removed in 4.0. + */ + @Deprecated + RandomDataGenerator getDelegate() { + return delegate; + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: hex strings are generated using a + * 2-step process. + *

    + *
  1. {@code len / 2 + 1} binary bytes are generated using the underlying + * Random
    2. + *
  2. Each binary byte is translated into 2 hex digits
    3. + *
+ *

+ * + * @param len the desired string length. + * @return the random string. + * @throws NotStrictlyPositiveException if {@code len <= 0}. + */ + public String nextHexString(int len) throws NotStrictlyPositiveException { + return delegate.nextHexString(len); + } + + /** {@inheritDoc} */ + public int nextInt(int lower, int upper) throws NumberIsTooLargeException { + return delegate.nextInt(lower, upper); + } + + /** {@inheritDoc} */ + public long nextLong(long lower, long upper) throws NumberIsTooLargeException { + return delegate.nextLong(lower, upper); + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: hex strings are generated in + * 40-byte segments using a 3-step process. + *

    + *
  1. + * 20 random bytes are generated using the underlying + * SecureRandom.
    2. + *
  2. + * SHA-1 hash is applied to yield a 20-byte binary digest.
    3. + *
  3. + * Each byte of the binary digest is converted to 2 hex digits.
    4. + *
+ *

+ */ + public String nextSecureHexString(int len) throws NotStrictlyPositiveException { + return delegate.nextSecureHexString(len); + } + + /** {@inheritDoc} */ + public int nextSecureInt(int lower, int upper) throws NumberIsTooLargeException { + return delegate.nextSecureInt(lower, upper); + } + + /** {@inheritDoc} */ + public long nextSecureLong(long lower, long upper) throws NumberIsTooLargeException { + return delegate.nextSecureLong(lower,upper); + } + + /** + * {@inheritDoc} + *

+ * Algorithm Description: + *

  • For small means, uses simulation of a Poisson process + * using Uniform deviates, as described + * here. + * The Poisson process (and hence value returned) is bounded by 1000 * mean.
    • + * + *
  • For large means, uses the rejection algorithm described in
    + * Devroye, Luc. (1981).The Computer Generation of Poisson Random Variables + * Computing vol. 26 pp. 197-207.

+ */ + public long nextPoisson(double mean) throws NotStrictlyPositiveException { + return delegate.nextPoisson(mean); + } + + /** {@inheritDoc} */ + public double nextGaussian(double mu, double sigma) throws NotStrictlyPositiveException { + return delegate.nextGaussian(mu,sigma); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: Uses the Algorithm SA (Ahrens) + * from p. 876 in: + * [1]: Ahrens, J. H. and Dieter, U. (1972). Computer methods for + * sampling from the exponential and normal distributions. + * Communications of the ACM, 15, 873-882. + *

+ */ + public double nextExponential(double mean) throws NotStrictlyPositiveException { + return delegate.nextExponential(mean); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: scales the output of + * Random.nextDouble(), but rejects 0 values (i.e., will generate another + * random double if Random.nextDouble() returns 0). This is necessary to + * provide a symmetric output interval (both endpoints excluded). + *

+ */ + public double nextUniform(double lower, double upper) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException { + return delegate.nextUniform(lower, upper); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: if the lower bound is excluded, + * scales the output of Random.nextDouble(), but rejects 0 values (i.e., + * will generate another random double if Random.nextDouble() returns 0). + * This is necessary to provide a symmetric output interval (both + * endpoints excluded). + *

+ * @since 3.0 + */ + public double nextUniform(double lower, double upper, boolean lowerInclusive) + throws NumberIsTooLargeException, NotFiniteNumberException, NotANumberException { + return delegate.nextUniform(lower, upper, lowerInclusive); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.BetaDistribution Beta Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param alpha first distribution shape parameter + * @param beta second distribution shape parameter + * @return random value sampled from the beta(alpha, beta) distribution + * @since 2.2 + */ + public double nextBeta(double alpha, double beta) { + return delegate.nextBeta(alpha, beta); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.BinomialDistribution Binomial Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param numberOfTrials number of trials of the Binomial distribution + * @param probabilityOfSuccess probability of success of the Binomial distribution + * @return random value sampled from the Binomial(numberOfTrials, probabilityOfSuccess) distribution + * @since 2.2 + */ + public int nextBinomial(int numberOfTrials, double probabilityOfSuccess) { + return delegate.nextBinomial(numberOfTrials, probabilityOfSuccess); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.CauchyDistribution Cauchy Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param median the median of the Cauchy distribution + * @param scale the scale parameter of the Cauchy distribution + * @return random value sampled from the Cauchy(median, scale) distribution + * @since 2.2 + */ + public double nextCauchy(double median, double scale) { + return delegate.nextCauchy(median, scale); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.ChiSquaredDistribution ChiSquare Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param df the degrees of freedom of the ChiSquare distribution + * @return random value sampled from the ChiSquare(df) distribution + * @since 2.2 + */ + public double nextChiSquare(double df) { + return delegate.nextChiSquare(df); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.FDistribution F Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param numeratorDf the numerator degrees of freedom of the F distribution + * @param denominatorDf the denominator degrees of freedom of the F distribution + * @return random value sampled from the F(numeratorDf, denominatorDf) distribution + * @throws NotStrictlyPositiveException if + * {@code numeratorDf <= 0} or {@code denominatorDf <= 0}. + * @since 2.2 + */ + public double nextF(double numeratorDf, double denominatorDf) throws NotStrictlyPositiveException { + return delegate.nextF(numeratorDf, denominatorDf); + } + + /** + *

Generates a random value from the + * {@link infodynamics.utils.commonsmath3.distribution.GammaDistribution Gamma Distribution}.

+ * + *

This implementation uses the following algorithms:

+ * + *

For 0 < shape < 1:
+ * Ahrens, J. H. and Dieter, U., Computer methods for + * sampling from gamma, beta, Poisson and binomial distributions. + * Computing, 12, 223-246, 1974.

+ * + *

For shape >= 1:
+ * Marsaglia and Tsang, A Simple Method for Generating + * Gamma Variables. ACM Transactions on Mathematical Software, + * Volume 26 Issue 3, September, 2000.

+ * + * @param shape the median of the Gamma distribution + * @param scale the scale parameter of the Gamma distribution + * @return random value sampled from the Gamma(shape, scale) distribution + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + * @since 2.2 + */ + public double nextGamma(double shape, double scale) throws NotStrictlyPositiveException { + return delegate.nextGamma(shape, scale); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.HypergeometricDistribution Hypergeometric Distribution}. + * This implementation uses {@link #nextInversionDeviate(IntegerDistribution) inversion} + * to generate random values. + * + * @param populationSize the population size of the Hypergeometric distribution + * @param numberOfSuccesses number of successes in the population of the Hypergeometric distribution + * @param sampleSize the sample size of the Hypergeometric distribution + * @return random value sampled from the Hypergeometric(numberOfSuccesses, sampleSize) distribution + * @throws NumberIsTooLargeException if {@code numberOfSuccesses > populationSize}, + * or {@code sampleSize > populationSize}. + * @throws NotStrictlyPositiveException if {@code populationSize <= 0}. + * @throws NotPositiveException if {@code numberOfSuccesses < 0}. + * @since 2.2 + */ + public int nextHypergeometric(int populationSize, int numberOfSuccesses, int sampleSize) + throws NotPositiveException, NotStrictlyPositiveException, NumberIsTooLargeException { + return delegate.nextHypergeometric(populationSize, numberOfSuccesses, sampleSize); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.PascalDistribution Pascal Distribution}. + * This implementation uses {@link #nextInversionDeviate(IntegerDistribution) inversion} + * to generate random values. + * + * @param r the number of successes of the Pascal distribution + * @param p the probability of success of the Pascal distribution + * @return random value sampled from the Pascal(r, p) distribution + * @since 2.2 + * @throws NotStrictlyPositiveException if the number of successes is not positive + * @throws OutOfRangeException if the probability of success is not in the + * range {@code [0, 1]}. + */ + public int nextPascal(int r, double p) + throws NotStrictlyPositiveException, OutOfRangeException { + return delegate.nextPascal(r, p); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.TDistribution T Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param df the degrees of freedom of the T distribution + * @return random value from the T(df) distribution + * @since 2.2 + * @throws NotStrictlyPositiveException if {@code df <= 0} + */ + public double nextT(double df) throws NotStrictlyPositiveException { + return delegate.nextT(df); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.WeibullDistribution Weibull Distribution}. + * This implementation uses {@link #nextInversionDeviate(RealDistribution) inversion} + * to generate random values. + * + * @param shape the shape parameter of the Weibull distribution + * @param scale the scale parameter of the Weibull distribution + * @return random value sampled from the Weibull(shape, size) distribution + * @since 2.2 + * @throws NotStrictlyPositiveException if {@code shape <= 0} or + * {@code scale <= 0}. + */ + public double nextWeibull(double shape, double scale) throws NotStrictlyPositiveException { + return delegate.nextWeibull(shape, scale); + } + + /** + * Generates a random value from the {@link infodynamics.utils.commonsmath3.distribution.ZipfDistribution Zipf Distribution}. + * This implementation uses {@link #nextInversionDeviate(IntegerDistribution) inversion} + * to generate random values. + * + * @param numberOfElements the number of elements of the ZipfDistribution + * @param exponent the exponent of the ZipfDistribution + * @return random value sampled from the Zipf(numberOfElements, exponent) distribution + * @since 2.2 + * @exception NotStrictlyPositiveException if {@code numberOfElements <= 0} + * or {@code exponent <= 0}. + */ + public int nextZipf(int numberOfElements, double exponent) throws NotStrictlyPositiveException { + return delegate.nextZipf(numberOfElements, exponent); + } + + + /** + * Reseeds the random number generator with the supplied seed. + *

+ * Will create and initialize if null. + *

+ * + * @param seed + * the seed value to use + */ + public void reSeed(long seed) { + delegate.reSeed(seed); + } + + /** + * Reseeds the secure random number generator with the current time in + * milliseconds. + *

+ * Will create and initialize if null. + *

+ */ + public void reSeedSecure() { + delegate.reSeedSecure(); + } + + /** + * Reseeds the secure random number generator with the supplied seed. + *

+ * Will create and initialize if null. + *

+ * + * @param seed + * the seed value to use + */ + public void reSeedSecure(long seed) { + delegate.reSeedSecure(seed); + } + + /** + * Reseeds the random number generator with + * {@code System.currentTimeMillis() + System.identityHashCode(this))}. + */ + public void reSeed() { + delegate.reSeed(); + } + + /** + * Sets the PRNG algorithm for the underlying SecureRandom instance using + * the Security Provider API. The Security Provider API is defined in + * Java Cryptography Architecture API Specification & Reference. + *

+ * USAGE NOTE: This method carries significant + * overhead and may take several seconds to execute. + *

+ * + * @param algorithm + * the name of the PRNG algorithm + * @param provider + * the name of the provider + * @throws NoSuchAlgorithmException + * if the specified algorithm is not available + * @throws NoSuchProviderException + * if the specified provider is not installed + */ + public void setSecureAlgorithm(String algorithm, String provider) + throws NoSuchAlgorithmException, NoSuchProviderException { + delegate.setSecureAlgorithm(algorithm, provider); + } + + /** + * {@inheritDoc} + * + *

+ * Uses a 2-cycle permutation shuffle. The shuffling process is described + * here. + *

+ */ + public int[] nextPermutation(int n, int k) + throws NotStrictlyPositiveException, NumberIsTooLargeException { + return delegate.nextPermutation(n, k); + } + + /** + * {@inheritDoc} + * + *

+ * Algorithm Description: Uses a 2-cycle permutation + * shuffle to generate a random permutation of c.size() and + * then returns the elements whose indexes correspond to the elements of the + * generated permutation. This technique is described, and proven to + * generate random samples + * here + *

+ */ + public Object[] nextSample(Collection c, int k) + throws NotStrictlyPositiveException, NumberIsTooLargeException { + return delegate.nextSample(c, k); + } + + /** + * Generate a random deviate from the given distribution using the + * inversion method. + * + * @param distribution Continuous distribution to generate a random value from + * @return a random value sampled from the given distribution + * @throws MathIllegalArgumentException if the underlynig distribution throws one + * @since 2.2 + * @deprecated use the distribution's sample() method + */ + @Deprecated + public double nextInversionDeviate(RealDistribution distribution) + throws MathIllegalArgumentException { + return distribution.inverseCumulativeProbability(nextUniform(0, 1)); + + } + + /** + * Generate a random deviate from the given distribution using the + * inversion method. + * + * @param distribution Integer distribution to generate a random value from + * @return a random value sampled from the given distribution + * @throws MathIllegalArgumentException if the underlynig distribution throws one + * @since 2.2 + * @deprecated use the distribution's sample() method + */ + @Deprecated + public int nextInversionDeviate(IntegerDistribution distribution) + throws MathIllegalArgumentException { + return distribution.inverseCumulativeProbability(nextUniform(0, 1)); + } + +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/RandomGenerator.java b/java/source/infodynamics/utils/commonsmath3/random/RandomGenerator.java new file mode 100644 index 0000000..7dd3bb0 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/RandomGenerator.java @@ -0,0 +1,176 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.random; + + +/** + * Interface extracted from java.util.Random. This interface is + * implemented by {@link AbstractRandomGenerator}. + * + * @since 1.1 + */ +public interface RandomGenerator { + + /** + * Sets the seed of the underlying random number generator using an + * int seed. + *

Sequences of values generated starting with the same seeds + * should be identical. + *

+ * @param seed the seed value + */ + void setSeed(int seed); + + /** + * Sets the seed of the underlying random number generator using an + * int array seed. + *

Sequences of values generated starting with the same seeds + * should be identical. + *

+ * @param seed the seed value + */ + void setSeed(int[] seed); + + /** + * Sets the seed of the underlying random number generator using a + * long seed. + *

Sequences of values generated starting with the same seeds + * should be identical. + *

+ * @param seed the seed value + */ + void setSeed(long seed); + + /** + * Generates random bytes and places them into a user-supplied + * byte array. The number of random bytes produced is equal to + * the length of the byte array. + * + * @param bytes the non-null byte array in which to put the + * random bytes + */ + void nextBytes(byte[] bytes); + + /** + * Returns the next pseudorandom, uniformly distributed int + * value from this random number generator's sequence. + * All 232 possible {@code int} values + * should be produced with (approximately) equal probability. + * + * @return the next pseudorandom, uniformly distributed int + * value from this random number generator's sequence + */ + int nextInt(); + + /** + * Returns a pseudorandom, uniformly distributed {@code int} value + * between 0 (inclusive) and the specified value (exclusive), drawn from + * this random number generator's sequence. + * + * @param n the bound on the random number to be returned. Must be + * positive. + * @return a pseudorandom, uniformly distributed {@code int} + * value between 0 (inclusive) and n (exclusive). + * @throws IllegalArgumentException if n is not positive. + */ + int nextInt(int n); + + /** + * Returns the next pseudorandom, uniformly distributed long + * value from this random number generator's sequence. All + * 264 possible {@code long} values + * should be produced with (approximately) equal probability. + * + * @return the next pseudorandom, uniformly distributed long + *value from this random number generator's sequence + */ + long nextLong(); + + /** + * Returns the next pseudorandom, uniformly distributed + * boolean value from this random number generator's + * sequence. + * + * @return the next pseudorandom, uniformly distributed + * boolean value from this random number generator's + * sequence + */ + boolean nextBoolean(); + + /** + * Returns the next pseudorandom, uniformly distributed float + * value between 0.0 and 1.0 from this random + * number generator's sequence. + * + * @return the next pseudorandom, uniformly distributed float + * value between 0.0 and 1.0 from this + * random number generator's sequence + */ + float nextFloat(); + + /** + * Returns the next pseudorandom, uniformly distributed + * double value between 0.0 and + * 1.0 from this random number generator's sequence. + * + * @return the next pseudorandom, uniformly distributed + * double value between 0.0 and + * 1.0 from this random number generator's sequence + */ + double nextDouble(); + + /** + * Returns the next pseudorandom, Gaussian ("normally") distributed + * double value with mean 0.0 and standard + * deviation 1.0 from this random number generator's sequence. + * + * @return the next pseudorandom, Gaussian ("normally") distributed + * double value with mean 0.0 and + * standard deviation 1.0 from this random number + * generator's sequence + */ + double nextGaussian(); +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/RandomGeneratorFactory.java b/java/source/infodynamics/utils/commonsmath3/random/RandomGeneratorFactory.java new file mode 100644 index 0000000..64bd1f5 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/RandomGeneratorFactory.java @@ -0,0 +1,150 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.random; + +import java.util.Random; +import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException; + +/** + * Utilities for creating {@link RandomGenerator} instances. + * + * @since 3.3 + */ +public class RandomGeneratorFactory { + /** + * Class contains only static methods. + */ + private RandomGeneratorFactory() {} + + /** + * Creates a {@link RandomDataGenerator} instance that wraps a + * {@link Random} instance. + * + * @param rng JDK {@link Random} instance that will generate the + * the random data. + * @return the given RNG, wrapped in a {@link RandomGenerator}. + */ + public static RandomGenerator createRandomGenerator(final Random rng) { + return new RandomGenerator() { + /** {@inheritDoc} */ + public void setSeed(int seed) { + rng.setSeed((long) seed); + } + + /** {@inheritDoc} */ + public void setSeed(int[] seed) { + rng.setSeed(convertToLong(seed)); + } + + /** {@inheritDoc} */ + public void setSeed(long seed) { + rng.setSeed(seed); + } + + /** {@inheritDoc} */ + public void nextBytes(byte[] bytes) { + rng.nextBytes(bytes); + } + + /** {@inheritDoc} */ + public int nextInt() { + return rng.nextInt(); + } + + /** {@inheritDoc} */ + public int nextInt(int n) { + if (n <= 0) { + throw new NotStrictlyPositiveException(n); + } + return rng.nextInt(n); + } + + /** {@inheritDoc} */ + public long nextLong() { + return rng.nextLong(); + } + + /** {@inheritDoc} */ + public boolean nextBoolean() { + return rng.nextBoolean(); + } + + /** {@inheritDoc} */ + public float nextFloat() { + return rng.nextFloat(); + } + + /** {@inheritDoc} */ + public double nextDouble() { + return rng.nextDouble(); + } + + /** {@inheritDoc} */ + public double nextGaussian() { + return rng.nextGaussian(); + } + }; + } + + /** + * Converts seed from one representation to another. + * + * @param seed Original seed. + * @return the converted seed. + */ + public static long convertToLong(int[] seed) { + // The following number is the largest prime that fits + // in 32 bits (i.e. 2^32 - 5). + final long prime = 4294967291l; + + long combined = 0l; + for (int s : seed) { + combined = combined * prime + s; + } + + return combined; + } +} diff --git a/java/source/infodynamics/utils/commonsmath3/random/Well19937c.java b/java/source/infodynamics/utils/commonsmath3/random/Well19937c.java new file mode 100644 index 0000000..4b5a65a --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/random/Well19937c.java @@ -0,0 +1,143 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.random; + + +/** This class implements the WELL19937c pseudo-random number generator + * from François Panneton, Pierre L'Ecuyer and Makoto Matsumoto. + + *

This generator is described in a paper by François Panneton, + * Pierre L'Ecuyer and Makoto Matsumoto Improved + * Long-Period Generators Based on Linear Recurrences Modulo 2 ACM + * Transactions on Mathematical Software, 32, 1 (2006). The errata for the paper + * are in wellrng-errata.txt.

+ + * @see WELL Random number generator + * @since 2.2 + + */ +public class Well19937c extends AbstractWell { + + /** Serializable version identifier. */ + private static final long serialVersionUID = -7203498180754925124L; + + /** Number of bits in the pool. */ + private static final int K = 19937; + + /** First parameter of the algorithm. */ + private static final int M1 = 70; + + /** Second parameter of the algorithm. */ + private static final int M2 = 179; + + /** Third parameter of the algorithm. */ + private static final int M3 = 449; + + /** Creates a new random number generator. + *

The instance is initialized using the current time as the + * seed.

+ */ + public Well19937c() { + super(K, M1, M2, M3); + } + + /** Creates a new random number generator using a single int seed. + * @param seed the initial seed (32 bits integer) + */ + public Well19937c(int seed) { + super(K, M1, M2, M3, seed); + } + + /** Creates a new random number generator using an int array seed. + * @param seed the initial seed (32 bits integers array), if null + * the seed of the generator will be related to the current time + */ + public Well19937c(int[] seed) { + super(K, M1, M2, M3, seed); + } + + /** Creates a new random number generator using a single long seed. + * @param seed the initial seed (64 bits integer) + */ + public Well19937c(long seed) { + super(K, M1, M2, M3, seed); + } + + /** {@inheritDoc} */ + @Override + protected int next(final int bits) { + + final int indexRm1 = iRm1[index]; + final int indexRm2 = iRm2[index]; + + final int v0 = v[index]; + final int vM1 = v[i1[index]]; + final int vM2 = v[i2[index]]; + final int vM3 = v[i3[index]]; + + final int z0 = (0x80000000 & v[indexRm1]) ^ (0x7FFFFFFF & v[indexRm2]); + final int z1 = (v0 ^ (v0 << 25)) ^ (vM1 ^ (vM1 >>> 27)); + final int z2 = (vM2 >>> 9) ^ (vM3 ^ (vM3 >>> 1)); + final int z3 = z1 ^ z2; + int z4 = z0 ^ (z1 ^ (z1 << 9)) ^ (z2 ^ (z2 << 21)) ^ (z3 ^ (z3 >>> 21)); + + v[index] = z3; + v[indexRm1] = z4; + v[indexRm2] &= 0x80000000; + index = indexRm1; + + + // add Matsumoto-Kurita tempering + // to get a maximally-equidistributed generator + z4 ^= (z4 << 7) & 0xe46e1700; + z4 ^= (z4 << 15) & 0x9b868000; + + return z4 >>> (32 - bits); + + } + +} diff --git a/java/source/infodynamics/utils/commonsmath3/special/Beta.java b/java/source/infodynamics/utils/commonsmath3/special/Beta.java new file mode 100644 index 0000000..1f03e53 --- /dev/null +++ b/java/source/infodynamics/utils/commonsmath3/special/Beta.java @@ -0,0 +1,542 @@ +/* + * Java Information Dynamics Toolkit (JIDT) + * Copyright (C) 2017, Joseph T. Lizier + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +/* + * This class was originally distributed as part of the Apache Commons + * Math3 library (3.6.1), under the Apache License Version 2.0, which is + * copied below. This Apache 2 software is now included as a derivative + * work in the GPLv3 licensed JIDT project, as per: + * http://www.apache.org/licenses/GPL-compatibility.html + * + * The original Apache source code has been modified as follows: + * -- We have modified package names to sit inside the JIDT structure. + */ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package infodynamics.utils.commonsmath3.special; + +import infodynamics.utils.commonsmath3.exception.NumberIsTooSmallException; +import infodynamics.utils.commonsmath3.exception.OutOfRangeException; +import infodynamics.utils.commonsmath3.util.ContinuedFraction; +import infodynamics.utils.commonsmath3.util.FastMath; + +/** + *

+ * This is a utility class that provides computation methods related to the + * Beta family of functions. + *

+ *

+ * Implementation of {@link #logBeta(double, double)} is based on the + * algorithms described in + *

+ * and implemented in the + * NSWC Library of Mathematical Functions, + * available + * here. + * This library is "approved for public release", and the + * Copyright guidance + * indicates that unless otherwise stated in the code, all FORTRAN functions in + * this library are license free. Since no such notice appears in the code these + * functions can safely be ported to Commons-Math. + *

+ * + * + */ +public class Beta { + /** Maximum allowed numerical error. */ + private static final double DEFAULT_EPSILON = 1E-14; + + /** The constant value of ½log 2π. */ + private static final double HALF_LOG_TWO_PI = .9189385332046727; + + /** + *

+ * The coefficients of the series expansion of the Δ function. This function + * is defined as follows + *

+ *
Δ(x) = log Γ(x) - (x - 0.5) log a + a - 0.5 log 2π,
+ *

+ * see equation (23) in Didonato and Morris (1992). The series expansion, + * which applies for x ≥ 10, reads + *

+ * 
+     *                 14
+     *                ====
+     *             1  \                2 n
+     *     Δ(x) = ---  >    d  (10 / x)
+     *             x  /      n
+     *                ====
+     *                n = 0
+     * 
+     */
+    private static final double[] DELTA = {
+        .833333333333333333333333333333E-01,
+        -.277777777777777777777777752282E-04,
+        .793650793650793650791732130419E-07,
+        -.595238095238095232389839236182E-09,
+        .841750841750832853294451671990E-11,
+        -.191752691751854612334149171243E-12,
+        .641025640510325475730918472625E-14,
+        -.295506514125338232839867823991E-15,
+        .179643716359402238723287696452E-16,
+        -.139228964661627791231203060395E-17,
+        .133802855014020915603275339093E-18,
+        -.154246009867966094273710216533E-19,
+        .197701992980957427278370133333E-20,
+        -.234065664793997056856992426667E-21,
+        .171348014966398575409015466667E-22
+    };
+
+    /**
+     * Default constructor.  Prohibit instantiation.
+     */
+    private Beta() {}
+
+    /**
+     * Returns the
+     * 
+     * regularized beta function I(x, a, b).
+     *
+     * @param x Value.
+     * @param a Parameter {@code a}.
+     * @param b Parameter {@code b}.
+     * @return the regularized beta function I(x, a, b).
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     */
+    public static double regularizedBeta(double x, double a, double b) {
+        return regularizedBeta(x, a, b, DEFAULT_EPSILON, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Returns the
+     * 
+     * regularized beta function I(x, a, b).
+     *
+     * @param x Value.
+     * @param a Parameter {@code a}.
+     * @param b Parameter {@code b}.
+     * @param epsilon When the absolute value of the nth item in the
+     * series is less than epsilon the approximation ceases to calculate
+     * further elements in the series.
+     * @return the regularized beta function I(x, a, b)
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     */
+    public static double regularizedBeta(double x,
+                                         double a, double b,
+                                         double epsilon) {
+        return regularizedBeta(x, a, b, epsilon, Integer.MAX_VALUE);
+    }
+
+    /**
+     * Returns the regularized beta function I(x, a, b).
+     *
+     * @param x the value.
+     * @param a Parameter {@code a}.
+     * @param b Parameter {@code b}.
+     * @param maxIterations Maximum number of "iterations" to complete.
+     * @return the regularized beta function I(x, a, b)
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     */
+    public static double regularizedBeta(double x,
+                                         double a, double b,
+                                         int maxIterations) {
+        return regularizedBeta(x, a, b, DEFAULT_EPSILON, maxIterations);
+    }
+
+    /**
+     * Returns the regularized beta function I(x, a, b).
+     *
+     * The implementation of this method is based on:
+     * 
+     *
+     * @param x the value.
+     * @param a Parameter {@code a}.
+     * @param b Parameter {@code b}.
+     * @param epsilon When the absolute value of the nth item in the
+     * series is less than epsilon the approximation ceases to calculate
+     * further elements in the series.
+     * @param maxIterations Maximum number of "iterations" to complete.
+     * @return the regularized beta function I(x, a, b)
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     */
+    public static double regularizedBeta(double x,
+                                         final double a, final double b,
+                                         double epsilon, int maxIterations) {
+        double ret;
+
+        if (Double.isNaN(x) ||
+            Double.isNaN(a) ||
+            Double.isNaN(b) ||
+            x < 0 ||
+            x > 1 ||
+            a <= 0 ||
+            b <= 0) {
+            ret = Double.NaN;
+        } else if (x > (a + 1) / (2 + b + a) &&
+                   1 - x <= (b + 1) / (2 + b + a)) {
+            ret = 1 - regularizedBeta(1 - x, b, a, epsilon, maxIterations);
+        } else {
+            ContinuedFraction fraction = new ContinuedFraction() {
+
+                /** {@inheritDoc} */
+                @Override
+                protected double getB(int n, double x) {
+                    double ret;
+                    double m;
+                    if (n % 2 == 0) { // even
+                        m = n / 2.0;
+                        ret = (m * (b - m) * x) /
+                            ((a + (2 * m) - 1) * (a + (2 * m)));
+                    } else {
+                        m = (n - 1.0) / 2.0;
+                        ret = -((a + m) * (a + b + m) * x) /
+                                ((a + (2 * m)) * (a + (2 * m) + 1.0));
+                    }
+                    return ret;
+                }
+
+                /** {@inheritDoc} */
+                @Override
+                protected double getA(int n, double x) {
+                    return 1.0;
+                }
+            };
+            ret = FastMath.exp((a * FastMath.log(x)) + (b * FastMath.log1p(-x)) -
+                FastMath.log(a) - logBeta(a, b)) *
+                1.0 / fraction.evaluate(x, epsilon, maxIterations);
+        }
+
+        return ret;
+    }
+
+    /**
+     * Returns the natural logarithm of the beta function B(a, b).
+     *
+     * The implementation of this method is based on:
+     * 
+     *
+     * @param a Parameter {@code a}.
+     * @param b Parameter {@code b}.
+     * @param epsilon This parameter is ignored.
+     * @param maxIterations This parameter is ignored.
+     * @return log(B(a, b)).
+     * @deprecated as of version 3.1, this method is deprecated as the
+     * computation of the beta function is no longer iterative; it will be
+     * removed in version 4.0. Current implementation of this method
+     * internally calls {@link #logBeta(double, double)}.
+     */
+    @Deprecated
+    public static double logBeta(double a, double b,
+                                 double epsilon,
+                                 int maxIterations) {
+
+        return logBeta(a, b);
+    }
+
+
+    /**
+     * Returns the value of log Γ(a + b) for 1 ≤ a, b ≤ 2. Based on the
+     * NSWC Library of Mathematics Subroutines double precision
+     * implementation, {@code DGSMLN}. In {@code BetaTest.testLogGammaSum()},
+     * this private method is accessed through reflection.
+     *
+     * @param a First argument.
+     * @param b Second argument.
+     * @return the value of {@code log(Gamma(a + b))}.
+     * @throws OutOfRangeException if {@code a} or {@code b} is lower than
+     * {@code 1.0} or greater than {@code 2.0}.
+     */
+    private static double logGammaSum(final double a, final double b)
+        throws OutOfRangeException {
+
+        if ((a < 1.0) || (a > 2.0)) {
+            throw new OutOfRangeException(a, 1.0, 2.0);
+        }
+        if ((b < 1.0) || (b > 2.0)) {
+            throw new OutOfRangeException(b, 1.0, 2.0);
+        }
+
+        final double x = (a - 1.0) + (b - 1.0);
+        if (x <= 0.5) {
+            return Gamma.logGamma1p(1.0 + x);
+        } else if (x <= 1.5) {
+            return Gamma.logGamma1p(x) + FastMath.log1p(x);
+        } else {
+            return Gamma.logGamma1p(x - 1.0) + FastMath.log(x * (1.0 + x));
+        }
+    }
+
+    /**
+     * Returns the value of log[Γ(b) / Γ(a + b)] for a ≥ 0 and b ≥ 10. Based on
+     * the NSWC Library of Mathematics Subroutines double precision
+     * implementation, {@code DLGDIV}. In
+     * {@code BetaTest.testLogGammaMinusLogGammaSum()}, this private method is
+     * accessed through reflection.
+     *
+     * @param a First argument.
+     * @param b Second argument.
+     * @return the value of {@code log(Gamma(b) / Gamma(a + b))}.
+     * @throws NumberIsTooSmallException if {@code a < 0.0} or {@code b < 10.0}.
+     */
+    private static double logGammaMinusLogGammaSum(final double a,
+                                                   final double b)
+        throws NumberIsTooSmallException {
+
+        if (a < 0.0) {
+            throw new NumberIsTooSmallException(a, 0.0, true);
+        }
+        if (b < 10.0) {
+            throw new NumberIsTooSmallException(b, 10.0, true);
+        }
+
+        /*
+         * d = a + b - 0.5
+         */
+        final double d;
+        final double w;
+        if (a <= b) {
+            d = b + (a - 0.5);
+            w = deltaMinusDeltaSum(a, b);
+        } else {
+            d = a + (b - 0.5);
+            w = deltaMinusDeltaSum(b, a);
+        }
+
+        final double u = d * FastMath.log1p(a / b);
+        final double v = a * (FastMath.log(b) - 1.0);
+
+        return u <= v ? (w - u) - v : (w - v) - u;
+    }
+
+    /**
+     * Returns the value of Δ(b) - Δ(a + b), with 0 ≤ a ≤ b and b ≥ 10. Based
+     * on equations (26), (27) and (28) in Didonato and Morris (1992).
+     *
+     * @param a First argument.
+     * @param b Second argument.
+     * @return the value of {@code Delta(b) - Delta(a + b)}
+     * @throws OutOfRangeException if {@code a < 0} or {@code a > b}
+     * @throws NumberIsTooSmallException if {@code b < 10}
+     */
+    private static double deltaMinusDeltaSum(final double a,
+                                             final double b)
+        throws OutOfRangeException, NumberIsTooSmallException {
+
+        if ((a < 0) || (a > b)) {
+            throw new OutOfRangeException(a, 0, b);
+        }
+        if (b < 10) {
+            throw new NumberIsTooSmallException(b, 10, true);
+        }
+
+        final double h = a / b;
+        final double p = h / (1.0 + h);
+        final double q = 1.0 / (1.0 + h);
+        final double q2 = q * q;
+        /*
+         * s[i] = 1 + q + ... - q**(2 * i)
+         */
+        final double[] s = new double[DELTA.length];
+        s[0] = 1.0;
+        for (int i = 1; i < s.length; i++) {
+            s[i] = 1.0 + (q + q2 * s[i - 1]);
+        }
+        /*
+         * w = Delta(b) - Delta(a + b)
+         */
+        final double sqrtT = 10.0 / b;
+        final double t = sqrtT * sqrtT;
+        double w = DELTA[DELTA.length - 1] * s[s.length - 1];
+        for (int i = DELTA.length - 2; i >= 0; i--) {
+            w = t * w + DELTA[i] * s[i];
+        }
+        return w * p / b;
+    }
+
+    /**
+     * Returns the value of Δ(p) + Δ(q) - Δ(p + q), with p, q ≥ 10. Based on
+     * the NSWC Library of Mathematics Subroutines double precision
+     * implementation, {@code DBCORR}. In
+     * {@code BetaTest.testSumDeltaMinusDeltaSum()}, this private method is
+     * accessed through reflection.
+     *
+     * @param p First argument.
+     * @param q Second argument.
+     * @return the value of {@code Delta(p) + Delta(q) - Delta(p + q)}.
+     * @throws NumberIsTooSmallException if {@code p < 10.0} or {@code q < 10.0}.
+     */
+    private static double sumDeltaMinusDeltaSum(final double p,
+                                                final double q) {
+
+        if (p < 10.0) {
+            throw new NumberIsTooSmallException(p, 10.0, true);
+        }
+        if (q < 10.0) {
+            throw new NumberIsTooSmallException(q, 10.0, true);
+        }
+
+        final double a = FastMath.min(p, q);
+        final double b = FastMath.max(p, q);
+        final double sqrtT = 10.0 / a;
+        final double t = sqrtT * sqrtT;
+        double z = DELTA[DELTA.length - 1];
+        for (int i = DELTA.length - 2; i >= 0; i--) {
+            z = t * z + DELTA[i];
+        }
+        return z / a + deltaMinusDeltaSum(a, b);
+    }
+
+    /**
+     * Returns the value of log B(p, q) for 0 ≤ x ≤ 1 and p, q > 0. Based on the
+     * NSWC Library of Mathematics Subroutines implementation,
+     * {@code DBETLN}.
+     *
+     * @param p First argument.
+     * @param q Second argument.
+     * @return the value of {@code log(Beta(p, q))}, {@code NaN} if
+     * {@code p <= 0} or {@code q <= 0}.
+     */
+    public static double logBeta(final double p, final double q) {
+        if (Double.isNaN(p) || Double.isNaN(q) || (p <= 0.0) || (q <= 0.0)) {
+            return Double.NaN;
+        }
+
+        final double a = FastMath.min(p, q);
+        final double b = FastMath.max(p, q);
+        if (a >= 10.0) {
+            final double w = sumDeltaMinusDeltaSum(a, b);
+            final double h = a / b;
+            final double c = h / (1.0 + h);
+            final double u = -(a - 0.5) * FastMath.log(c);
+            final double v = b * FastMath.log1p(h);
+            if (u <= v) {
+                return (((-0.5 * FastMath.log(b) + HALF_LOG_TWO_PI) + w) - u) - v;
+            } else {
+                return (((-0.5 * FastMath.log(b) + HALF_LOG_TWO_PI) + w) - v) - u;
+            }
+        } else if (a > 2.0) {
+            if (b > 1000.0) {
+                final int n = (int) FastMath.floor(a - 1.0);
+                double prod = 1.0;
+                double ared = a;
+                for (int i = 0; i < n; i++) {
+                    ared -= 1.0;
+                    prod *= ared / (1.0 + ared / b);
+                }
+                return (FastMath.log(prod) - n * FastMath.log(b)) +
+                        (Gamma.logGamma(ared) +
+                         logGammaMinusLogGammaSum(ared, b));
+            } else {
+                double prod1 = 1.0;
+                double ared = a;
+                while (ared > 2.0) {
+                    ared -= 1.0;
+                    final double h = ared / b;
+                    prod1 *= h / (1.0 + h);
+                }
+                if (b < 10.0) {
+                    double prod2 = 1.0;
+                    double bred = b;
+                    while (bred > 2.0) {
+                        bred -= 1.0;
+                        prod2 *= bred / (ared + bred);
+                    }
+                    return FastMath.log(prod1) +
+                           FastMath.log(prod2) +
+                           (Gamma.logGamma(ared) +
+                           (Gamma.logGamma(bred) -
+                            logGammaSum(ared, bred)));
+                } else {
+                    return FastMath.log(prod1) +
+                           Gamma.logGamma(ared) +
+                           logGammaMinusLogGammaSum(ared, b);
+                }
+            }
+        } else if (a >= 1.0) {
+            if (b > 2.0) {
+                if (b < 10.0) {
+                    double prod = 1.0;
+                    double bred = b;
+                    while (bred > 2.0) {
+                        bred -= 1.0;
+                        prod *= bred / (a + bred);
+                    }
+                    return FastMath.log(prod) +
+                           (Gamma.logGamma(a) +
+                            (Gamma.logGamma(bred) -
+                             logGammaSum(a, bred)));
+                } else {
+                    return Gamma.logGamma(a) +
+                           logGammaMinusLogGammaSum(a, b);
+                }
+            } else {
+                return Gamma.logGamma(a) +
+                       Gamma.logGamma(b) -
+                       logGammaSum(a, b);
+            }
+        } else {
+            if (b >= 10.0) {
+                return Gamma.logGamma(a) +
+                       logGammaMinusLogGammaSum(a, b);
+            } else {
+                // The following command is the original NSWC implementation.
+                // return Gamma.logGamma(a) +
+                // (Gamma.logGamma(b) - Gamma.logGamma(a + b));
+                // The following command turns out to be more accurate.
+                return FastMath.log(Gamma.gamma(a) * Gamma.gamma(b) /
+                                    Gamma.gamma(a + b));
+            }
+        }
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/special/Erf.java b/java/source/infodynamics/utils/commonsmath3/special/Erf.java
new file mode 100644
index 0000000..8dd7e03
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/special/Erf.java
@@ -0,0 +1,272 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.special;
+
+import infodynamics.utils.commonsmath3.util.FastMath;
+
+/**
+ * This is a utility class that provides computation methods related to the
+ * error functions.
+ *
+ */
+public class Erf {
+
+    /**
+     * The number {@code X_CRIT} is used by {@link #erf(double, double)} internally.
+     * This number solves {@code erf(x)=0.5} within 1ulp.
+     * More precisely, the current implementations of
+     * {@link #erf(double)} and {@link #erfc(double)} satisfy:

+     * {@code erf(X_CRIT) < 0.5},

+     * {@code erf(Math.nextUp(X_CRIT) > 0.5},

+     * {@code erfc(X_CRIT) = 0.5}, and

+     * {@code erfc(Math.nextUp(X_CRIT) < 0.5}
+     */
+    private static final double X_CRIT = 0.4769362762044697;
+
+    /**
+     * Default constructor.  Prohibit instantiation.
+     */
+    private Erf() {}
+
+    /**
+     * Returns the error function.
+     *
+     * 
erf(x) = 2/√π 0∫x e-t2dt 

+     *
+     * 
This implementation computes erf(x) using the
+     * {@link Gamma#regularizedGammaP(double, double, double, int) regularized gamma function},
+     * following  Erf, equation (3)

+     *
+     * 
The value returned is always between -1 and 1 (inclusive).
+     * If {@code abs(x) > 40}, then {@code erf(x)} is indistinguishable from
+     * either 1 or -1 as a double, so the appropriate extreme value is returned.
+     * 

+     *
+     * @param x the value.
+     * @return the error function erf(x)
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     * @see Gamma#regularizedGammaP(double, double, double, int)
+     */
+    public static double erf(double x) {
+        if (FastMath.abs(x) > 40) {
+            return x > 0 ? 1 : -1;
+        }
+        final double ret = Gamma.regularizedGammaP(0.5, x * x, 1.0e-15, 10000);
+        return x < 0 ? -ret : ret;
+    }
+
+    /**
+     * Returns the complementary error function.
+     *
+     * 
erfc(x) = 2/√π x∫∞ e-t2dt
+     * 

+     *    = 1 - {@link #erf(double) erf(x)} 

+     *
+     * 
This implementation computes erfc(x) using the
+     * {@link Gamma#regularizedGammaQ(double, double, double, int) regularized gamma function},
+     * following  Erf, equation (3).

+     *
+     * 
The value returned is always between 0 and 2 (inclusive).
+     * If {@code abs(x) > 40}, then {@code erf(x)} is indistinguishable from
+     * either 0 or 2 as a double, so the appropriate extreme value is returned.
+     * 

+     *
+     * @param x the value
+     * @return the complementary error function erfc(x)
+     * @throws infodynamics.utils.commonsmath3.exception.MaxCountExceededException
+     * if the algorithm fails to converge.
+     * @see Gamma#regularizedGammaQ(double, double, double, int)
+     * @since 2.2
+     */
+    public static double erfc(double x) {
+        if (FastMath.abs(x) > 40) {
+            return x > 0 ? 0 : 2;
+        }
+        final double ret = Gamma.regularizedGammaQ(0.5, x * x, 1.0e-15, 10000);
+        return x < 0 ? 2 - ret : ret;
+    }
+
+    /**
+     * Returns the difference between erf(x1) and erf(x2).
+     *
+     * The implementation uses either erf(double) or erfc(double)
+     * depending on which provides the most precise result.
+     *
+     * @param x1 the first value
+     * @param x2 the second value
+     * @return erf(x2) - erf(x1)
+     */
+    public static double erf(double x1, double x2) {
+        if(x1 > x2) {
+            return -erf(x2, x1);
+        }
+
+        return
+        x1 < -X_CRIT ?
+            x2 < 0.0 ?
+                erfc(-x2) - erfc(-x1) :
+                erf(x2) - erf(x1) :
+            x2 > X_CRIT && x1 > 0.0 ?
+                erfc(x1) - erfc(x2) :
+                erf(x2) - erf(x1);
+    }
+
+    /**
+     * Returns the inverse erf.
+     * 

+     * This implementation is described in the paper:
+     * Approximating
+     * the erfinv function by Mike Giles, Oxford-Man Institute of Quantitative Finance,
+     * which was published in GPU Computing Gems, volume 2, 2010.
+     * The source code is available here.
+     * 

+     * @param x the value
+     * @return t such that x = erf(t)
+     * @since 3.2
+     */
+    public static double erfInv(final double x) {
+
+        // beware that the logarithm argument must be
+        // commputed as (1.0 - x) * (1.0 + x),
+        // it must NOT be simplified as 1.0 - x * x as this
+        // would induce rounding errors near the boundaries +/-1
+        double w = - FastMath.log((1.0 - x) * (1.0 + x));
+        double p;
+
+        if (w < 6.25) {
+            w -= 3.125;
+            p =  -3.6444120640178196996e-21;
+            p =   -1.685059138182016589e-19 + p * w;
+            p =   1.2858480715256400167e-18 + p * w;
+            p =    1.115787767802518096e-17 + p * w;
+            p =   -1.333171662854620906e-16 + p * w;
+            p =   2.0972767875968561637e-17 + p * w;
+            p =   6.6376381343583238325e-15 + p * w;
+            p =  -4.0545662729752068639e-14 + p * w;
+            p =  -8.1519341976054721522e-14 + p * w;
+            p =   2.6335093153082322977e-12 + p * w;
+            p =  -1.2975133253453532498e-11 + p * w;
+            p =  -5.4154120542946279317e-11 + p * w;
+            p =    1.051212273321532285e-09 + p * w;
+            p =  -4.1126339803469836976e-09 + p * w;
+            p =  -2.9070369957882005086e-08 + p * w;
+            p =   4.2347877827932403518e-07 + p * w;
+            p =  -1.3654692000834678645e-06 + p * w;
+            p =  -1.3882523362786468719e-05 + p * w;
+            p =    0.0001867342080340571352 + p * w;
+            p =  -0.00074070253416626697512 + p * w;
+            p =   -0.0060336708714301490533 + p * w;
+            p =      0.24015818242558961693 + p * w;
+            p =       1.6536545626831027356 + p * w;
+        } else if (w < 16.0) {
+            w = FastMath.sqrt(w) - 3.25;
+            p =   2.2137376921775787049e-09;
+            p =   9.0756561938885390979e-08 + p * w;
+            p =  -2.7517406297064545428e-07 + p * w;
+            p =   1.8239629214389227755e-08 + p * w;
+            p =   1.5027403968909827627e-06 + p * w;
+            p =   -4.013867526981545969e-06 + p * w;
+            p =   2.9234449089955446044e-06 + p * w;
+            p =   1.2475304481671778723e-05 + p * w;
+            p =  -4.7318229009055733981e-05 + p * w;
+            p =   6.8284851459573175448e-05 + p * w;
+            p =   2.4031110387097893999e-05 + p * w;
+            p =   -0.0003550375203628474796 + p * w;
+            p =   0.00095328937973738049703 + p * w;
+            p =   -0.0016882755560235047313 + p * w;
+            p =    0.0024914420961078508066 + p * w;
+            p =   -0.0037512085075692412107 + p * w;
+            p =     0.005370914553590063617 + p * w;
+            p =       1.0052589676941592334 + p * w;
+            p =       3.0838856104922207635 + p * w;
+        } else if (!Double.isInfinite(w)) {
+            w = FastMath.sqrt(w) - 5.0;
+            p =  -2.7109920616438573243e-11;
+            p =  -2.5556418169965252055e-10 + p * w;
+            p =   1.5076572693500548083e-09 + p * w;
+            p =  -3.7894654401267369937e-09 + p * w;
+            p =   7.6157012080783393804e-09 + p * w;
+            p =  -1.4960026627149240478e-08 + p * w;
+            p =   2.9147953450901080826e-08 + p * w;
+            p =  -6.7711997758452339498e-08 + p * w;
+            p =   2.2900482228026654717e-07 + p * w;
+            p =  -9.9298272942317002539e-07 + p * w;
+            p =   4.5260625972231537039e-06 + p * w;
+            p =  -1.9681778105531670567e-05 + p * w;
+            p =   7.5995277030017761139e-05 + p * w;
+            p =  -0.00021503011930044477347 + p * w;
+            p =  -0.00013871931833623122026 + p * w;
+            p =       1.0103004648645343977 + p * w;
+            p =       4.8499064014085844221 + p * w;
+        } else {
+            // this branch does not appears in the original code, it
+            // was added because the previous branch does not handle
+            // x = +/-1 correctly. In this case, w is positive infinity
+            // and as the first coefficient (-2.71e-11) is negative.
+            // Once the first multiplication is done, p becomes negative
+            // infinity and remains so throughout the polynomial evaluation.
+            // So the branch above incorrectly returns negative infinity
+            // instead of the correct positive infinity.
+            p = Double.POSITIVE_INFINITY;
+        }
+
+        return p * x;
+
+    }
+
+    /**
+     * Returns the inverse erfc.
+     * @param x the value
+     * @return t such that x = erfc(t)
+     * @since 3.2
+     */
+    public static double erfcInv(final double x) {
+        return erfInv(1 - x);
+    }
+
+}
+
diff --git a/java/source/infodynamics/utils/commonsmath3/special/Gamma.java b/java/source/infodynamics/utils/commonsmath3/special/Gamma.java
old mode 100755
new mode 100644
index 5c34d77..b5951f8
--- a/java/source/infodynamics/utils/commonsmath3/special/Gamma.java
+++ b/java/source/infodynamics/utils/commonsmath3/special/Gamma.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -78,7 +78,6 @@ import infodynamics.utils.commonsmath3.util.FastMath;
  * functions can safely be ported to Commons-Math.
  * 

  *
- * @version $Id$
  */
 public class Gamma {
     /**
@@ -432,11 +431,13 @@ public class Gamma {
             // create continued fraction
             ContinuedFraction cf = new ContinuedFraction() {
 
+                /** {@inheritDoc} */
                 @Override
                 protected double getA(int n, double x) {
                     return ((2.0 * n) + 1.0) - a + x;
                 }
 
+                /** {@inheritDoc} */
                 @Override
                 protected double getB(int n, double x) {
                     return n * (a - n);
@@ -472,6 +473,10 @@ public class Gamma {
      * @since 2.0
      */
     public static double digamma(double x) {
+        if (Double.isNaN(x) || Double.isInfinite(x)) {
+            return x;
+        }
+
         if (x > 0 && x <= S_LIMIT) {
             // use method 5 from Bernardo AS103
             // accurate to O(x)
@@ -502,6 +507,10 @@ public class Gamma {
      * @since 2.0
      */
     public static double trigamma(double x) {
+        if (Double.isNaN(x) || Double.isInfinite(x)) {
+            return x;
+        }
+
         if (x > 0 && x <= S_LIMIT) {
             return 1 / (x * x);
         }
@@ -717,7 +726,7 @@ public class Gamma {
             }
         } else {
             final double y = absX + LANCZOS_G + 0.5;
-            final double gammaAbs = SQRT_TWO_PI / x *
+            final double gammaAbs = SQRT_TWO_PI / absX *
                                     FastMath.pow(y, absX + 0.5) *
                                     FastMath.exp(-y) * lanczos(absX);
             if (x > 0.0) {
diff --git a/java/source/infodynamics/utils/commonsmath3/util/ArithmeticUtils.java b/java/source/infodynamics/utils/commonsmath3/util/ArithmeticUtils.java
new file mode 100644
index 0000000..e15ca01
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/ArithmeticUtils.java
@@ -0,0 +1,936 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+import java.math.BigInteger;
+
+import infodynamics.utils.commonsmath3.exception.MathArithmeticException;
+import infodynamics.utils.commonsmath3.exception.NotPositiveException;
+import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException;
+import infodynamics.utils.commonsmath3.exception.util.Localizable;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
+/**
+ * Some useful, arithmetics related, additions to the built-in functions in
+ * {@link Math}.
+ *
+ */
+public final class ArithmeticUtils {
+
+    /** Private constructor. */
+    private ArithmeticUtils() {
+        super();
+    }
+
+    /**
+     * Add two integers, checking for overflow.
+     *
+     * @param x an addend
+     * @param y an addend
+     * @return the sum {@code x+y}
+     * @throws MathArithmeticException if the result can not be represented
+     * as an {@code int}.
+     * @since 1.1
+     */
+    public static int addAndCheck(int x, int y)
+            throws MathArithmeticException {
+        long s = (long)x + (long)y;
+        if (s < Integer.MIN_VALUE || s > Integer.MAX_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, x, y);
+        }
+        return (int)s;
+    }
+
+    /**
+     * Add two long integers, checking for overflow.
+     *
+     * @param a an addend
+     * @param b an addend
+     * @return the sum {@code a+b}
+     * @throws MathArithmeticException if the result can not be represented as an long
+     * @since 1.2
+     */
+    public static long addAndCheck(long a, long b) throws MathArithmeticException {
+        return addAndCheck(a, b, LocalizedFormats.OVERFLOW_IN_ADDITION);
+    }
+
+    /**
+     * Returns an exact representation of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code IllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code long}. The
+     * largest value of {@code n} for which all coefficients are
+     * {@code  < Long.MAX_VALUE} is 66. If the computed value exceeds
+     * {@code Long.MAX_VALUE} an {@code ArithMeticException} is
+     * thrown.
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     * @deprecated use {@link CombinatoricsUtils#binomialCoefficient(int, int)}
+     */
+    @Deprecated
+    public static long binomialCoefficient(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+       return CombinatoricsUtils.binomialCoefficient(n, k);
+    }
+
+    /**
+     * Returns a {@code double} representation of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code IllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code double}. The
+     * largest value of {@code n} for which all coefficients are <
+     * Double.MAX_VALUE is 1029. If the computed value exceeds Double.MAX_VALUE,
+     * Double.POSITIVE_INFINITY is returned
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     * @deprecated use {@link CombinatoricsUtils#binomialCoefficientDouble(int, int)}
+     */
+    @Deprecated
+    public static double binomialCoefficientDouble(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        return CombinatoricsUtils.binomialCoefficientDouble(n, k);
+    }
+
+    /**
+     * Returns the natural {@code log} of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code IllegalArgumentException} is thrown)
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     * @deprecated use {@link CombinatoricsUtils#binomialCoefficientLog(int, int)}
+     */
+    @Deprecated
+    public static double binomialCoefficientLog(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        return CombinatoricsUtils.binomialCoefficientLog(n, k);
+    }
+
+    /**
+     * Returns n!. Shorthand for {@code n}  Factorial, the
+     * product of the numbers {@code 1,...,n}.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code n >= 0} (otherwise
+     * {@code IllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code long}. The
+     * largest value of {@code n} for which {@code n!} <
+     * Long.MAX_VALUE} is 20. If the computed value exceeds {@code Long.MAX_VALUE}
+     * an {@code ArithMeticException } is thrown.
    • 
+     * 

+     * 

+     *
+     * @param n argument
+     * @return {@code n!}
+     * @throws MathArithmeticException if the result is too large to be represented
+     * by a {@code long}.
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws MathArithmeticException if {@code n > 20}: The factorial value is too
+     * large to fit in a {@code long}.
+     * @deprecated use {@link CombinatoricsUtils#factorial(int)}
+     */
+    @Deprecated
+    public static long factorial(final int n) throws NotPositiveException, MathArithmeticException {
+        return CombinatoricsUtils.factorial(n);
+    }
+
+    /**
+     * Compute n!, the
+     * factorial of {@code n} (the product of the numbers 1 to n), as a
+     * {@code double}.
+     * The result should be small enough to fit into a {@code double}: The
+     * largest {@code n} for which {@code n! < Double.MAX_VALUE} is 170.
+     * If the computed value exceeds {@code Double.MAX_VALUE},
+     * {@code Double.POSITIVE_INFINITY} is returned.
+     *
+     * @param n Argument.
+     * @return {@code n!}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @deprecated use {@link CombinatoricsUtils#factorialDouble(int)}
+     */
+    @Deprecated
+    public static double factorialDouble(final int n) throws NotPositiveException {
+         return CombinatoricsUtils.factorialDouble(n);
+    }
+
+    /**
+     * Compute the natural logarithm of the factorial of {@code n}.
+     *
+     * @param n Argument.
+     * @return {@code n!}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @deprecated use {@link CombinatoricsUtils#factorialLog(int)}
+     */
+    @Deprecated
+    public static double factorialLog(final int n) throws NotPositiveException {
+        return CombinatoricsUtils.factorialLog(n);
+    }
+
+    /**
+     * Computes the greatest common divisor of the absolute value of two
+     * numbers, using a modified version of the "binary gcd" method.
+     * See Knuth 4.5.2 algorithm B.
+     * The algorithm is due to Josef Stein (1961).
+     * 

+     * Special cases:
+     * 
    
+     *  
  • The invocations
+     *   {@code gcd(Integer.MIN_VALUE, Integer.MIN_VALUE)},
+     *   {@code gcd(Integer.MIN_VALUE, 0)} and
+     *   {@code gcd(0, Integer.MIN_VALUE)} throw an
+     *   {@code ArithmeticException}, because the result would be 2^31, which
+     *   is too large for an int value.
    • 
+     *  
  • The result of {@code gcd(x, x)}, {@code gcd(0, x)} and
+     *   {@code gcd(x, 0)} is the absolute value of {@code x}, except
+     *   for the special cases above.
    • 
+     *  
  • The invocation {@code gcd(0, 0)} is the only one which returns
+     *   {@code 0}.
    • 
+     * 

+     *
+     * @param p Number.
+     * @param q Number.
+     * @return the greatest common divisor (never negative).
+     * @throws MathArithmeticException if the result cannot be represented as
+     * a non-negative {@code int} value.
+     * @since 1.1
+     */
+    public static int gcd(int p, int q) throws MathArithmeticException {
+        int a = p;
+        int b = q;
+        if (a == 0 ||
+            b == 0) {
+            if (a == Integer.MIN_VALUE ||
+                b == Integer.MIN_VALUE) {
+                throw new MathArithmeticException(LocalizedFormats.GCD_OVERFLOW_32_BITS,
+                                                  p, q);
+            }
+            return FastMath.abs(a + b);
+        }
+
+        long al = a;
+        long bl = b;
+        boolean useLong = false;
+        if (a < 0) {
+            if(Integer.MIN_VALUE == a) {
+                useLong = true;
+            } else {
+                a = -a;
+            }
+            al = -al;
+        }
+        if (b < 0) {
+            if (Integer.MIN_VALUE == b) {
+                useLong = true;
+            } else {
+                b = -b;
+            }
+            bl = -bl;
+        }
+        if (useLong) {
+            if(al == bl) {
+                throw new MathArithmeticException(LocalizedFormats.GCD_OVERFLOW_32_BITS,
+                                                  p, q);
+            }
+            long blbu = bl;
+            bl = al;
+            al = blbu % al;
+            if (al == 0) {
+                if (bl > Integer.MAX_VALUE) {
+                    throw new MathArithmeticException(LocalizedFormats.GCD_OVERFLOW_32_BITS,
+                                                      p, q);
+                }
+                return (int) bl;
+            }
+            blbu = bl;
+
+            // Now "al" and "bl" fit in an "int".
+            b = (int) al;
+            a = (int) (blbu % al);
+        }
+
+        return gcdPositive(a, b);
+    }
+
+    /**
+     * Computes the greatest common divisor of two positive numbers
+     * (this precondition is not checked and the result is undefined
+     * if not fulfilled) using the "binary gcd" method which avoids division
+     * and modulo operations.
+     * See Knuth 4.5.2 algorithm B.
+     * The algorithm is due to Josef Stein (1961).
+     * 

+     * Special cases:
+     * 
    
+     *  
  • The result of {@code gcd(x, x)}, {@code gcd(0, x)} and
+     *   {@code gcd(x, 0)} is the value of {@code x}.
    • 
+     *  
  • The invocation {@code gcd(0, 0)} is the only one which returns
+     *   {@code 0}.
    • 
+     * 

+     *
+     * @param a Positive number.
+     * @param b Positive number.
+     * @return the greatest common divisor.
+     */
+    private static int gcdPositive(int a, int b) {
+        if (a == 0) {
+            return b;
+        }
+        else if (b == 0) {
+            return a;
+        }
+
+        // Make "a" and "b" odd, keeping track of common power of 2.
+        final int aTwos = Integer.numberOfTrailingZeros(a);
+        a >>= aTwos;
+        final int bTwos = Integer.numberOfTrailingZeros(b);
+        b >>= bTwos;
+        final int shift = FastMath.min(aTwos, bTwos);
+
+        // "a" and "b" are positive.
+        // If a > b then "gdc(a, b)" is equal to "gcd(a - b, b)".
+        // If a < b then "gcd(a, b)" is equal to "gcd(b - a, a)".
+        // Hence, in the successive iterations:
+        //  "a" becomes the absolute difference of the current values,
+        //  "b" becomes the minimum of the current values.
+        while (a != b) {
+            final int delta = a - b;
+            b = Math.min(a, b);
+            a = Math.abs(delta);
+
+            // Remove any power of 2 in "a" ("b" is guaranteed to be odd).
+            a >>= Integer.numberOfTrailingZeros(a);
+        }
+
+        // Recover the common power of 2.
+        return a << shift;
+    }
+
+    /**
+     * 

+     * Gets the greatest common divisor of the absolute value of two numbers,
+     * using the "binary gcd" method which avoids division and modulo
+     * operations. See Knuth 4.5.2 algorithm B. This algorithm is due to Josef
+     * Stein (1961).
+     * 

+     * Special cases:
+     * 
    
+     * 
  • The invocations
+     * {@code gcd(Long.MIN_VALUE, Long.MIN_VALUE)},
+     * {@code gcd(Long.MIN_VALUE, 0L)} and
+     * {@code gcd(0L, Long.MIN_VALUE)} throw an
+     * {@code ArithmeticException}, because the result would be 2^63, which
+     * is too large for a long value.
    • 
+     * 
  • The result of {@code gcd(x, x)}, {@code gcd(0L, x)} and
+     * {@code gcd(x, 0L)} is the absolute value of {@code x}, except
+     * for the special cases above.
+     * 
  • The invocation {@code gcd(0L, 0L)} is the only one which returns
+     * {@code 0L}.
    • 
+     * 

+     *
+     * @param p Number.
+     * @param q Number.
+     * @return the greatest common divisor, never negative.
+     * @throws MathArithmeticException if the result cannot be represented as
+     * a non-negative {@code long} value.
+     * @since 2.1
+     */
+    public static long gcd(final long p, final long q) throws MathArithmeticException {
+        long u = p;
+        long v = q;
+        if ((u == 0) || (v == 0)) {
+            if ((u == Long.MIN_VALUE) || (v == Long.MIN_VALUE)){
+                throw new MathArithmeticException(LocalizedFormats.GCD_OVERFLOW_64_BITS,
+                                                  p, q);
+            }
+            return FastMath.abs(u) + FastMath.abs(v);
+        }
+        // keep u and v negative, as negative integers range down to
+        // -2^63, while positive numbers can only be as large as 2^63-1
+        // (i.e. we can't necessarily negate a negative number without
+        // overflow)
+        /* assert u!=0 && v!=0; */
+        if (u > 0) {
+            u = -u;
+        } // make u negative
+        if (v > 0) {
+            v = -v;
+        } // make v negative
+        // B1. [Find power of 2]
+        int k = 0;
+        while ((u & 1) == 0 && (v & 1) == 0 && k < 63) { // while u and v are
+                                                            // both even...
+            u /= 2;
+            v /= 2;
+            k++; // cast out twos.
+        }
+        if (k == 63) {
+            throw new MathArithmeticException(LocalizedFormats.GCD_OVERFLOW_64_BITS,
+                                              p, q);
+        }
+        // B2. Initialize: u and v have been divided by 2^k and at least
+        // one is odd.
+        long t = ((u & 1) == 1) ? v : -(u / 2)/* B3 */;
+        // t negative: u was odd, v may be even (t replaces v)
+        // t positive: u was even, v is odd (t replaces u)
+        do {
+            /* assert u<0 && v<0; */
+            // B4/B3: cast out twos from t.
+            while ((t & 1) == 0) { // while t is even..
+                t /= 2; // cast out twos
+            }
+            // B5 [reset max(u,v)]
+            if (t > 0) {
+                u = -t;
+            } else {
+                v = t;
+            }
+            // B6/B3. at this point both u and v should be odd.
+            t = (v - u) / 2;
+            // |u| larger: t positive (replace u)
+            // |v| larger: t negative (replace v)
+        } while (t != 0);
+        return -u * (1L << k); // gcd is u*2^k
+    }
+
+    /**
+     * 

+     * Returns the least common multiple of the absolute value of two numbers,
+     * using the formula {@code lcm(a,b) = (a / gcd(a,b)) * b}.
+     * 

+     * Special cases:
+     * 
    
+     * 
  • The invocations {@code lcm(Integer.MIN_VALUE, n)} and
+     * {@code lcm(n, Integer.MIN_VALUE)}, where {@code abs(n)} is a
+     * power of 2, throw an {@code ArithmeticException}, because the result
+     * would be 2^31, which is too large for an int value.
    • 
+     * 
  • The result of {@code lcm(0, x)} and {@code lcm(x, 0)} is
+     * {@code 0} for any {@code x}.
+     * 

+     *
+     * @param a Number.
+     * @param b Number.
+     * @return the least common multiple, never negative.
+     * @throws MathArithmeticException if the result cannot be represented as
+     * a non-negative {@code int} value.
+     * @since 1.1
+     */
+    public static int lcm(int a, int b) throws MathArithmeticException {
+        if (a == 0 || b == 0){
+            return 0;
+        }
+        int lcm = FastMath.abs(ArithmeticUtils.mulAndCheck(a / gcd(a, b), b));
+        if (lcm == Integer.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.LCM_OVERFLOW_32_BITS,
+                                              a, b);
+        }
+        return lcm;
+    }
+
+    /**
+     * 

+     * Returns the least common multiple of the absolute value of two numbers,
+     * using the formula {@code lcm(a,b) = (a / gcd(a,b)) * b}.
+     * 

+     * Special cases:
+     * 
    
+     * 
  • The invocations {@code lcm(Long.MIN_VALUE, n)} and
+     * {@code lcm(n, Long.MIN_VALUE)}, where {@code abs(n)} is a
+     * power of 2, throw an {@code ArithmeticException}, because the result
+     * would be 2^63, which is too large for an int value.
    • 
+     * 
  • The result of {@code lcm(0L, x)} and {@code lcm(x, 0L)} is
+     * {@code 0L} for any {@code x}.
+     * 

+     *
+     * @param a Number.
+     * @param b Number.
+     * @return the least common multiple, never negative.
+     * @throws MathArithmeticException if the result cannot be represented
+     * as a non-negative {@code long} value.
+     * @since 2.1
+     */
+    public static long lcm(long a, long b) throws MathArithmeticException {
+        if (a == 0 || b == 0){
+            return 0;
+        }
+        long lcm = FastMath.abs(ArithmeticUtils.mulAndCheck(a / gcd(a, b), b));
+        if (lcm == Long.MIN_VALUE){
+            throw new MathArithmeticException(LocalizedFormats.LCM_OVERFLOW_64_BITS,
+                                              a, b);
+        }
+        return lcm;
+    }
+
+    /**
+     * Multiply two integers, checking for overflow.
+     *
+     * @param x Factor.
+     * @param y Factor.
+     * @return the product {@code x * y}.
+     * @throws MathArithmeticException if the result can not be
+     * represented as an {@code int}.
+     * @since 1.1
+     */
+    public static int mulAndCheck(int x, int y) throws MathArithmeticException {
+        long m = ((long)x) * ((long)y);
+        if (m < Integer.MIN_VALUE || m > Integer.MAX_VALUE) {
+            throw new MathArithmeticException();
+        }
+        return (int)m;
+    }
+
+    /**
+     * Multiply two long integers, checking for overflow.
+     *
+     * @param a Factor.
+     * @param b Factor.
+     * @return the product {@code a * b}.
+     * @throws MathArithmeticException if the result can not be represented
+     * as a {@code long}.
+     * @since 1.2
+     */
+    public static long mulAndCheck(long a, long b) throws MathArithmeticException {
+        long ret;
+        if (a > b) {
+            // use symmetry to reduce boundary cases
+            ret = mulAndCheck(b, a);
+        } else {
+            if (a < 0) {
+                if (b < 0) {
+                    // check for positive overflow with negative a, negative b
+                    if (a >= Long.MAX_VALUE / b) {
+                        ret = a * b;
+                    } else {
+                        throw new MathArithmeticException();
+                    }
+                } else if (b > 0) {
+                    // check for negative overflow with negative a, positive b
+                    if (Long.MIN_VALUE / b <= a) {
+                        ret = a * b;
+                    } else {
+                        throw new MathArithmeticException();
+
+                    }
+                } else {
+                    // assert b == 0
+                    ret = 0;
+                }
+            } else if (a > 0) {
+                // assert a > 0
+                // assert b > 0
+
+                // check for positive overflow with positive a, positive b
+                if (a <= Long.MAX_VALUE / b) {
+                    ret = a * b;
+                } else {
+                    throw new MathArithmeticException();
+                }
+            } else {
+                // assert a == 0
+                ret = 0;
+            }
+        }
+        return ret;
+    }
+
+    /**
+     * Subtract two integers, checking for overflow.
+     *
+     * @param x Minuend.
+     * @param y Subtrahend.
+     * @return the difference {@code x - y}.
+     * @throws MathArithmeticException if the result can not be represented
+     * as an {@code int}.
+     * @since 1.1
+     */
+    public static int subAndCheck(int x, int y) throws MathArithmeticException {
+        long s = (long)x - (long)y;
+        if (s < Integer.MIN_VALUE || s > Integer.MAX_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_SUBTRACTION, x, y);
+        }
+        return (int)s;
+    }
+
+    /**
+     * Subtract two long integers, checking for overflow.
+     *
+     * @param a Value.
+     * @param b Value.
+     * @return the difference {@code a - b}.
+     * @throws MathArithmeticException if the result can not be represented as a
+     * {@code long}.
+     * @since 1.2
+     */
+    public static long subAndCheck(long a, long b) throws MathArithmeticException {
+        long ret;
+        if (b == Long.MIN_VALUE) {
+            if (a < 0) {
+                ret = a - b;
+            } else {
+                throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, a, -b);
+            }
+        } else {
+            // use additive inverse
+            ret = addAndCheck(a, -b, LocalizedFormats.OVERFLOW_IN_ADDITION);
+        }
+        return ret;
+    }
+
+    /**
+     * Raise an int to an int power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return \( k^e \)
+     * @throws NotPositiveException if {@code e < 0}.
+     * @throws MathArithmeticException if the result would overflow.
+     */
+    public static int pow(final int k,
+                          final int e)
+        throws NotPositiveException,
+               MathArithmeticException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        try {
+            int exp = e;
+            int result = 1;
+            int k2p    = k;
+            while (true) {
+                if ((exp & 0x1) != 0) {
+                    result = mulAndCheck(result, k2p);
+                }
+
+                exp >>= 1;
+                if (exp == 0) {
+                    break;
+                }
+
+                k2p = mulAndCheck(k2p, k2p);
+            }
+
+            return result;
+        } catch (MathArithmeticException mae) {
+            // Add context information.
+            mae.getContext().addMessage(LocalizedFormats.OVERFLOW);
+            mae.getContext().addMessage(LocalizedFormats.BASE, k);
+            mae.getContext().addMessage(LocalizedFormats.EXPONENT, e);
+
+            // Rethrow.
+            throw mae;
+        }
+    }
+
+    /**
+     * Raise an int to a long power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return ke
+     * @throws NotPositiveException if {@code e < 0}.
+     * @deprecated As of 3.3. Please use {@link #pow(int,int)} instead.
+     */
+    @Deprecated
+    public static int pow(final int k, long e) throws NotPositiveException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        int result = 1;
+        int k2p    = k;
+        while (e != 0) {
+            if ((e & 0x1) != 0) {
+                result *= k2p;
+            }
+            k2p *= k2p;
+            e >>= 1;
+        }
+
+        return result;
+    }
+
+    /**
+     * Raise a long to an int power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return \( k^e \)
+     * @throws NotPositiveException if {@code e < 0}.
+     * @throws MathArithmeticException if the result would overflow.
+     */
+    public static long pow(final long k,
+                           final int e)
+        throws NotPositiveException,
+               MathArithmeticException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        try {
+            int exp = e;
+            long result = 1;
+            long k2p    = k;
+            while (true) {
+                if ((exp & 0x1) != 0) {
+                    result = mulAndCheck(result, k2p);
+                }
+
+                exp >>= 1;
+                if (exp == 0) {
+                    break;
+                }
+
+                k2p = mulAndCheck(k2p, k2p);
+            }
+
+            return result;
+        } catch (MathArithmeticException mae) {
+            // Add context information.
+            mae.getContext().addMessage(LocalizedFormats.OVERFLOW);
+            mae.getContext().addMessage(LocalizedFormats.BASE, k);
+            mae.getContext().addMessage(LocalizedFormats.EXPONENT, e);
+
+            // Rethrow.
+            throw mae;
+        }
+    }
+
+    /**
+     * Raise a long to a long power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return ke
+     * @throws NotPositiveException if {@code e < 0}.
+     * @deprecated As of 3.3. Please use {@link #pow(long,int)} instead.
+     */
+    @Deprecated
+    public static long pow(final long k, long e) throws NotPositiveException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        long result = 1l;
+        long k2p    = k;
+        while (e != 0) {
+            if ((e & 0x1) != 0) {
+                result *= k2p;
+            }
+            k2p *= k2p;
+            e >>= 1;
+        }
+
+        return result;
+    }
+
+    /**
+     * Raise a BigInteger to an int power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return ke
+     * @throws NotPositiveException if {@code e < 0}.
+     */
+    public static BigInteger pow(final BigInteger k, int e) throws NotPositiveException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        return k.pow(e);
+    }
+
+    /**
+     * Raise a BigInteger to a long power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return ke
+     * @throws NotPositiveException if {@code e < 0}.
+     */
+    public static BigInteger pow(final BigInteger k, long e) throws NotPositiveException {
+        if (e < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        BigInteger result = BigInteger.ONE;
+        BigInteger k2p    = k;
+        while (e != 0) {
+            if ((e & 0x1) != 0) {
+                result = result.multiply(k2p);
+            }
+            k2p = k2p.multiply(k2p);
+            e >>= 1;
+        }
+
+        return result;
+
+    }
+
+    /**
+     * Raise a BigInteger to a BigInteger power.
+     *
+     * @param k Number to raise.
+     * @param e Exponent (must be positive or zero).
+     * @return ke
+     * @throws NotPositiveException if {@code e < 0}.
+     */
+    public static BigInteger pow(final BigInteger k, BigInteger e) throws NotPositiveException {
+        if (e.compareTo(BigInteger.ZERO) < 0) {
+            throw new NotPositiveException(LocalizedFormats.EXPONENT, e);
+        }
+
+        BigInteger result = BigInteger.ONE;
+        BigInteger k2p    = k;
+        while (!BigInteger.ZERO.equals(e)) {
+            if (e.testBit(0)) {
+                result = result.multiply(k2p);
+            }
+            k2p = k2p.multiply(k2p);
+            e = e.shiftRight(1);
+        }
+
+        return result;
+    }
+
+    /**
+     * Returns the 
+     * Stirling number of the second kind, "{@code S(n,k)}", the number of
+     * ways of partitioning an {@code n}-element set into {@code k} non-empty
+     * subsets.
+     * 

+     * The preconditions are {@code 0 <= k <= n } (otherwise
+     * {@code NotPositiveException} is thrown)
+     * 

+     * @param n the size of the set
+     * @param k the number of non-empty subsets
+     * @return {@code S(n,k)}
+     * @throws NotPositiveException if {@code k < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if some overflow happens, typically for n exceeding 25 and
+     * k between 20 and n-2 (S(n,n-1) is handled specifically and does not overflow)
+     * @since 3.1
+     * @deprecated use {@link CombinatoricsUtils#stirlingS2(int, int)}
+     */
+    @Deprecated
+    public static long stirlingS2(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        return CombinatoricsUtils.stirlingS2(n, k);
+
+    }
+
+    /**
+     * Add two long integers, checking for overflow.
+     *
+     * @param a Addend.
+     * @param b Addend.
+     * @param pattern Pattern to use for any thrown exception.
+     * @return the sum {@code a + b}.
+     * @throws MathArithmeticException if the result cannot be represented
+     * as a {@code long}.
+     * @since 1.2
+     */
+     private static long addAndCheck(long a, long b, Localizable pattern) throws MathArithmeticException {
+         final long result = a + b;
+         if (!((a ^ b) < 0 | (a ^ result) >= 0)) {
+             throw new MathArithmeticException(pattern, a, b);
+         }
+         return result;
+    }
+
+    /**
+     * Returns true if the argument is a power of two.
+     *
+     * @param n the number to test
+     * @return true if the argument is a power of two
+     */
+    public static boolean isPowerOfTwo(long n) {
+        return (n > 0) && ((n & (n - 1)) == 0);
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/Combinations.java b/java/source/infodynamics/utils/commonsmath3/util/Combinations.java
new file mode 100644
index 0000000..4d90089
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/Combinations.java
@@ -0,0 +1,434 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+import java.util.Iterator;
+import java.util.Comparator;
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+import java.io.Serializable;
+import infodynamics.utils.commonsmath3.exception.MathInternalError;
+import infodynamics.utils.commonsmath3.exception.DimensionMismatchException;
+import infodynamics.utils.commonsmath3.exception.OutOfRangeException;
+
+/**
+ * Utility to create 
+ * combinations {@code (n, k)} of {@code k} elements in a set of
+ * {@code n} elements.
+ *
+ * @since 3.3
+ */
+public class Combinations implements Iterable {
+    /** Size of the set from which combinations are drawn. */
+    private final int n;
+    /** Number of elements in each combination. */
+    private final int k;
+    /** Iteration order. */
+    private final IterationOrder iterationOrder;
+
+    /**
+     * Describes the type of iteration performed by the
+     * {@link #iterator() iterator}.
+     */
+    private enum IterationOrder {
+        /** Lexicographic order. */
+        LEXICOGRAPHIC
+    }
+
+   /**
+     * Creates an instance whose range is the k-element subsets of
+     * {0, ..., n - 1} represented as {@code int[]} arrays.
+     * 

+     * The iteration order is lexicographic: the arrays returned by the
+     * {@link #iterator() iterator} are sorted in descending order and
+     * they are visited in lexicographic order with significance from
+     * right to left.
+     * For example, {@code new Combinations(4, 2).iterator()} returns
+     * an iterator that will generate the following sequence of arrays
+     * on successive calls to
+     * {@code next()}:

+     * {@code [0, 1], [0, 2], [1, 2], [0, 3], [1, 3], [2, 3]}
+     * 

+     * If {@code k == 0} an iterator containing an empty array is returned;
+     * if {@code k == n} an iterator containing [0, ..., n - 1] is returned.
+     *
+     * @param n Size of the set from which subsets are selected.
+     * @param k Size of the subsets to be enumerated.
+     * @throws infodynamics.utils.commonsmath3.exception.NotPositiveException if {@code n < 0}.
+     * @throws infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException if {@code k > n}.
+     */
+    public Combinations(int n,
+                        int k) {
+        this(n, k, IterationOrder.LEXICOGRAPHIC);
+    }
+
+    /**
+     * Creates an instance whose range is the k-element subsets of
+     * {0, ..., n - 1} represented as {@code int[]} arrays.
+     * 

+     * If the {@code iterationOrder} argument is set to
+     * {@link IterationOrder#LEXICOGRAPHIC}, the arrays returned by the
+     * {@link #iterator() iterator} are sorted in descending order and
+     * they are visited in lexicographic order with significance from
+     * right to left.
+     * For example, {@code new Combinations(4, 2).iterator()} returns
+     * an iterator that will generate the following sequence of arrays
+     * on successive calls to
+     * {@code next()}:

+     * {@code [0, 1], [0, 2], [1, 2], [0, 3], [1, 3], [2, 3]}
+     * 

+     * If {@code k == 0} an iterator containing an empty array is returned;
+     * if {@code k == n} an iterator containing [0, ..., n - 1] is returned.
+     *
+     * @param n Size of the set from which subsets are selected.
+     * @param k Size of the subsets to be enumerated.
+     * @param iterationOrder Specifies the {@link #iterator() iteration order}.
+     * @throws infodynamics.utils.commonsmath3.exception.NotPositiveException if {@code n < 0}.
+     * @throws infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException if {@code k > n}.
+     */
+    private Combinations(int n,
+                         int k,
+                         IterationOrder iterationOrder) {
+        CombinatoricsUtils.checkBinomial(n, k);
+        this.n = n;
+        this.k = k;
+        this.iterationOrder = iterationOrder;
+    }
+
+    /**
+     * Gets the size of the set from which combinations are drawn.
+     *
+     * @return the size of the universe.
+     */
+    public int getN() {
+        return n;
+    }
+
+    /**
+     * Gets the number of elements in each combination.
+     *
+     * @return the size of the subsets to be enumerated.
+     */
+    public int getK() {
+        return k;
+    }
+
+    /** {@inheritDoc} */
+    public Iterator iterator() {
+        if (k == 0 ||
+            k == n) {
+            return new SingletonIterator(MathArrays.natural(k));
+        }
+
+        switch (iterationOrder) {
+        case LEXICOGRAPHIC:
+            return new LexicographicIterator(n, k);
+        default:
+            throw new MathInternalError(); // Should never happen.
+        }
+    }
+
+    /**
+     * Defines a lexicographic ordering of combinations.
+     * The returned comparator allows to compare any two combinations
+     * that can be produced by this instance's {@link #iterator() iterator}.
+     * Its {@code compare(int[],int[])} method will throw exceptions if
+     * passed combinations that are inconsistent with this instance:
+     * 
    
+     *  
  • {@code DimensionMismatchException} if the array lengths are not
+     *      equal to {@code k},
    • 
+     *  
  • {@code OutOfRangeException} if an element of the array is not
+     *      within the interval [0, {@code n}).
    • 
+     * 

+     * @return a lexicographic comparator.
+     */
+    public Comparator comparator() {
+        return new LexicographicComparator(n, k);
+    }
+
+    /**
+     * Lexicographic combinations iterator.
+     * 

+     * Implementation follows Algorithm T in The Art of Computer Programming
+     * Internet Draft (PRE-FASCICLE 3A), "A Draft of Section 7.2.1.3 Generating All
+     * Combinations, D. Knuth, 2004.

+     * 

+     * The degenerate cases {@code k == 0} and {@code k == n} are NOT handled by this
+     * implementation.  If constructor arguments satisfy {@code k == 0}
+     * or {@code k >= n}, no exception is generated, but the iterator is empty.
+     * 

+     *
+     */
+    private static class LexicographicIterator implements Iterator {
+        /** Size of subsets returned by the iterator */
+        private final int k;
+
+        /**
+         * c[1], ..., c[k] stores the next combination; c[k + 1], c[k + 2] are
+         * sentinels.
+         * 

+         * Note that c[0] is "wasted" but this makes it a little easier to
+         * follow the code.
+         * 

+         */
+        private final int[] c;
+
+        /** Return value for {@link #hasNext()} */
+        private boolean more = true;
+
+        /** Marker: smallest index such that c[j + 1] > j */
+        private int j;
+
+        /**
+         * Construct a CombinationIterator to enumerate k-sets from n.
+         * 

+         * NOTE: If {@code k === 0} or {@code k >= n}, the Iterator will be empty
+         * (that is, {@link #hasNext()} will return {@code false} immediately.
+         * 

+         *
+         * @param n size of the set from which subsets are enumerated
+         * @param k size of the subsets to enumerate
+         */
+        LexicographicIterator(int n, int k) {
+            this.k = k;
+            c = new int[k + 3];
+            if (k == 0 || k >= n) {
+                more = false;
+                return;
+            }
+            // Initialize c to start with lexicographically first k-set
+            for (int i = 1; i <= k; i++) {
+                c[i] = i - 1;
+            }
+            // Initialize sentinels
+            c[k + 1] = n;
+            c[k + 2] = 0;
+            j = k; // Set up invariant: j is smallest index such that c[j + 1] > j
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        public boolean hasNext() {
+            return more;
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        public int[] next() {
+            if (!more) {
+                throw new NoSuchElementException();
+            }
+            // Copy return value (prepared by last activation)
+            final int[] ret = new int[k];
+            System.arraycopy(c, 1, ret, 0, k);
+
+            // Prepare next iteration
+            // T2 and T6 loop
+            int x = 0;
+            if (j > 0) {
+                x = j;
+                c[j] = x;
+                j--;
+                return ret;
+            }
+            // T3
+            if (c[1] + 1 < c[2]) {
+                c[1]++;
+                return ret;
+            } else {
+                j = 2;
+            }
+            // T4
+            boolean stepDone = false;
+            while (!stepDone) {
+                c[j - 1] = j - 2;
+                x = c[j] + 1;
+                if (x == c[j + 1]) {
+                    j++;
+                } else {
+                    stepDone = true;
+                }
+            }
+            // T5
+            if (j > k) {
+                more = false;
+                return ret;
+            }
+            // T6
+            c[j] = x;
+            j--;
+            return ret;
+        }
+
+        /**
+         * Not supported.
+         */
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Iterator with just one element to handle degenerate cases (full array,
+     * empty array) for combination iterator.
+     */
+    private static class SingletonIterator implements Iterator {
+        /** Singleton array */
+        private final int[] singleton;
+        /** True on initialization, false after first call to next */
+        private boolean more = true;
+        /**
+         * Create a singleton iterator providing the given array.
+         * @param singleton array returned by the iterator
+         */
+        SingletonIterator(final int[] singleton) {
+            this.singleton = singleton;
+        }
+        /** @return True until next is called the first time, then false */
+        public boolean hasNext() {
+            return more;
+        }
+        /** @return the singleton in first activation; throws NSEE thereafter */
+        public int[] next() {
+            if (more) {
+                more = false;
+                return singleton;
+            } else {
+                throw new NoSuchElementException();
+            }
+        }
+        /** Not supported */
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Defines the lexicographic ordering of combinations, using
+     * the {@link #lexNorm(int[])} method.
+     */
+    private static class LexicographicComparator
+        implements Comparator, Serializable {
+        /** Serializable version identifier. */
+        private static final long serialVersionUID = 20130906L;
+        /** Size of the set from which combinations are drawn. */
+        private final int n;
+        /** Number of elements in each combination. */
+        private final int k;
+
+        /**
+         * @param n Size of the set from which subsets are selected.
+         * @param k Size of the subsets to be enumerated.
+         */
+        LexicographicComparator(int n, int k) {
+            this.n = n;
+            this.k = k;
+        }
+
+        /**
+         * {@inheritDoc}
+         *
+         * @throws DimensionMismatchException if the array lengths are not
+         * equal to {@code k}.
+         * @throws OutOfRangeException if an element of the array is not
+         * within the interval [0, {@code n}).
+         */
+        public int compare(int[] c1,
+                           int[] c2) {
+            if (c1.length != k) {
+                throw new DimensionMismatchException(c1.length, k);
+            }
+            if (c2.length != k) {
+                throw new DimensionMismatchException(c2.length, k);
+            }
+
+            // Method "lexNorm" works with ordered arrays.
+            final int[] c1s = MathArrays.copyOf(c1);
+            Arrays.sort(c1s);
+            final int[] c2s = MathArrays.copyOf(c2);
+            Arrays.sort(c2s);
+
+            final long v1 = lexNorm(c1s);
+            final long v2 = lexNorm(c2s);
+
+            if (v1 < v2) {
+                return -1;
+            } else if (v1 > v2) {
+                return 1;
+            } else {
+                return 0;
+            }
+        }
+
+        /**
+         * Computes the value (in base 10) represented by the digit
+         * (interpreted in base {@code n}) in the input array in reverse
+         * order.
+         * For example if {@code c} is {@code {3, 2, 1}}, and {@code n}
+         * is 3, the method will return 18.
+         *
+         * @param c Input array.
+         * @return the lexicographic norm.
+         * @throws OutOfRangeException if an element of the array is not
+         * within the interval [0, {@code n}).
+         */
+        private long lexNorm(int[] c) {
+            long ret = 0;
+            for (int i = 0; i < c.length; i++) {
+                final int digit = c[i];
+                if (digit < 0 ||
+                    digit >= n) {
+                    throw new OutOfRangeException(digit, 0, n - 1);
+                }
+
+                ret += c[i] * ArithmeticUtils.pow(n, i);
+            }
+            return ret;
+        }
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/CombinatoricsUtils.java b/java/source/infodynamics/utils/commonsmath3/util/CombinatoricsUtils.java
new file mode 100644
index 0000000..670ddbe
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/CombinatoricsUtils.java
@@ -0,0 +1,491 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicReference;
+
+import infodynamics.utils.commonsmath3.exception.MathArithmeticException;
+import infodynamics.utils.commonsmath3.exception.NotPositiveException;
+import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
+/**
+ * Combinatorial utilities.
+ *
+ * @since 3.3
+ */
+public final class CombinatoricsUtils {
+
+    /** All long-representable factorials */
+    static final long[] FACTORIALS = new long[] {
+                       1l,                  1l,                   2l,
+                       6l,                 24l,                 120l,
+                     720l,               5040l,               40320l,
+                  362880l,            3628800l,            39916800l,
+               479001600l,         6227020800l,         87178291200l,
+           1307674368000l,     20922789888000l,     355687428096000l,
+        6402373705728000l, 121645100408832000l, 2432902008176640000l };
+
+    /** Stirling numbers of the second kind. */
+    static final AtomicReference STIRLING_S2 = new AtomicReference (null);
+
+    /** Private constructor (class contains only static methods). */
+    private CombinatoricsUtils() {}
+
+
+    /**
+     * Returns an exact representation of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code MathIllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code long}. The
+     * largest value of {@code n} for which all coefficients are
+     * {@code  < Long.MAX_VALUE} is 66. If the computed value exceeds
+     * {@code Long.MAX_VALUE} a {@code MathArithMeticException} is
+     * thrown.
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     */
+    public static long binomialCoefficient(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        CombinatoricsUtils.checkBinomial(n, k);
+        if ((n == k) || (k == 0)) {
+            return 1;
+        }
+        if ((k == 1) || (k == n - 1)) {
+            return n;
+        }
+        // Use symmetry for large k
+        if (k > n / 2) {
+            return binomialCoefficient(n, n - k);
+        }
+
+        // We use the formula
+        // (n choose k) = n! / (n-k)! / k!
+        // (n choose k) == ((n-k+1)*...*n) / (1*...*k)
+        // which could be written
+        // (n choose k) == (n-1 choose k-1) * n / k
+        long result = 1;
+        if (n <= 61) {
+            // For n <= 61, the naive implementation cannot overflow.
+            int i = n - k + 1;
+            for (int j = 1; j <= k; j++) {
+                result = result * i / j;
+                i++;
+            }
+        } else if (n <= 66) {
+            // For n > 61 but n <= 66, the result cannot overflow,
+            // but we must take care not to overflow intermediate values.
+            int i = n - k + 1;
+            for (int j = 1; j <= k; j++) {
+                // We know that (result * i) is divisible by j,
+                // but (result * i) may overflow, so we split j:
+                // Filter out the gcd, d, so j/d and i/d are integer.
+                // result is divisible by (j/d) because (j/d)
+                // is relative prime to (i/d) and is a divisor of
+                // result * (i/d).
+                final long d = ArithmeticUtils.gcd(i, j);
+                result = (result / (j / d)) * (i / d);
+                i++;
+            }
+        } else {
+            // For n > 66, a result overflow might occur, so we check
+            // the multiplication, taking care to not overflow
+            // unnecessary.
+            int i = n - k + 1;
+            for (int j = 1; j <= k; j++) {
+                final long d = ArithmeticUtils.gcd(i, j);
+                result = ArithmeticUtils.mulAndCheck(result / (j / d), i / d);
+                i++;
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Returns a {@code double} representation of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code IllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code double}. The
+     * largest value of {@code n} for which all coefficients are less than
+     * Double.MAX_VALUE is 1029. If the computed value exceeds Double.MAX_VALUE,
+     * Double.POSITIVE_INFINITY is returned
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     */
+    public static double binomialCoefficientDouble(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        CombinatoricsUtils.checkBinomial(n, k);
+        if ((n == k) || (k == 0)) {
+            return 1d;
+        }
+        if ((k == 1) || (k == n - 1)) {
+            return n;
+        }
+        if (k > n/2) {
+            return binomialCoefficientDouble(n, n - k);
+        }
+        if (n < 67) {
+            return binomialCoefficient(n,k);
+        }
+
+        double result = 1d;
+        for (int i = 1; i <= k; i++) {
+             result *= (double)(n - k + i) / (double)i;
+        }
+
+        return FastMath.floor(result + 0.5);
+    }
+
+    /**
+     * Returns the natural {@code log} of the  Binomial
+     * Coefficient, "{@code n choose k}", the number of
+     * {@code k}-element subsets that can be selected from an
+     * {@code n}-element set.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code 0 <= k <= n } (otherwise
+     * {@code MathIllegalArgumentException} is thrown)
    • 
+     * 

+     *
+     * @param n the size of the set
+     * @param k the size of the subsets to be counted
+     * @return {@code n choose k}
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if the result is too large to be
+     * represented by a long integer.
+     */
+    public static double binomialCoefficientLog(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        CombinatoricsUtils.checkBinomial(n, k);
+        if ((n == k) || (k == 0)) {
+            return 0;
+        }
+        if ((k == 1) || (k == n - 1)) {
+            return FastMath.log(n);
+        }
+
+        /*
+         * For values small enough to do exact integer computation,
+         * return the log of the exact value
+         */
+        if (n < 67) {
+            return FastMath.log(binomialCoefficient(n,k));
+        }
+
+        /*
+         * Return the log of binomialCoefficientDouble for values that will not
+         * overflow binomialCoefficientDouble
+         */
+        if (n < 1030) {
+            return FastMath.log(binomialCoefficientDouble(n, k));
+        }
+
+        if (k > n / 2) {
+            return binomialCoefficientLog(n, n - k);
+        }
+
+        /*
+         * Sum logs for values that could overflow
+         */
+        double logSum = 0;
+
+        // n!/(n-k)!
+        for (int i = n - k + 1; i <= n; i++) {
+            logSum += FastMath.log(i);
+        }
+
+        // divide by k!
+        for (int i = 2; i <= k; i++) {
+            logSum -= FastMath.log(i);
+        }
+
+        return logSum;
+    }
+
+    /**
+     * Returns n!. Shorthand for {@code n}  Factorial, the
+     * product of the numbers {@code 1,...,n}.
+     * 

+     * Preconditions:
+     * 
    
+     * 
  •  {@code n >= 0} (otherwise
+     * {@code MathIllegalArgumentException} is thrown)
    • 
+     * 
  •  The result is small enough to fit into a {@code long}. The
+     * largest value of {@code n} for which {@code n!} does not exceed
+     * Long.MAX_VALUE} is 20. If the computed value exceeds {@code Long.MAX_VALUE}
+     * an {@code MathArithMeticException } is thrown.
    • 
+     * 

+     * 

+     *
+     * @param n argument
+     * @return {@code n!}
+     * @throws MathArithmeticException if the result is too large to be represented
+     * by a {@code long}.
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws MathArithmeticException if {@code n > 20}: The factorial value is too
+     * large to fit in a {@code long}.
+     */
+    public static long factorial(final int n) throws NotPositiveException, MathArithmeticException {
+        if (n < 0) {
+            throw new NotPositiveException(LocalizedFormats.FACTORIAL_NEGATIVE_PARAMETER,
+                                           n);
+        }
+        if (n > 20) {
+            throw new MathArithmeticException();
+        }
+        return FACTORIALS[n];
+    }
+
+    /**
+     * Compute n!, the
+     * factorial of {@code n} (the product of the numbers 1 to n), as a
+     * {@code double}.
+     * The result should be small enough to fit into a {@code double}: The
+     * largest {@code n} for which {@code n!} does not exceed
+     * {@code Double.MAX_VALUE} is 170. If the computed value exceeds
+     * {@code Double.MAX_VALUE}, {@code Double.POSITIVE_INFINITY} is returned.
+     *
+     * @param n Argument.
+     * @return {@code n!}
+     * @throws NotPositiveException if {@code n < 0}.
+     */
+    public static double factorialDouble(final int n) throws NotPositiveException {
+        if (n < 0) {
+            throw new NotPositiveException(LocalizedFormats.FACTORIAL_NEGATIVE_PARAMETER,
+                                           n);
+        }
+        if (n < 21) {
+            return FACTORIALS[n];
+        }
+        return FastMath.floor(FastMath.exp(CombinatoricsUtils.factorialLog(n)) + 0.5);
+    }
+
+    /**
+     * Compute the natural logarithm of the factorial of {@code n}.
+     *
+     * @param n Argument.
+     * @return {@code n!}
+     * @throws NotPositiveException if {@code n < 0}.
+     */
+    public static double factorialLog(final int n) throws NotPositiveException {
+        if (n < 0) {
+            throw new NotPositiveException(LocalizedFormats.FACTORIAL_NEGATIVE_PARAMETER,
+                                           n);
+        }
+        if (n < 21) {
+            return FastMath.log(FACTORIALS[n]);
+        }
+        double logSum = 0;
+        for (int i = 2; i <= n; i++) {
+            logSum += FastMath.log(i);
+        }
+        return logSum;
+    }
+
+    /**
+     * Returns the 
+     * Stirling number of the second kind, "{@code S(n,k)}", the number of
+     * ways of partitioning an {@code n}-element set into {@code k} non-empty
+     * subsets.
+     * 

+     * The preconditions are {@code 0 <= k <= n } (otherwise
+     * {@code NotPositiveException} is thrown)
+     * 

+     * @param n the size of the set
+     * @param k the number of non-empty subsets
+     * @return {@code S(n,k)}
+     * @throws NotPositiveException if {@code k < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     * @throws MathArithmeticException if some overflow happens, typically for n exceeding 25 and
+     * k between 20 and n-2 (S(n,n-1) is handled specifically and does not overflow)
+     * @since 3.1
+     */
+    public static long stirlingS2(final int n, final int k)
+        throws NotPositiveException, NumberIsTooLargeException, MathArithmeticException {
+        if (k < 0) {
+            throw new NotPositiveException(k);
+        }
+        if (k > n) {
+            throw new NumberIsTooLargeException(k, n, true);
+        }
+
+        long[][] stirlingS2 = STIRLING_S2.get();
+
+        if (stirlingS2 == null) {
+            // the cache has never been initialized, compute the first numbers
+            // by direct recurrence relation
+
+            // as S(26,9) = 11201516780955125625 is larger than Long.MAX_VALUE
+            // we must stop computation at row 26
+            final int maxIndex = 26;
+            stirlingS2 = new long[maxIndex][];
+            stirlingS2[0] = new long[] { 1l };
+            for (int i = 1; i < stirlingS2.length; ++i) {
+                stirlingS2[i] = new long[i + 1];
+                stirlingS2[i][0] = 0;
+                stirlingS2[i][1] = 1;
+                stirlingS2[i][i] = 1;
+                for (int j = 2; j < i; ++j) {
+                    stirlingS2[i][j] = j * stirlingS2[i - 1][j] + stirlingS2[i - 1][j - 1];
+                }
+            }
+
+            // atomically save the cache
+            STIRLING_S2.compareAndSet(null, stirlingS2);
+
+        }
+
+        if (n < stirlingS2.length) {
+            // the number is in the small cache
+            return stirlingS2[n][k];
+        } else {
+            // use explicit formula to compute the number without caching it
+            if (k == 0) {
+                return 0;
+            } else if (k == 1 || k == n) {
+                return 1;
+            } else if (k == 2) {
+                return (1l << (n - 1)) - 1l;
+            } else if (k == n - 1) {
+                return binomialCoefficient(n, 2);
+            } else {
+                // definition formula: note that this may trigger some overflow
+                long sum = 0;
+                long sign = ((k & 0x1) == 0) ? 1 : -1;
+                for (int j = 1; j <= k; ++j) {
+                    sign = -sign;
+                    sum += sign * binomialCoefficient(k, j) * ArithmeticUtils.pow(j, n);
+                    if (sum < 0) {
+                        // there was an overflow somewhere
+                        throw new MathArithmeticException(LocalizedFormats.ARGUMENT_OUTSIDE_DOMAIN,
+                                                          n, 0, stirlingS2.length - 1);
+                    }
+                }
+                return sum / factorial(k);
+            }
+        }
+
+    }
+
+    /**
+     * Returns an iterator whose range is the k-element subsets of {0, ..., n - 1}
+     * represented as {@code int[]} arrays.
+     * 

+     * The arrays returned by the iterator are sorted in descending order and
+     * they are visited in lexicographic order with significance from right to
+     * left. For example, combinationsIterator(4, 2) returns an Iterator that
+     * will generate the following sequence of arrays on successive calls to
+     * {@code next()}:

+     * {@code [0, 1], [0, 2], [1, 2], [0, 3], [1, 3], [2, 3]}
+     * 

+     * If {@code k == 0} an Iterator containing an empty array is returned and
+     * if {@code k == n} an Iterator containing [0, ..., n -1] is returned.

+     *
+     * @param n Size of the set from which subsets are selected.
+     * @param k Size of the subsets to be enumerated.
+     * @return an {@link Iterator iterator} over the k-sets in n.
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     */
+    public static Iterator combinationsIterator(int n, int k) {
+        return new Combinations(n, k).iterator();
+    }
+
+    /**
+     * Check binomial preconditions.
+     *
+     * @param n Size of the set.
+     * @param k Size of the subsets to be counted.
+     * @throws NotPositiveException if {@code n < 0}.
+     * @throws NumberIsTooLargeException if {@code k > n}.
+     */
+    public static void checkBinomial(final int n,
+                                     final int k)
+        throws NumberIsTooLargeException,
+               NotPositiveException {
+        if (n < k) {
+            throw new NumberIsTooLargeException(LocalizedFormats.BINOMIAL_INVALID_PARAMETERS_ORDER,
+                                                k, n, true);
+        }
+        if (n < 0) {
+            throw new NotPositiveException(LocalizedFormats.BINOMIAL_NEGATIVE_PARAMETER, n);
+        }
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/ContinuedFraction.java b/java/source/infodynamics/utils/commonsmath3/util/ContinuedFraction.java
old mode 100755
new mode 100644
index 7762e1b..52c6173
--- a/java/source/infodynamics/utils/commonsmath3/util/ContinuedFraction.java
+++ b/java/source/infodynamics/utils/commonsmath3/util/ContinuedFraction.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -61,7 +61,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
  * 
  * 

  *
- * @version $Id$
  */
 public abstract class ContinuedFraction {
     /** Maximum allowed numerical error. */
diff --git a/java/source/infodynamics/utils/commonsmath3/util/DoubleArray.java b/java/source/infodynamics/utils/commonsmath3/util/DoubleArray.java
new file mode 100644
index 0000000..3c3594b
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/DoubleArray.java
@@ -0,0 +1,139 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+
+/**
+ * Provides a standard interface for double arrays.  Allows different
+ * array implementations to support various storage mechanisms
+ * such as automatic expansion, contraction, and array "rolling".
+ *
+ */
+public interface DoubleArray {
+
+    /**
+     * Returns the number of elements currently in the array.  Please note
+     * that this may be different from the length of the internal storage array.
+     *
+     * @return number of elements
+     */
+    int getNumElements();
+
+    /**
+     * Returns the element at the specified index.  Note that if an
+     * out of bounds index is supplied a ArrayIndexOutOfBoundsException
+     * will be thrown.
+     *
+     * @param index index to fetch a value from
+     * @return value stored at the specified index
+     * @throws ArrayIndexOutOfBoundsException if index is less than
+     *         zero or is greater than getNumElements() - 1.
+     */
+    double getElement(int index);
+
+    /**
+     * Sets the element at the specified index.  If the specified index is greater than
+     * getNumElements() - 1, the numElements property
+     * is increased to index +1 and additional storage is allocated
+     * (if necessary) for the new element and all  (uninitialized) elements
+     * between the new element and the previous end of the array).
+     *
+     * @param index index to store a value in
+     * @param value value to store at the specified index
+     * @throws ArrayIndexOutOfBoundsException if index is less than
+     *         zero.
+     */
+    void setElement(int index, double value);
+
+    /**
+     * Adds an element to the end of this expandable array
+     *
+     * @param value to be added to end of array
+     */
+    void addElement(double value);
+
+    /**
+     * Adds elements to the end of this expandable array
+     *
+     * @param values to be added to end of array
+     */
+    void addElements(double[] values);
+
+    /**
+     * 

+     * Adds an element to the end of the array and removes the first
+     * element in the array.  Returns the discarded first element.
+     * The effect is similar to a push operation in a FIFO queue.
+     * 

+     * 

+     * Example: If the array contains the elements 1, 2, 3, 4 (in that order)
+     * and addElementRolling(5) is invoked, the result is an array containing
+     * the entries 2, 3, 4, 5 and the value returned is 1.
+     * 

+     *
+     * @param value the value to be added to the array
+     * @return the value which has been discarded or "pushed" out of the array
+     *         by this rolling insert
+     */
+    double addElementRolling(double value);
+
+    /**
+     * Returns a double[] array containing the elements of this
+     * DoubleArray.  If the underlying implementation is
+     * array-based, this method should always return a copy, rather than a
+     * reference to the underlying array so that changes made to the returned
+     *  array have no effect on the DoubleArray.
+     *
+     * @return all elements added to the array
+     */
+    double[] getElements();
+
+    /**
+     * Clear the double array
+     */
+    void clear();
+
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/FastMath.java b/java/source/infodynamics/utils/commonsmath3/util/FastMath.java
old mode 100755
new mode 100644
index e6dac2d..cfe720c
--- a/java/source/infodynamics/utils/commonsmath3/util/FastMath.java
+++ b/java/source/infodynamics/utils/commonsmath3/util/FastMath.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -47,6 +47,9 @@ package infodynamics.utils.commonsmath3.util;
 
 import java.io.PrintStream;
 
+import infodynamics.utils.commonsmath3.exception.MathArithmeticException;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
 /**
  * Faster, more accurate, portable alternative to {@link Math} and
  * {@link StrictMath} for large scale computation.
@@ -103,7 +106,6 @@ import java.io.PrintStream;
  * 
  • {@link #scalb(float, int)}
    • 
  * 
  * 
    
- * @version $Id$
  * @since 2.2
  */
 public class FastMath {
@@ -342,10 +344,17 @@ public class FastMath {
     /** Mask used to clear the non-sign part of a long. */
     private static final long MASK_NON_SIGN_LONG = 0x7fffffffffffffffl;
 
+    /** Mask used to extract exponent from double bits. */
+    private static final long MASK_DOUBLE_EXPONENT = 0x7ff0000000000000L;
+
+    /** Mask used to extract mantissa from double bits. */
+    private static final long MASK_DOUBLE_MANTISSA = 0x000fffffffffffffL;
+
+    /** Mask used to add implicit high order bit for normalized double. */
+    private static final long IMPLICIT_HIGH_BIT = 0x0010000000000000L;
+
     /** 2^52 - double numbers this large must be integral (no fraction) or NaN or Infinite */
     private static final double TWO_POWER_52 = 4503599627370496.0;
-    /** 2^53 - double numbers this large must be even. */
-    private static final double TWO_POWER_53 = 2 * TWO_POWER_52;
 
     /** Constant: {@value}. */
     private static final double F_1_3 = 1d / 3d;
@@ -834,6 +843,24 @@ public class FastMath {
         return nextAfter(a, Float.POSITIVE_INFINITY);
     }
 
+    /** Compute next number towards negative infinity.
+     * @param a number to which neighbor should be computed
+     * @return neighbor of a towards negative infinity
+     * @since 3.4
+     */
+    public static double nextDown(final double a) {
+        return nextAfter(a, Double.NEGATIVE_INFINITY);
+    }
+
+    /** Compute next number towards negative infinity.
+     * @param a number to which neighbor should be computed
+     * @return neighbor of a towards negative infinity
+     * @since 3.4
+     */
+    public static float nextDown(final float a) {
+        return nextAfter(a, Float.NEGATIVE_INFINITY);
+    }
+
     /** Returns a pseudo-random number between 0.0 and 1.0.
      * 
    Note: this implementation currently delegates to {@link Math#random}
      * @return a random number between 0.0 and 1.0
@@ -847,7 +874,7 @@ public class FastMath {
      *
      * Computes exp(x), function result is nearly rounded.   It will be correctly
      * rounded to the theoretical value for 99.9% of input values, otherwise it will
-     * have a 1 UPL error.
+     * have a 1 ULP error.
      *
      * Method:
      *    Lookup intVal = exp(int(x))
@@ -876,16 +903,17 @@ public class FastMath {
     private static double exp(double x, double extra, double[] hiPrec) {
         double intPartA;
         double intPartB;
-        int intVal;
+        int intVal = (int) x;
 
         /* Lookup exp(floor(x)).
          * intPartA will have the upper 22 bits, intPartB will have the lower
          * 52 bits.
          */
         if (x < 0.0) {
-            intVal = (int) -x;
 
-            if (intVal > 746) {
+            // We don't check against intVal here as conversion of large negative double values
+            // may be affected by a JIT bug. Subsequent comparisons can safely use intVal
+            if (x < -746d) {
                 if (hiPrec != null) {
                     hiPrec[0] = 0.0;
                     hiPrec[1] = 0.0;
@@ -893,7 +921,7 @@ public class FastMath {
                 return 0.0;
             }
 
-            if (intVal > 709) {
+            if (intVal < -709) {
                 /* This will produce a subnormal output */
                 final double result = exp(x+40.19140625, extra, hiPrec) / 285040095144011776.0;
                 if (hiPrec != null) {
@@ -903,7 +931,7 @@ public class FastMath {
                 return result;
             }
 
-            if (intVal == 709) {
+            if (intVal == -709) {
                 /* exp(1.494140625) is nearly a machine number... */
                 final double result = exp(x+1.494140625, extra, hiPrec) / 4.455505956692756620;
                 if (hiPrec != null) {
@@ -913,15 +941,9 @@ public class FastMath {
                 return result;
             }
 
-            intVal++;
+            intVal--;
 
-            intPartA = ExpIntTable.EXP_INT_TABLE_A[EXP_INT_TABLE_MAX_INDEX-intVal];
-            intPartB = ExpIntTable.EXP_INT_TABLE_B[EXP_INT_TABLE_MAX_INDEX-intVal];
-
-            intVal = -intVal;
         } else {
-            intVal = (int) x;
-
             if (intVal > 709) {
                 if (hiPrec != null) {
                     hiPrec[0] = Double.POSITIVE_INFINITY;
@@ -930,10 +952,11 @@ public class FastMath {
                 return Double.POSITIVE_INFINITY;
             }
 
-            intPartA = ExpIntTable.EXP_INT_TABLE_A[EXP_INT_TABLE_MAX_INDEX+intVal];
-            intPartB = ExpIntTable.EXP_INT_TABLE_B[EXP_INT_TABLE_MAX_INDEX+intVal];
         }
 
+        intPartA = ExpIntTable.EXP_INT_TABLE_A[EXP_INT_TABLE_MAX_INDEX+intVal];
+        intPartB = ExpIntTable.EXP_INT_TABLE_B[EXP_INT_TABLE_MAX_INDEX+intVal];
+
         /* Get the fractional part of x, find the greatest multiple of 2^-10 less than
          * x and look up the exp function of it.
          * fracPartA will have the upper 22 bits, fracPartB the lower 52 bits.
@@ -944,13 +967,13 @@ public class FastMath {
 
         /* epsilon is the difference in x from the nearest multiple of 2^-10.  It
          * has a value in the range 0 <= epsilon < 2^-10.
-         * Do the subtraction from x as the last step to avoid possible loss of percison.
+         * Do the subtraction from x as the last step to avoid possible loss of precision.
          */
         final double epsilon = x - (intVal + intFrac / 1024.0);
 
         /* Compute z = exp(epsilon) - 1.0 via a minimax polynomial.  z has
        full double precision (52 bits).  Since z < 2^-10, we will have
-       62 bits of precision when combined with the contant 1.  This will be
+       62 bits of precision when combined with the constant 1.  This will be
        used in the last addition below to get proper rounding. */
 
         /* Remez generated polynomial.  Converges on the interval [0, 2^-10], error
@@ -974,6 +997,13 @@ public class FastMath {
        much larger than the others.  If there are extra bits specified from the
        pow() function, use them. */
         final double tempC = tempB + tempA;
+
+        // If tempC is positive infinite, the evaluation below could result in NaN,
+        // because z could be negative at the same time.
+        if (tempC == Double.POSITIVE_INFINITY) {
+            return Double.POSITIVE_INFINITY;
+        }
+
         final double result;
         if (extra != 0.0) {
             result = tempC*extra*z + tempC*extra + tempC*z + tempB + tempA;
@@ -1470,167 +1500,142 @@ public class FastMath {
      * @param y   a double
      * @return double
      */
-    public static double pow(double x, double y) {
-        final double lns[] = new double[2];
+    public static double pow(final double x, final double y) {
 
-        if (y == 0.0) {
+        if (y == 0) {
+            // y = -0 or y = +0
             return 1.0;
-        }
-
-        if (x != x) { // X is NaN
-            return x;
-        }
-
-
-        if (x == 0) {
-            long bits = Double.doubleToRawLongBits(x);
-            if ((bits & 0x8000000000000000L) != 0) {
-                // -zero
-                long yi = (long) y;
-
-                if (y < 0 && y == yi && (yi & 1) == 1) {
-                    return Double.NEGATIVE_INFINITY;
-                }
-
-                if (y > 0 && y == yi && (yi & 1) == 1) {
-                    return -0.0;
-                }
-            }
-
-            if (y < 0) {
-                return Double.POSITIVE_INFINITY;
-            }
-            if (y > 0) {
-                return 0.0;
-            }
-
-            return Double.NaN;
-        }
-
-        if (x == Double.POSITIVE_INFINITY) {
-            if (y != y) { // y is NaN
-                return y;
-            }
-            if (y < 0.0) {
-                return 0.0;
-            } else {
-                return Double.POSITIVE_INFINITY;
-            }
-        }
-
-        if (y == Double.POSITIVE_INFINITY) {
-            if (x * x == 1.0) {
-                return Double.NaN;
-            }
-
-            if (x * x > 1.0) {
-                return Double.POSITIVE_INFINITY;
-            } else {
-                return 0.0;
-            }
-        }
-
-        if (x == Double.NEGATIVE_INFINITY) {
-            if (y != y) { // y is NaN
-                return y;
-            }
-
-            if (y < 0) {
-                long yi = (long) y;
-                if (y == yi && (yi & 1) == 1) {
-                    return -0.0;
-                }
-
-                return 0.0;
-            }
-
-            if (y > 0)  {
-                long yi = (long) y;
-                if (y == yi && (yi & 1) == 1) {
-                    return Double.NEGATIVE_INFINITY;
-                }
-
-                return Double.POSITIVE_INFINITY;
-            }
-        }
-
-        if (y == Double.NEGATIVE_INFINITY) {
-
-            if (x * x == 1.0) {
-                return Double.NaN;
-            }
-
-            if (x * x < 1.0) {
-                return Double.POSITIVE_INFINITY;
-            } else {
-                return 0.0;
-            }
-        }
-
-        /* Handle special case x<0 */
-        if (x < 0) {
-            // y is an even integer in this case
-            if (y >= TWO_POWER_53 || y <= -TWO_POWER_53) {
-                return pow(-x, y);
-            }
-
-            if (y == (long) y) {
-                // If y is an integer
-                return ((long)y & 1) == 0 ? pow(-x, y) : -pow(-x, y);
-            } else {
-                return Double.NaN;
-            }
-        }
-
-        /* Split y into ya and yb such that y = ya+yb */
-        double ya;
-        double yb;
-        if (y < 8e298 && y > -8e298) {
-            double tmp1 = y * HEX_40000000;
-            ya = y + tmp1 - tmp1;
-            yb = y - ya;
         } else {
-            double tmp1 = y * 9.31322574615478515625E-10;
-            double tmp2 = tmp1 * 9.31322574615478515625E-10;
-            ya = (tmp1 + tmp2 - tmp1) * HEX_40000000 * HEX_40000000;
-            yb = y - ya;
+
+            final long yBits        = Double.doubleToRawLongBits(y);
+            final int  yRawExp      = (int) ((yBits & MASK_DOUBLE_EXPONENT) >> 52);
+            final long yRawMantissa = yBits & MASK_DOUBLE_MANTISSA;
+            final long xBits        = Double.doubleToRawLongBits(x);
+            final int  xRawExp      = (int) ((xBits & MASK_DOUBLE_EXPONENT) >> 52);
+            final long xRawMantissa = xBits & MASK_DOUBLE_MANTISSA;
+
+            if (yRawExp > 1085) {
+                // y is either a very large integral value that does not fit in a long or it is a special number
+
+                if ((yRawExp == 2047 && yRawMantissa != 0) ||
+                    (xRawExp == 2047 && xRawMantissa != 0)) {
+                    // NaN
+                    return Double.NaN;
+                } else if (xRawExp == 1023 && xRawMantissa == 0) {
+                    // x = -1.0 or x = +1.0
+                    if (yRawExp == 2047) {
+                        // y is infinite
+                        return Double.NaN;
+                    } else {
+                        // y is a large even integer
+                        return 1.0;
+                    }
+                } else {
+                    // the absolute value of x is either greater or smaller than 1.0
+
+                    // if yRawExp == 2047 and mantissa is 0, y = -infinity or y = +infinity
+                    // if 1085 < yRawExp < 2047, y is simply a large number, however, due to limited
+                    // accuracy, at this magnitude it behaves just like infinity with regards to x
+                    if ((y > 0) ^ (xRawExp < 1023)) {
+                        // either y = +infinity (or large engouh) and abs(x) > 1.0
+                        // or     y = -infinity (or large engouh) and abs(x) < 1.0
+                        return Double.POSITIVE_INFINITY;
+                    } else {
+                        // either y = +infinity (or large engouh) and abs(x) < 1.0
+                        // or     y = -infinity (or large engouh) and abs(x) > 1.0
+                        return +0.0;
+                    }
+                }
+
+            } else {
+                // y is a regular non-zero number
+
+                if (yRawExp >= 1023) {
+                    // y may be an integral value, which should be handled specifically
+                    final long yFullMantissa = IMPLICIT_HIGH_BIT | yRawMantissa;
+                    if (yRawExp < 1075) {
+                        // normal number with negative shift that may have a fractional part
+                        final long integralMask = (-1L) << (1075 - yRawExp);
+                        if ((yFullMantissa & integralMask) == yFullMantissa) {
+                            // all fractional bits are 0, the number is really integral
+                            final long l = yFullMantissa >> (1075 - yRawExp);
+                            return FastMath.pow(x, (y < 0) ? -l : l);
+                        }
+                    } else {
+                        // normal number with positive shift, always an integral value
+                        // we know it fits in a primitive long because yRawExp > 1085 has been handled above
+                        final long l =  yFullMantissa << (yRawExp - 1075);
+                        return FastMath.pow(x, (y < 0) ? -l : l);
+                    }
+                }
+
+                // y is a non-integral value
+
+                if (x == 0) {
+                    // x = -0 or x = +0
+                    // the integer powers have already been handled above
+                    return y < 0 ? Double.POSITIVE_INFINITY : +0.0;
+                } else if (xRawExp == 2047) {
+                    if (xRawMantissa == 0) {
+                        // x = -infinity or x = +infinity
+                        return (y < 0) ? +0.0 : Double.POSITIVE_INFINITY;
+                    } else {
+                        // NaN
+                        return Double.NaN;
+                    }
+                } else if (x < 0) {
+                    // the integer powers have already been handled above
+                    return Double.NaN;
+                } else {
+
+                    // this is the general case, for regular fractional numbers x and y
+
+                    // Split y into ya and yb such that y = ya+yb
+                    final double tmp = y * HEX_40000000;
+                    final double ya = (y + tmp) - tmp;
+                    final double yb = y - ya;
+
+                    /* Compute ln(x) */
+                    final double lns[] = new double[2];
+                    final double lores = log(x, lns);
+                    if (Double.isInfinite(lores)) { // don't allow this to be converted to NaN
+                        return lores;
+                    }
+
+                    double lna = lns[0];
+                    double lnb = lns[1];
+
+                    /* resplit lns */
+                    final double tmp1 = lna * HEX_40000000;
+                    final double tmp2 = (lna + tmp1) - tmp1;
+                    lnb += lna - tmp2;
+                    lna = tmp2;
+
+                    // y*ln(x) = (aa+ab)
+                    final double aa = lna * ya;
+                    final double ab = lna * yb + lnb * ya + lnb * yb;
+
+                    lna = aa+ab;
+                    lnb = -(lna - aa - ab);
+
+                    double z = 1.0 / 120.0;
+                    z = z * lnb + (1.0 / 24.0);
+                    z = z * lnb + (1.0 / 6.0);
+                    z = z * lnb + 0.5;
+                    z = z * lnb + 1.0;
+                    z *= lnb;
+
+                    final double result = exp(lna, z, null);
+                    //result = result + result * z;
+                    return result;
+
+                }
+            }
+
         }
 
-        /* Compute ln(x) */
-        final double lores = log(x, lns);
-        if (Double.isInfinite(lores)){ // don't allow this to be converted to NaN
-            return lores;
-        }
-
-        double lna = lns[0];
-        double lnb = lns[1];
-
-        /* resplit lns */
-        double tmp1 = lna * HEX_40000000;
-        double tmp2 = lna + tmp1 - tmp1;
-        lnb += lna - tmp2;
-        lna = tmp2;
-
-        // y*ln(x) = (aa+ab)
-        final double aa = lna * ya;
-        final double ab = lna * yb + lnb * ya + lnb * yb;
-
-        lna = aa+ab;
-        lnb = -(lna - aa - ab);
-
-        double z = 1.0 / 120.0;
-        z = z * lnb + (1.0 / 24.0);
-        z = z * lnb + (1.0 / 6.0);
-        z = z * lnb + 0.5;
-        z = z * lnb + 1.0;
-        z *= lnb;
-
-        final double result = exp(lna, z, null);
-        //result = result + result * z;
-        return result;
     }
 
-
     /**
      * Raise a double to an int power.
      *
@@ -1640,62 +1645,151 @@ public class FastMath {
      * @since 3.1
      */
     public static double pow(double d, int e) {
+        return pow(d, (long) e);
+    }
 
+    /**
+     * Raise a double to a long power.
+     *
+     * @param d Number to raise.
+     * @param e Exponent.
+     * @return de
+     * @since 3.6
+     */
+    public static double pow(double d, long e) {
         if (e == 0) {
             return 1.0;
-        } else if (e < 0) {
-            e = -e;
-            d = 1.0 / d;
+        } else if (e > 0) {
+            return new Split(d).pow(e).full;
+        } else {
+            return new Split(d).reciprocal().pow(-e).full;
+        }
+    }
+
+    /** Class operator on double numbers split into one 26 bits number and one 27 bits number. */
+    private static class Split {
+
+        /** Split version of NaN. */
+        public static final Split NAN = new Split(Double.NaN, 0);
+
+        /** Split version of positive infinity. */
+        public static final Split POSITIVE_INFINITY = new Split(Double.POSITIVE_INFINITY, 0);
+
+        /** Split version of negative infinity. */
+        public static final Split NEGATIVE_INFINITY = new Split(Double.NEGATIVE_INFINITY, 0);
+
+        /** Full number. */
+        private final double full;
+
+        /** High order bits. */
+        private final double high;
+
+        /** Low order bits. */
+        private final double low;
+
+        /** Simple constructor.
+         * @param x number to split
+         */
+        Split(final double x) {
+            full = x;
+            high = Double.longBitsToDouble(Double.doubleToRawLongBits(x) & ((-1L) << 27));
+            low  = x - high;
         }
 
-        // split d as two 26 bits numbers
-        // beware the following expressions must NOT be simplified, they rely on floating point arithmetic properties
-        final int splitFactor = 0x8000001;
-        final double cd       = splitFactor * d;
-        final double d1High   = cd - (cd - d);
-        final double d1Low    = d - d1High;
+        /** Simple constructor.
+         * @param high high order bits
+         * @param low low order bits
+         */
+        Split(final double high, final double low) {
+            this(high == 0.0 ? (low == 0.0 && Double.doubleToRawLongBits(high) == Long.MIN_VALUE /* negative zero */ ? -0.0 : low) : high + low, high, low);
+        }
 
-        // prepare result
-        double resultHigh = 1;
-        double resultLow  = 0;
+        /** Simple constructor.
+         * @param full full number
+         * @param high high order bits
+         * @param low low order bits
+         */
+        Split(final double full, final double high, final double low) {
+            this.full = full;
+            this.high = high;
+            this.low  = low;
+        }
 
-        // d^(2p)
-        double d2p     = d;
-        double d2pHigh = d1High;
-        double d2pLow  = d1Low;
+        /** Multiply the instance by another one.
+         * @param b other instance to multiply by
+         * @return product
+         */
+        public Split multiply(final Split b) {
+            // beware the following expressions must NOT be simplified, they rely on floating point arithmetic properties
+            final Split  mulBasic  = new Split(full * b.full);
+            final double mulError  = low * b.low - (((mulBasic.full - high * b.high) - low * b.high) - high * b.low);
+            return new Split(mulBasic.high, mulBasic.low + mulError);
+        }
 
-        while (e != 0) {
+        /** Compute the reciprocal of the instance.
+         * @return reciprocal of the instance
+         */
+        public Split reciprocal() {
+
+            final double approximateInv = 1.0 / full;
+            final Split  splitInv       = new Split(approximateInv);
+
+            // if 1.0/d were computed perfectly, remultiplying it by d should give 1.0
+            // we want to estimate the error so we can fix the low order bits of approximateInvLow
+            // beware the following expressions must NOT be simplified, they rely on floating point arithmetic properties
+            final Split product = multiply(splitInv);
+            final double error  = (product.high - 1) + product.low;
+
+            // better accuracy estimate of reciprocal
+            return Double.isNaN(error) ? splitInv : new Split(splitInv.high, splitInv.low - error / full);
+
+        }
+
+        /** Computes this^e.
+         * @param e exponent (beware, here it MUST be > 0; the only exclusion is Long.MIN_VALUE)
+         * @return d^e, split in high and low bits
+         * @since 3.6
+         */
+        private Split pow(final long e) {
+
+            // prepare result
+            Split result = new Split(1);
+
+            // d^(2p)
+            Split d2p = new Split(full, high, low);
+
+            for (long p = e; p != 0; p >>>= 1) {
+
+                if ((p & 0x1) != 0) {
+                    // accurate multiplication result = result * d^(2p) using Veltkamp TwoProduct algorithm
+                    result = result.multiply(d2p);
+                }
+
+                // accurate squaring d^(2(p+1)) = d^(2p) * d^(2p) using Veltkamp TwoProduct algorithm
+                d2p = d2p.multiply(d2p);
 
-            if ((e & 0x1) != 0) {
-                // accurate multiplication result = result * d^(2p) using Veltkamp TwoProduct algorithm
-                // beware the following expressions must NOT be simplified, they rely on floating point arithmetic properties
-                final double tmpHigh = resultHigh * d2p;
-                final double cRH     = splitFactor * resultHigh;
-                final double rHH     = cRH - (cRH - resultHigh);
-                final double rHL     = resultHigh - rHH;
-                final double tmpLow  = rHL * d2pLow - (((tmpHigh - rHH * d2pHigh) - rHL * d2pHigh) - rHH * d2pLow);
-                resultHigh = tmpHigh;
-                resultLow  = resultLow * d2p + tmpLow;
             }
 
-            // accurate squaring d^(2(p+1)) = d^(2p) * d^(2p) using Veltkamp TwoProduct algorithm
-            // beware the following expressions must NOT be simplified, they rely on floating point arithmetic properties
-            final double tmpHigh = d2pHigh * d2p;
-            final double cD2pH   = splitFactor * d2pHigh;
-            final double d2pHH   = cD2pH - (cD2pH - d2pHigh);
-            final double d2pHL   = d2pHigh - d2pHH;
-            final double tmpLow  = d2pHL * d2pLow - (((tmpHigh - d2pHH * d2pHigh) - d2pHL * d2pHigh) - d2pHH * d2pLow);
-            final double cTmpH   = splitFactor * tmpHigh;
-            d2pHigh = cTmpH - (cTmpH - tmpHigh);
-            d2pLow  = d2pLow * d2p + tmpLow + (tmpHigh - d2pHigh);
-            d2p     = d2pHigh + d2pLow;
-
-            e >>= 1;
+            if (Double.isNaN(result.full)) {
+                if (Double.isNaN(full)) {
+                    return Split.NAN;
+                } else {
+                    // some intermediate numbers exceeded capacity,
+                    // and the low order bits became NaN (because infinity - infinity = NaN)
+                    if (FastMath.abs(full) < 1) {
+                        return new Split(FastMath.copySign(0.0, full), 0.0);
+                    } else if (full < 0 && (e & 0x1) == 1) {
+                        return Split.NEGATIVE_INFINITY;
+                    } else {
+                        return Split.POSITIVE_INFINITY;
+                    }
+                }
+            } else {
+                return result;
+            }
 
         }
 
-        return resultHigh + resultLow;
-
     }
 
     /**
@@ -3664,6 +3758,319 @@ public class FastMath {
         return StrictMath.IEEEremainder(dividend, divisor); // TODO provide our own implementation
     }
 
+    /** Convert a long to interger, detecting overflows
+     * @param n number to convert to int
+     * @return integer with same valie as n if no overflows occur
+     * @exception MathArithmeticException if n cannot fit into an int
+     * @since 3.4
+     */
+    public static int toIntExact(final long n) throws MathArithmeticException {
+        if (n < Integer.MIN_VALUE || n > Integer.MAX_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW);
+        }
+        return (int) n;
+    }
+
+    /** Increment a number, detecting overflows.
+     * @param n number to increment
+     * @return n+1 if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static int incrementExact(final int n) throws MathArithmeticException {
+
+        if (n == Integer.MAX_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, n, 1);
+        }
+
+        return n + 1;
+
+    }
+
+    /** Increment a number, detecting overflows.
+     * @param n number to increment
+     * @return n+1 if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static long incrementExact(final long n) throws MathArithmeticException {
+
+        if (n == Long.MAX_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, n, 1);
+        }
+
+        return n + 1;
+
+    }
+
+    /** Decrement a number, detecting overflows.
+     * @param n number to decrement
+     * @return n-1 if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static int decrementExact(final int n) throws MathArithmeticException {
+
+        if (n == Integer.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_SUBTRACTION, n, 1);
+        }
+
+        return n - 1;
+
+    }
+
+    /** Decrement a number, detecting overflows.
+     * @param n number to decrement
+     * @return n-1 if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static long decrementExact(final long n) throws MathArithmeticException {
+
+        if (n == Long.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_SUBTRACTION, n, 1);
+        }
+
+        return n - 1;
+
+    }
+
+    /** Add two numbers, detecting overflows.
+     * @param a first number to add
+     * @param b second number to add
+     * @return a+b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static int addExact(final int a, final int b) throws MathArithmeticException {
+
+        // compute sum
+        final int sum = a + b;
+
+        // check for overflow
+        if ((a ^ b) >= 0 && (sum ^ b) < 0) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, a, b);
+        }
+
+        return sum;
+
+    }
+
+    /** Add two numbers, detecting overflows.
+     * @param a first number to add
+     * @param b second number to add
+     * @return a+b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static long addExact(final long a, final long b) throws MathArithmeticException {
+
+        // compute sum
+        final long sum = a + b;
+
+        // check for overflow
+        if ((a ^ b) >= 0 && (sum ^ b) < 0) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_ADDITION, a, b);
+        }
+
+        return sum;
+
+    }
+
+    /** Subtract two numbers, detecting overflows.
+     * @param a first number
+     * @param b second number to subtract from a
+     * @return a-b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static int subtractExact(final int a, final int b) {
+
+        // compute subtraction
+        final int sub = a - b;
+
+        // check for overflow
+        if ((a ^ b) < 0 && (sub ^ b) >= 0) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_SUBTRACTION, a, b);
+        }
+
+        return sub;
+
+    }
+
+    /** Subtract two numbers, detecting overflows.
+     * @param a first number
+     * @param b second number to subtract from a
+     * @return a-b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static long subtractExact(final long a, final long b) {
+
+        // compute subtraction
+        final long sub = a - b;
+
+        // check for overflow
+        if ((a ^ b) < 0 && (sub ^ b) >= 0) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_SUBTRACTION, a, b);
+        }
+
+        return sub;
+
+    }
+
+    /** Multiply two numbers, detecting overflows.
+     * @param a first number to multiply
+     * @param b second number to multiply
+     * @return a*b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static int multiplyExact(final int a, final int b) {
+        if (((b  >  0)  && (a > Integer.MAX_VALUE / b || a < Integer.MIN_VALUE / b)) ||
+            ((b  < -1)  && (a > Integer.MIN_VALUE / b || a < Integer.MAX_VALUE / b)) ||
+            ((b == -1)  && (a == Integer.MIN_VALUE))) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_MULTIPLICATION, a, b);
+        }
+        return a * b;
+    }
+
+    /** Multiply two numbers, detecting overflows.
+     * @param a first number to multiply
+     * @param b second number to multiply
+     * @return a*b if no overflows occur
+     * @exception MathArithmeticException if an overflow occurs
+     * @since 3.4
+     */
+    public static long multiplyExact(final long a, final long b) {
+        if (((b  >  0l)  && (a > Long.MAX_VALUE / b || a < Long.MIN_VALUE / b)) ||
+            ((b  < -1l)  && (a > Long.MIN_VALUE / b || a < Long.MAX_VALUE / b)) ||
+            ((b == -1l)  && (a == Long.MIN_VALUE))) {
+                throw new MathArithmeticException(LocalizedFormats.OVERFLOW_IN_MULTIPLICATION, a, b);
+            }
+            return a * b;
+    }
+
+    /** Finds q such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0.
+     * 
    
+     * This methods returns the same value as integer division when
+     * a and b are same signs, but returns a different value when
+     * they are opposite (i.e. q is negative).
+     * 
    
+     * @param a dividend
+     * @param b divisor
+     * @return q such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0
+     * @exception MathArithmeticException if b == 0
+     * @see #floorMod(int, int)
+     * @since 3.4
+     */
+    public static int floorDiv(final int a, final int b) throws MathArithmeticException {
+
+        if (b == 0) {
+            throw new MathArithmeticException(LocalizedFormats.ZERO_DENOMINATOR);
+        }
+
+        final int m = a % b;
+        if ((a ^ b) >= 0 || m == 0) {
+            // a an b have same sign, or division is exact
+            return a / b;
+        } else {
+            // a and b have opposite signs and division is not exact
+            return (a / b) - 1;
+        }
+
+    }
+
+    /** Finds q such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0.
+     * 
    
+     * This methods returns the same value as integer division when
+     * a and b are same signs, but returns a different value when
+     * they are opposite (i.e. q is negative).
+     * 
    
+     * @param a dividend
+     * @param b divisor
+     * @return q such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0
+     * @exception MathArithmeticException if b == 0
+     * @see #floorMod(long, long)
+     * @since 3.4
+     */
+    public static long floorDiv(final long a, final long b) throws MathArithmeticException {
+
+        if (b == 0l) {
+            throw new MathArithmeticException(LocalizedFormats.ZERO_DENOMINATOR);
+        }
+
+        final long m = a % b;
+        if ((a ^ b) >= 0l || m == 0l) {
+            // a an b have same sign, or division is exact
+            return a / b;
+        } else {
+            // a and b have opposite signs and division is not exact
+            return (a / b) - 1l;
+        }
+
+    }
+
+    /** Finds r such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0.
+     * 
    
+     * This methods returns the same value as integer modulo when
+     * a and b are same signs, but returns a different value when
+     * they are opposite (i.e. q is negative).
+     * 
    
+     * @param a dividend
+     * @param b divisor
+     * @return r such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0
+     * @exception MathArithmeticException if b == 0
+     * @see #floorDiv(int, int)
+     * @since 3.4
+     */
+    public static int floorMod(final int a, final int b) throws MathArithmeticException {
+
+        if (b == 0) {
+            throw new MathArithmeticException(LocalizedFormats.ZERO_DENOMINATOR);
+        }
+
+        final int m = a % b;
+        if ((a ^ b) >= 0 || m == 0) {
+            // a an b have same sign, or division is exact
+            return m;
+        } else {
+            // a and b have opposite signs and division is not exact
+            return b + m;
+        }
+
+    }
+
+    /** Finds r such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0.
+     * 
    
+     * This methods returns the same value as integer modulo when
+     * a and b are same signs, but returns a different value when
+     * they are opposite (i.e. q is negative).
+     * 
    
+     * @param a dividend
+     * @param b divisor
+     * @return r such that a = q b + r with 0 <= r < b if b > 0 and b < r <= 0 if b < 0
+     * @exception MathArithmeticException if b == 0
+     * @see #floorDiv(long, long)
+     * @since 3.4
+     */
+    public static long floorMod(final long a, final long b) {
+
+        if (b == 0l) {
+            throw new MathArithmeticException(LocalizedFormats.ZERO_DENOMINATOR);
+        }
+
+        final long m = a % b;
+        if ((a ^ b) >= 0l || m == 0l) {
+            // a an b have same sign, or division is exact
+            return m;
+        } else {
+            // a and b have opposite signs and division is not exact
+            return b + m;
+        }
+
+    }
+
     /**
      * Returns the first argument with the sign of the second argument.
      * A NaN {@code sign} argument is treated as positive.
diff --git a/java/source/infodynamics/utils/commonsmath3/util/FastMathCalc.java b/java/source/infodynamics/utils/commonsmath3/util/FastMathCalc.java
old mode 100755
new mode 100644
index 52c7a3c..6128a77
--- a/java/source/infodynamics/utils/commonsmath3/util/FastMathCalc.java
+++ b/java/source/infodynamics/utils/commonsmath3/util/FastMathCalc.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -50,7 +50,6 @@ import java.io.PrintStream;
 import infodynamics.utils.commonsmath3.exception.DimensionMismatchException;
 
 /** Class used to compute the classical functions tables.
- * @version $Id$
  * @since 3.0
  */
 class FastMathCalc {
diff --git a/java/source/infodynamics/utils/commonsmath3/util/FastMathLiteralArrays.java b/java/source/infodynamics/utils/commonsmath3/util/FastMathLiteralArrays.java
old mode 100755
new mode 100644
index 6b04f95..dde4d40
--- a/java/source/infodynamics/utils/commonsmath3/util/FastMathLiteralArrays.java
+++ b/java/source/infodynamics/utils/commonsmath3/util/FastMathLiteralArrays.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -49,7 +49,6 @@ package infodynamics.utils.commonsmath3.util;
 /**
  * Utility class for loading tabulated data used by {@link FastMath}.
  *
- * @version $Id$
  */
 class FastMathLiteralArrays {
     /** Exponential evaluated at integer values,
diff --git a/java/source/infodynamics/utils/commonsmath3/util/IntegerSequence.java b/java/source/infodynamics/utils/commonsmath3/util/IntegerSequence.java
new file mode 100644
index 0000000..4e06e39
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/IntegerSequence.java
@@ -0,0 +1,391 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+import java.util.Iterator;
+import infodynamics.utils.commonsmath3.exception.MaxCountExceededException;
+import infodynamics.utils.commonsmath3.exception.NullArgumentException;
+import infodynamics.utils.commonsmath3.exception.MathUnsupportedOperationException;
+import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException;
+import infodynamics.utils.commonsmath3.exception.ZeroException;
+
+/**
+ * Provides a sequence of integers.
+ *
+ * @since 3.6
+ */
+public class IntegerSequence {
+    /**
+     * Utility class contains only static methods.
+     */
+    private IntegerSequence() {}
+
+    /**
+     * Creates a sequence {@code [start .. end]}.
+     * It calls {@link #range(int,int,int) range(start, end, 1)}.
+     *
+     * @param start First value of the range.
+     * @param end Last value of the range.
+     * @return a range.
+     */
+    public static Range range(int start,
+                              int end) {
+        return range(start, end, 1);
+    }
+
+    /**
+     * Creates a sequence \( a_i, i < 0 <= n \)
+     * where \( a_i = start + i * step \)
+     * and \( n \) is such that \( a_n <= max \) and \( a_{n+1} > max \).
+     *
+     * @param start First value of the range.
+     * @param max Last value of the range that satisfies the above
+     * construction rule.
+     * @param step Increment.
+     * @return a range.
+     */
+    public static Range range(final int start,
+                              final int max,
+                              final int step) {
+        return new Range(start, max, step);
+    }
+
+    /**
+     * Generates a sequence of integers.
+     */
+    public static class Range implements Iterable {
+        /** Number of integers contained in this range. */
+        private final int size;
+        /** First value. */
+        private final int start;
+        /** Final value. */
+        private final int max;
+        /** Increment. */
+        private final int step;
+
+        /**
+         * Creates a sequence \( a_i, i < 0 <= n \)
+         * where \( a_i = start + i * step \)
+         * and \( n \) is such that \( a_n <= max \) and \( a_{n+1} > max \).
+         *
+         * @param start First value of the range.
+         * @param max Last value of the range that satisfies the above
+         * construction rule.
+         * @param step Increment.
+         */
+        public Range(int start,
+                     int max,
+                     int step) {
+            this.start = start;
+            this.max = max;
+            this.step = step;
+
+            final int s = (max - start) / step + 1;
+            this.size = s < 0 ? 0 : s;
+        }
+
+        /**
+         * Gets the number of elements contained in the range.
+         *
+         * @return the size of the range.
+         */
+        public int size() {
+            return size;
+        }
+
+        /** {@inheritDoc} */
+        public Iterator iterator() {
+            return Incrementor.create()
+                .withStart(start)
+                .withMaximalCount(max + (step > 0 ? 1 : -1))
+                .withIncrement(step);
+        }
+    }
+
+    /**
+     * Utility that increments a counter until a maximum is reached, at
+     * which point, the instance will by default throw a
+     * {@link MaxCountExceededException}.
+     * However, the user is able to override this behaviour by defining a
+     * custom {@link MaxCountExceededCallback callback}, in order to e.g.
+     * select which exception must be thrown.
+     */
+    public static class Incrementor implements Iterator {
+        /** Default callback. */
+        private static final MaxCountExceededCallback CALLBACK
+            = new MaxCountExceededCallback() {
+                    /** {@inheritDoc} */
+                    public void trigger(int max) throws MaxCountExceededException {
+                        throw new MaxCountExceededException(max);
+                    }
+                };
+
+        /** Initial value the counter. */
+        private final int init;
+        /** Upper limit for the counter. */
+        private final int maximalCount;
+        /** Increment. */
+        private final int increment;
+        /** Function called at counter exhaustion. */
+        private final MaxCountExceededCallback maxCountCallback;
+        /** Current count. */
+        private int count = 0;
+
+        /**
+         * Defines a method to be called at counter exhaustion.
+         * The {@link #trigger(int) trigger} method should usually throw an exception.
+         */
+        public interface MaxCountExceededCallback {
+            /**
+             * Function called when the maximal count has been reached.
+             *
+             * @param maximalCount Maximal count.
+             * @throws MaxCountExceededException at counter exhaustion
+             */
+            void trigger(int maximalCount) throws MaxCountExceededException;
+        }
+
+        /**
+         * Creates an incrementor.
+         * The counter will be exhausted either when {@code max} is reached
+         * or when {@code nTimes} increments have been performed.
+         *
+         * @param start Initial value.
+         * @param max Maximal count.
+         * @param step Increment.
+         * @param cb Function to be called when the maximal count has been reached.
+         * @throws NullArgumentException if {@code cb} is {@code null}.
+         */
+        private Incrementor(int start,
+                            int max,
+                            int step,
+                            MaxCountExceededCallback cb)
+            throws NullArgumentException {
+            if (cb == null) {
+                throw new NullArgumentException();
+            }
+            this.init = start;
+            this.maximalCount = max;
+            this.increment = step;
+            this.maxCountCallback = cb;
+            this.count = start;
+        }
+
+        /**
+         * Factory method that creates a default instance.
+         * The initial and maximal values are set to 0.
+         * For the new instance to be useful, the maximal count must be set
+         * by calling {@link #withMaximalCount(int) withMaximalCount}.
+         *
+         * @return an new instance.
+         */
+        public static Incrementor create() {
+            return new Incrementor(0, 0, 1, CALLBACK);
+        }
+
+        /**
+         * Creates a new instance with a given initial value.
+         * The counter is reset to the initial value.
+         *
+         * @param start Initial value of the counter.
+         * @return a new instance.
+         */
+        public Incrementor withStart(int start) {
+            return new Incrementor(start,
+                                   this.maximalCount,
+                                   this.increment,
+                                   this.maxCountCallback);
+        }
+
+        /**
+         * Creates a new instance with a given maximal count.
+         * The counter is reset to the initial value.
+         *
+         * @param max Maximal count.
+         * @return a new instance.
+         */
+        public Incrementor withMaximalCount(int max) {
+            return new Incrementor(this.init,
+                                   max,
+                                   this.increment,
+                                   this.maxCountCallback);
+        }
+
+        /**
+         * Creates a new instance with a given increment.
+         * The counter is reset to the initial value.
+         *
+         * @param step Increment.
+         * @return a new instance.
+         */
+        public Incrementor withIncrement(int step) {
+            if (step == 0) {
+                throw new ZeroException();
+            }
+            return new Incrementor(this.init,
+                                   this.maximalCount,
+                                   step,
+                                   this.maxCountCallback);
+        }
+
+        /**
+         * Creates a new instance with a given callback.
+         * The counter is reset to the initial value.
+         *
+         * @param cb Callback to be called at counter exhaustion.
+         * @return a new instance.
+         */
+        public Incrementor withCallback(MaxCountExceededCallback cb) {
+            return new Incrementor(this.init,
+                                   this.maximalCount,
+                                   this.increment,
+                                   cb);
+        }
+
+        /**
+         * Gets the upper limit of the counter.
+         *
+         * @return the counter upper limit.
+         */
+        public int getMaximalCount() {
+            return maximalCount;
+        }
+
+        /**
+         * Gets the current count.
+         *
+         * @return the current count.
+         */
+        public int getCount() {
+            return count;
+        }
+
+        /**
+         * Checks whether incrementing the counter {@code nTimes} is allowed.
+         *
+         * @return {@code false} if calling {@link #increment()}
+         * will trigger a {@code MaxCountExceededException},
+         * {@code true} otherwise.
+         */
+        public boolean canIncrement() {
+            return canIncrement(1);
+        }
+
+        /**
+         * Checks whether incrementing the counter several times is allowed.
+         *
+         * @param nTimes Number of increments.
+         * @return {@code false} if calling {@link #increment(int)
+         * increment(nTimes)} would call the {@link MaxCountExceededCallback callback}
+         * {@code true} otherwise.
+         */
+        public boolean canIncrement(int nTimes) {
+            final int finalCount = count + nTimes * increment;
+            return increment < 0 ?
+                finalCount > maximalCount :
+                finalCount < maximalCount;
+        }
+
+        /**
+         * Performs multiple increments.
+         *
+         * @param nTimes Number of increments.
+         * @throws MaxCountExceededException at counter exhaustion.
+         * @throws NotStrictlyPositiveException if {@code nTimes <= 0}.
+         *
+         * @see #increment()
+         */
+        public void increment(int nTimes) throws MaxCountExceededException {
+            if (nTimes <= 0) {
+                throw new NotStrictlyPositiveException(nTimes);
+            }
+
+            if (!canIncrement(0)) {
+                maxCountCallback.trigger(maximalCount);
+            }
+            count += nTimes * increment;
+        }
+
+        /**
+         * Adds the increment value to the current iteration count.
+         * At counter exhaustion, this method will call the
+         * {@link MaxCountExceededCallback#trigger(int) trigger} method of the
+         * callback object passed to the
+         * {@link #withCallback(MaxCountExceededCallback)} method.
+         * If not explicitly set, a default callback is used that will throw
+         * a {@code MaxCountExceededException}.
+         *
+         * @throws MaxCountExceededException at counter exhaustion, unless a
+         * custom {@link MaxCountExceededCallback callback} has been set.
+         *
+         * @see #increment(int)
+         */
+        public void increment() throws MaxCountExceededException {
+            increment(1);
+        }
+
+        /** {@inheritDoc} */
+        public boolean hasNext() {
+            return canIncrement(0);
+        }
+
+        /** {@inheritDoc} */
+        public Integer next() {
+            final int value = count;
+            increment();
+            return value;
+        }
+
+        /**
+         * Not applicable.
+         *
+         * @throws MathUnsupportedOperationException
+         */
+        public void remove() {
+            throw new MathUnsupportedOperationException();
+        }
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/MathArrays.java b/java/source/infodynamics/utils/commonsmath3/util/MathArrays.java
new file mode 100644
index 0000000..432c0ac
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/MathArrays.java
@@ -0,0 +1,1964 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package infodynamics.utils.commonsmath3.util;
+
+import java.lang.reflect.Array;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeSet;
+
+import infodynamics.utils.commonsmath3.Field;
+import infodynamics.utils.commonsmath3.random.RandomGenerator;
+import infodynamics.utils.commonsmath3.random.Well19937c;
+import infodynamics.utils.commonsmath3.distribution.UniformIntegerDistribution;
+import infodynamics.utils.commonsmath3.exception.DimensionMismatchException;
+import infodynamics.utils.commonsmath3.exception.MathArithmeticException;
+import infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException;
+import infodynamics.utils.commonsmath3.exception.MathInternalError;
+import infodynamics.utils.commonsmath3.exception.NoDataException;
+import infodynamics.utils.commonsmath3.exception.NonMonotonicSequenceException;
+import infodynamics.utils.commonsmath3.exception.NotPositiveException;
+import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException;
+import infodynamics.utils.commonsmath3.exception.NullArgumentException;
+import infodynamics.utils.commonsmath3.exception.NumberIsTooLargeException;
+import infodynamics.utils.commonsmath3.exception.NotANumberException;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
+/**
+ * Arrays utilities.
+ *
+ * @since 3.0
+ */
+public class MathArrays {
+
+    /**
+     * Private constructor.
+     */
+    private MathArrays() {}
+
+    /**
+     * Real-valued function that operate on an array or a part of it.
+     * @since 3.1
+     */
+    public interface Function {
+        /**
+         * Operates on an entire array.
+         *
+         * @param array Array to operate on.
+         * @return the result of the operation.
+         */
+        double evaluate(double[] array);
+        /**
+         * @param array Array to operate on.
+         * @param startIndex Index of the first element to take into account.
+         * @param numElements Number of elements to take into account.
+         * @return the result of the operation.
+         */
+        double evaluate(double[] array,
+                        int startIndex,
+                        int numElements);
+    }
+
+    /**
+     * Create a copy of an array scaled by a value.
+     *
+     * @param arr Array to scale.
+     * @param val Scalar.
+     * @return scaled copy of array with each entry multiplied by val.
+     * @since 3.2
+     */
+    public static double[] scale(double val, final double[] arr) {
+        double[] newArr = new double[arr.length];
+        for (int i = 0; i < arr.length; i++) {
+            newArr[i] = arr[i] * val;
+        }
+        return newArr;
+    }
+
+    /**
+     * 
    Multiply each element of an array by a value.
    
+     *
+     * 
    The array is modified in place (no copy is created).
    
+     *
+     * @param arr Array to scale
+     * @param val Scalar
+     * @since 3.2
+     */
+    public static void scaleInPlace(double val, final double[] arr) {
+        for (int i = 0; i < arr.length; i++) {
+            arr[i] *= val;
+        }
+    }
+
+    /**
+     * Creates an array whose contents will be the element-by-element
+     * addition of the arguments.
+     *
+     * @param a First term of the addition.
+     * @param b Second term of the addition.
+     * @return a new array {@code r} where {@code r[i] = a[i] + b[i]}.
+     * @throws DimensionMismatchException if the array lengths differ.
+     * @since 3.1
+     */
+    public static double[] ebeAdd(double[] a, double[] b)
+        throws DimensionMismatchException {
+        checkEqualLength(a, b);
+
+        final double[] result = a.clone();
+        for (int i = 0; i < a.length; i++) {
+            result[i] += b[i];
+        }
+        return result;
+    }
+    /**
+     * Creates an array whose contents will be the element-by-element
+     * subtraction of the second argument from the first.
+     *
+     * @param a First term.
+     * @param b Element to be subtracted.
+     * @return a new array {@code r} where {@code r[i] = a[i] - b[i]}.
+     * @throws DimensionMismatchException if the array lengths differ.
+     * @since 3.1
+     */
+    public static double[] ebeSubtract(double[] a, double[] b)
+        throws DimensionMismatchException {
+        checkEqualLength(a, b);
+
+        final double[] result = a.clone();
+        for (int i = 0; i < a.length; i++) {
+            result[i] -= b[i];
+        }
+        return result;
+    }
+    /**
+     * Creates an array whose contents will be the element-by-element
+     * multiplication of the arguments.
+     *
+     * @param a First factor of the multiplication.
+     * @param b Second factor of the multiplication.
+     * @return a new array {@code r} where {@code r[i] = a[i] * b[i]}.
+     * @throws DimensionMismatchException if the array lengths differ.
+     * @since 3.1
+     */
+    public static double[] ebeMultiply(double[] a, double[] b)
+        throws DimensionMismatchException {
+        checkEqualLength(a, b);
+
+        final double[] result = a.clone();
+        for (int i = 0; i < a.length; i++) {
+            result[i] *= b[i];
+        }
+        return result;
+    }
+    /**
+     * Creates an array whose contents will be the element-by-element
+     * division of the first argument by the second.
+     *
+     * @param a Numerator of the division.
+     * @param b Denominator of the division.
+     * @return a new array {@code r} where {@code r[i] = a[i] / b[i]}.
+     * @throws DimensionMismatchException if the array lengths differ.
+     * @since 3.1
+     */
+    public static double[] ebeDivide(double[] a, double[] b)
+        throws DimensionMismatchException {
+        checkEqualLength(a, b);
+
+        final double[] result = a.clone();
+        for (int i = 0; i < a.length; i++) {
+            result[i] /= b[i];
+        }
+        return result;
+    }
+
+    /**
+     * Calculates the L1 (sum of abs) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L1 distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static double distance1(double[] p1, double[] p2)
+    throws DimensionMismatchException {
+        checkEqualLength(p1, p2);
+        double sum = 0;
+        for (int i = 0; i < p1.length; i++) {
+            sum += FastMath.abs(p1[i] - p2[i]);
+        }
+        return sum;
+    }
+
+    /**
+     * Calculates the L1 (sum of abs) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L1 distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static int distance1(int[] p1, int[] p2)
+    throws DimensionMismatchException {
+        checkEqualLength(p1, p2);
+        int sum = 0;
+        for (int i = 0; i < p1.length; i++) {
+            sum += FastMath.abs(p1[i] - p2[i]);
+        }
+        return sum;
+    }
+
+    /**
+     * Calculates the L2 (Euclidean) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L2 distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static double distance(double[] p1, double[] p2)
+    throws DimensionMismatchException {
+        checkEqualLength(p1, p2);
+        double sum = 0;
+        for (int i = 0; i < p1.length; i++) {
+            final double dp = p1[i] - p2[i];
+            sum += dp * dp;
+        }
+        return FastMath.sqrt(sum);
+    }
+
+    /**
+     * Calculates the cosine of the angle between two vectors.
+     *
+     * @param v1 Cartesian coordinates of the first vector.
+     * @param v2 Cartesian coordinates of the second vector.
+     * @return the cosine of the angle between the vectors.
+     * @since 3.6
+     */
+    public static double cosAngle(double[] v1, double[] v2) {
+        return linearCombination(v1, v2) / (safeNorm(v1) * safeNorm(v2));
+    }
+
+    /**
+     * Calculates the L2 (Euclidean) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L2 distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static double distance(int[] p1, int[] p2)
+    throws DimensionMismatchException {
+      checkEqualLength(p1, p2);
+      double sum = 0;
+      for (int i = 0; i < p1.length; i++) {
+          final double dp = p1[i] - p2[i];
+          sum += dp * dp;
+      }
+      return FastMath.sqrt(sum);
+    }
+
+    /**
+     * Calculates the L∞ (max of abs) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L∞ distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static double distanceInf(double[] p1, double[] p2)
+    throws DimensionMismatchException {
+        checkEqualLength(p1, p2);
+        double max = 0;
+        for (int i = 0; i < p1.length; i++) {
+            max = FastMath.max(max, FastMath.abs(p1[i] - p2[i]));
+        }
+        return max;
+    }
+
+    /**
+     * Calculates the L∞ (max of abs) distance between two points.
+     *
+     * @param p1 the first point
+     * @param p2 the second point
+     * @return the L∞ distance between the two points
+     * @throws DimensionMismatchException if the array lengths differ.
+     */
+    public static int distanceInf(int[] p1, int[] p2)
+    throws DimensionMismatchException {
+        checkEqualLength(p1, p2);
+        int max = 0;
+        for (int i = 0; i < p1.length; i++) {
+            max = FastMath.max(max, FastMath.abs(p1[i] - p2[i]));
+        }
+        return max;
+    }
+
+    /**
+     * Specification of ordering direction.
+     */
+    public enum OrderDirection {
+        /** Constant for increasing direction. */
+        INCREASING,
+        /** Constant for decreasing direction. */
+        DECREASING
+    }
+
+    /**
+     * Check that an array is monotonically increasing or decreasing.
+     *
+     * @param  the type of the elements in the specified array
+     * @param val Values.
+     * @param dir Ordering direction.
+     * @param strict Whether the order should be strict.
+     * @return {@code true} if sorted, {@code false} otherwise.
+     */
+    public static  > boolean isMonotonic(T[] val,
+                                      OrderDirection dir,
+                                      boolean strict) {
+        T previous = val[0];
+        final int max = val.length;
+        for (int i = 1; i < max; i++) {
+            final int comp;
+            switch (dir) {
+            case INCREASING:
+                comp = previous.compareTo(val[i]);
+                if (strict) {
+                    if (comp >= 0) {
+                        return false;
+                    }
+                } else {
+                    if (comp > 0) {
+                        return false;
+                    }
+                }
+                break;
+            case DECREASING:
+                comp = val[i].compareTo(previous);
+                if (strict) {
+                    if (comp >= 0) {
+                        return false;
+                    }
+                } else {
+                    if (comp > 0) {
+                       return false;
+                    }
+                }
+                break;
+            default:
+                // Should never happen.
+                throw new MathInternalError();
+            }
+
+            previous = val[i];
+        }
+        return true;
+    }
+
+    /**
+     * Check that an array is monotonically increasing or decreasing.
+     *
+     * @param val Values.
+     * @param dir Ordering direction.
+     * @param strict Whether the order should be strict.
+     * @return {@code true} if sorted, {@code false} otherwise.
+     */
+    public static boolean isMonotonic(double[] val, OrderDirection dir, boolean strict) {
+        return checkOrder(val, dir, strict, false);
+    }
+
+    /**
+     * Check that both arrays have the same length.
+     *
+     * @param a Array.
+     * @param b Array.
+     * @param abort Whether to throw an exception if the check fails.
+     * @return {@code true} if the arrays have the same length.
+     * @throws DimensionMismatchException if the lengths differ and
+     * {@code abort} is {@code true}.
+     * @since 3.6
+     */
+    public static boolean checkEqualLength(double[] a,
+                                           double[] b,
+                                           boolean abort) {
+        if (a.length == b.length) {
+            return true;
+        } else {
+            if (abort) {
+                throw new DimensionMismatchException(a.length, b.length);
+            }
+            return false;
+        }
+    }
+
+    /**
+     * Check that both arrays have the same length.
+     *
+     * @param a Array.
+     * @param b Array.
+     * @throws DimensionMismatchException if the lengths differ.
+     * @since 3.6
+     */
+    public static void checkEqualLength(double[] a,
+                                        double[] b) {
+        checkEqualLength(a, b, true);
+    }
+
+
+    /**
+     * Check that both arrays have the same length.
+     *
+     * @param a Array.
+     * @param b Array.
+     * @param abort Whether to throw an exception if the check fails.
+     * @return {@code true} if the arrays have the same length.
+     * @throws DimensionMismatchException if the lengths differ and
+     * {@code abort} is {@code true}.
+     * @since 3.6
+     */
+    public static boolean checkEqualLength(int[] a,
+                                           int[] b,
+                                           boolean abort) {
+        if (a.length == b.length) {
+            return true;
+        } else {
+            if (abort) {
+                throw new DimensionMismatchException(a.length, b.length);
+            }
+            return false;
+        }
+    }
+
+    /**
+     * Check that both arrays have the same length.
+     *
+     * @param a Array.
+     * @param b Array.
+     * @throws DimensionMismatchException if the lengths differ.
+     * @since 3.6
+     */
+    public static void checkEqualLength(int[] a,
+                                        int[] b) {
+        checkEqualLength(a, b, true);
+    }
+
+    /**
+     * Check that the given array is sorted.
+     *
+     * @param val Values.
+     * @param dir Ordering direction.
+     * @param strict Whether the order should be strict.
+     * @param abort Whether to throw an exception if the check fails.
+     * @return {@code true} if the array is sorted.
+     * @throws NonMonotonicSequenceException if the array is not sorted
+     * and {@code abort} is {@code true}.
+     */
+    public static boolean checkOrder(double[] val, OrderDirection dir,
+                                     boolean strict, boolean abort)
+        throws NonMonotonicSequenceException {
+        double previous = val[0];
+        final int max = val.length;
+
+        int index;
+        ITEM:
+        for (index = 1; index < max; index++) {
+            switch (dir) {
+            case INCREASING:
+                if (strict) {
+                    if (val[index] <= previous) {
+                        break ITEM;
+                    }
+                } else {
+                    if (val[index] < previous) {
+                        break ITEM;
+                    }
+                }
+                break;
+            case DECREASING:
+                if (strict) {
+                    if (val[index] >= previous) {
+                        break ITEM;
+                    }
+                } else {
+                    if (val[index] > previous) {
+                        break ITEM;
+                    }
+                }
+                break;
+            default:
+                // Should never happen.
+                throw new MathInternalError();
+            }
+
+            previous = val[index];
+        }
+
+        if (index == max) {
+            // Loop completed.
+            return true;
+        }
+
+        // Loop early exit means wrong ordering.
+        if (abort) {
+            throw new NonMonotonicSequenceException(val[index], previous, index, dir, strict);
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * Check that the given array is sorted.
+     *
+     * @param val Values.
+     * @param dir Ordering direction.
+     * @param strict Whether the order should be strict.
+     * @throws NonMonotonicSequenceException if the array is not sorted.
+     * @since 2.2
+     */
+    public static void checkOrder(double[] val, OrderDirection dir,
+                                  boolean strict) throws NonMonotonicSequenceException {
+        checkOrder(val, dir, strict, true);
+    }
+
+    /**
+     * Check that the given array is sorted in strictly increasing order.
+     *
+     * @param val Values.
+     * @throws NonMonotonicSequenceException if the array is not sorted.
+     * @since 2.2
+     */
+    public static void checkOrder(double[] val) throws NonMonotonicSequenceException {
+        checkOrder(val, OrderDirection.INCREASING, true);
+    }
+
+    /**
+     * Throws DimensionMismatchException if the input array is not rectangular.
+     *
+     * @param in array to be tested
+     * @throws NullArgumentException if input array is null
+     * @throws DimensionMismatchException if input array is not rectangular
+     * @since 3.1
+     */
+    public static void checkRectangular(final long[][] in)
+        throws NullArgumentException, DimensionMismatchException {
+        MathUtils.checkNotNull(in);
+        for (int i = 1; i < in.length; i++) {
+            if (in[i].length != in[0].length) {
+                throw new DimensionMismatchException(
+                        LocalizedFormats.DIFFERENT_ROWS_LENGTHS,
+                        in[i].length, in[0].length);
+            }
+        }
+    }
+
+    /**
+     * Check that all entries of the input array are strictly positive.
+     *
+     * @param in Array to be tested
+     * @throws NotStrictlyPositiveException if any entries of the array are not
+     * strictly positive.
+     * @since 3.1
+     */
+    public static void checkPositive(final double[] in)
+        throws NotStrictlyPositiveException {
+        for (int i = 0; i < in.length; i++) {
+            if (in[i] <= 0) {
+                throw new NotStrictlyPositiveException(in[i]);
+            }
+        }
+    }
+
+    /**
+     * Check that no entry of the input array is {@code NaN}.
+     *
+     * @param in Array to be tested.
+     * @throws NotANumberException if an entry is {@code NaN}.
+     * @since 3.4
+     */
+    public static void checkNotNaN(final double[] in)
+        throws NotANumberException {
+        for(int i = 0; i < in.length; i++) {
+            if (Double.isNaN(in[i])) {
+                throw new NotANumberException();
+            }
+        }
+    }
+
+    /**
+     * Check that all entries of the input array are >= 0.
+     *
+     * @param in Array to be tested
+     * @throws NotPositiveException if any array entries are less than 0.
+     * @since 3.1
+     */
+    public static void checkNonNegative(final long[] in)
+        throws NotPositiveException {
+        for (int i = 0; i < in.length; i++) {
+            if (in[i] < 0) {
+                throw new NotPositiveException(in[i]);
+            }
+        }
+    }
+
+    /**
+     * Check all entries of the input array are >= 0.
+     *
+     * @param in Array to be tested
+     * @throws NotPositiveException if any array entries are less than 0.
+     * @since 3.1
+     */
+    public static void checkNonNegative(final long[][] in)
+        throws NotPositiveException {
+        for (int i = 0; i < in.length; i ++) {
+            for (int j = 0; j < in[i].length; j++) {
+                if (in[i][j] < 0) {
+                    throw new NotPositiveException(in[i][j]);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns the Cartesian norm (2-norm), handling both overflow and underflow.
+     * Translation of the minpack enorm subroutine.
+     *
+     * The redistribution policy for MINPACK is available
+     * here, for
+     * convenience, it is reproduced below.
    
+     *
+     * 
+     * 
+     * 
+     * 
    
+     *    Minpack Copyright Notice (1999) University of Chicago.
+     *    All rights reserved
+     * 
    
+     * Redistribution and use in source and binary forms, with or without
+     * modification, are permitted provided that the following conditions
+     * are met:
+     * 
      
+     *  
    1. Redistributions of source code must retain the above copyright
+     *      notice, this list of conditions and the following disclaimer.
      2. 
+     * 
    2. Redistributions in binary form must reproduce the above
+     *     copyright notice, this list of conditions and the following
+     *     disclaimer in the documentation and/or other materials provided
+     *     with the distribution.
      3. 
+     * 
    3. The end-user documentation included with the redistribution, if any,
+     *     must include the following acknowledgment:
+     *     {@code This product includes software developed by the University of
+     *           Chicago, as Operator of Argonne National Laboratory.}
+     *     Alternately, this acknowledgment may appear in the software itself,
+     *     if and wherever such third-party acknowledgments normally appear.
      4. 
+     * 
    4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
+     *     WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
+     *     UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
+     *     THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
+     *     IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
+     *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
+     *     OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
+     *     OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
+     *     USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
+     *     THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
+     *     DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
+     *     UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
+     *     BE CORRECTED.
      5. 
+     * 
    5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
+     *     HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
+     *     ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
+     *     INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
+     *     ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
+     *     PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
+     *     SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
+     *     (INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
+     *     EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
+     *     POSSIBILITY OF SUCH LOSS OR DAMAGES.
      6. 
+     * 
      
+     *
+     * @param v Vector of doubles.
+     * @return the 2-norm of the vector.
+     * @since 2.2
+     */
+    public static double safeNorm(double[] v) {
+        double rdwarf = 3.834e-20;
+        double rgiant = 1.304e+19;
+        double s1 = 0;
+        double s2 = 0;
+        double s3 = 0;
+        double x1max = 0;
+        double x3max = 0;
+        double floatn = v.length;
+        double agiant = rgiant / floatn;
+        for (int i = 0; i < v.length; i++) {
+            double xabs = FastMath.abs(v[i]);
+            if (xabs < rdwarf || xabs > agiant) {
+                if (xabs > rdwarf) {
+                    if (xabs > x1max) {
+                        double r = x1max / xabs;
+                        s1= 1 + s1 * r * r;
+                        x1max = xabs;
+                    } else {
+                        double r = xabs / x1max;
+                        s1 += r * r;
+                    }
+                } else {
+                    if (xabs > x3max) {
+                        double r = x3max / xabs;
+                        s3= 1 + s3 * r * r;
+                        x3max = xabs;
+                    } else {
+                        if (xabs != 0) {
+                            double r = xabs / x3max;
+                            s3 += r * r;
+                        }
+                    }
+                }
+            } else {
+                s2 += xabs * xabs;
+            }
+        }
+        double norm;
+        if (s1 != 0) {
+            norm = x1max * Math.sqrt(s1 + (s2 / x1max) / x1max);
+        } else {
+            if (s2 == 0) {
+                norm = x3max * Math.sqrt(s3);
+            } else {
+                if (s2 >= x3max) {
+                    norm = Math.sqrt(s2 * (1 + (x3max / s2) * (x3max * s3)));
+                } else {
+                    norm = Math.sqrt(x3max * ((s2 / x3max) + (x3max * s3)));
+                }
+            }
+        }
+        return norm;
+    }
+
+    /**
+     * A helper data structure holding a double and an integer value.
+     */
+    private static class PairDoubleInteger {
+        /** Key */
+        private final double key;
+        /** Value */
+        private final int value;
+
+        /**
+         * @param key Key.
+         * @param value Value.
+         */
+        PairDoubleInteger(double key, int value) {
+            this.key = key;
+            this.value = value;
+        }
+
+        /** @return the key. */
+        public double getKey() {
+            return key;
+        }
+
+        /** @return the value. */
+        public int getValue() {
+            return value;
+        }
+    }
+
+    /**
+     * Sort an array in ascending order in place and perform the same reordering
+     * of entries on other arrays. For example, if
+     * {@code x = [3, 1, 2], y = [1, 2, 3]} and {@code z = [0, 5, 7]}, then
+     * {@code sortInPlace(x, y, z)} will update {@code x} to {@code [1, 2, 3]},
+     * {@code y} to {@code [2, 3, 1]} and {@code z} to {@code [5, 7, 0]}.
+     *
+     * @param x Array to be sorted and used as a pattern for permutation
+     * of the other arrays.
+     * @param yList Set of arrays whose permutations of entries will follow
+     * those performed on {@code x}.
+     * @throws DimensionMismatchException if any {@code y} is not the same
+     * size as {@code x}.
+     * @throws NullArgumentException if {@code x} or any {@code y} is null.
+     * @since 3.0
+     */
+    public static void sortInPlace(double[] x, double[] ... yList)
+        throws DimensionMismatchException, NullArgumentException {
+        sortInPlace(x, OrderDirection.INCREASING, yList);
+    }
+
+    /**
+     * Sort an array in place and perform the same reordering of entries on
+     * other arrays.  This method works the same as the other
+     * {@link #sortInPlace(double[], double[][]) sortInPlace} method, but
+     * allows the order of the sort to be provided in the {@code dir}
+     * parameter.
+     *
+     * @param x Array to be sorted and used as a pattern for permutation
+     * of the other arrays.
+     * @param dir Order direction.
+     * @param yList Set of arrays whose permutations of entries will follow
+     * those performed on {@code x}.
+     * @throws DimensionMismatchException if any {@code y} is not the same
+     * size as {@code x}.
+     * @throws NullArgumentException if {@code x} or any {@code y} is null
+     * @since 3.0
+     */
+    public static void sortInPlace(double[] x,
+                                   final OrderDirection dir,
+                                   double[] ... yList)
+        throws NullArgumentException,
+               DimensionMismatchException {
+
+        // Consistency checks.
+        if (x == null) {
+            throw new NullArgumentException();
+        }
+
+        final int yListLen = yList.length;
+        final int len = x.length;
+
+        for (int j = 0; j < yListLen; j++) {
+            final double[] y = yList[j];
+            if (y == null) {
+                throw new NullArgumentException();
+            }
+            if (y.length != len) {
+                throw new DimensionMismatchException(y.length, len);
+            }
+        }
+
+        // Associate each abscissa "x[i]" with its index "i".
+        final List list
+            = new ArrayList(len);
+        for (int i = 0; i < len; i++) {
+            list.add(new PairDoubleInteger(x[i], i));
+        }
+
+        // Create comparators for increasing and decreasing orders.
+        final Comparator comp
+            = dir == MathArrays.OrderDirection.INCREASING ?
+            new Comparator() {
+            /** {@inheritDoc} */
+            public int compare(PairDoubleInteger o1,
+                               PairDoubleInteger o2) {
+                return Double.compare(o1.getKey(), o2.getKey());
+            }
+        } : new Comparator() {
+            /** {@inheritDoc} */
+            public int compare(PairDoubleInteger o1,
+                               PairDoubleInteger o2) {
+                return Double.compare(o2.getKey(), o1.getKey());
+            }
+        };
+
+        // Sort.
+        Collections.sort(list, comp);
+
+        // Modify the original array so that its elements are in
+        // the prescribed order.
+        // Retrieve indices of original locations.
+        final int[] indices = new int[len];
+        for (int i = 0; i < len; i++) {
+            final PairDoubleInteger e = list.get(i);
+            x[i] = e.getKey();
+            indices[i] = e.getValue();
+        }
+
+        // In each of the associated arrays, move the
+        // elements to their new location.
+        for (int j = 0; j < yListLen; j++) {
+            // Input array will be modified in place.
+            final double[] yInPlace = yList[j];
+            final double[] yOrig = yInPlace.clone();
+
+            for (int i = 0; i < len; i++) {
+                yInPlace[i] = yOrig[indices[i]];
+            }
+        }
+    }
+
+    /**
+     * Creates a copy of the {@code source} array.
+     *
+     * @param source Array to be copied.
+     * @return the copied array.
+     */
+     public static int[] copyOf(int[] source) {
+         return copyOf(source, source.length);
+     }
+
+    /**
+     * Creates a copy of the {@code source} array.
+     *
+     * @param source Array to be copied.
+     * @return the copied array.
+     */
+     public static double[] copyOf(double[] source) {
+         return copyOf(source, source.length);
+     }
+
+    /**
+     * Creates a copy of the {@code source} array.
+     *
+     * @param source Array to be copied.
+     * @param len Number of entries to copy. If smaller then the source
+     * length, the copy will be truncated, if larger it will padded with
+     * zeroes.
+     * @return the copied array.
+     */
+    public static int[] copyOf(int[] source, int len) {
+         final int[] output = new int[len];
+         System.arraycopy(source, 0, output, 0, FastMath.min(len, source.length));
+         return output;
+     }
+
+    /**
+     * Creates a copy of the {@code source} array.
+     *
+     * @param source Array to be copied.
+     * @param len Number of entries to copy. If smaller then the source
+     * length, the copy will be truncated, if larger it will padded with
+     * zeroes.
+     * @return the copied array.
+     */
+    public static double[] copyOf(double[] source, int len) {
+         final double[] output = new double[len];
+         System.arraycopy(source, 0, output, 0, FastMath.min(len, source.length));
+         return output;
+     }
+
+    /**
+     * Creates a copy of the {@code source} array.
+     *
+     * @param source Array to be copied.
+     * @param from Initial index of the range to be copied, inclusive.
+     * @param to Final index of the range to be copied, exclusive. (This index may lie outside the array.)
+     * @return the copied array.
+     */
+    public static double[] copyOfRange(double[] source, int from, int to) {
+        final int len = to - from;
+        final double[] output = new double[len];
+        System.arraycopy(source, from, output, 0, FastMath.min(len, source.length - from));
+        return output;
+     }
+
+    /**
+     * Compute a linear combination accurately.
+     * This method computes the sum of the products
+     * ai bi to high accuracy.
+     * It does so by using specific multiplication and addition algorithms to
+     * preserve accuracy and reduce cancellation effects.
+     * 
      
+     * It is based on the 2005 paper
+     * 
+     * Accurate Sum and Dot Product by Takeshi Ogita, Siegfried M. Rump,
+     * and Shin'ichi Oishi published in SIAM J. Sci. Comput.
+     *
+     * @param a Factors.
+     * @param b Factors.
+     * @return Σi ai bi.
+     * @throws DimensionMismatchException if arrays dimensions don't match
+     */
+    public static double linearCombination(final double[] a, final double[] b)
+        throws DimensionMismatchException {
+        checkEqualLength(a, b);
+        final int len = a.length;
+
+        if (len == 1) {
+            // Revert to scalar multiplication.
+            return a[0] * b[0];
+        }
+
+        final double[] prodHigh = new double[len];
+        double prodLowSum = 0;
+
+        for (int i = 0; i < len; i++) {
+            final double ai    = a[i];
+            final double aHigh = Double.longBitsToDouble(Double.doubleToRawLongBits(ai) & ((-1L) << 27));
+            final double aLow  = ai - aHigh;
+
+            final double bi    = b[i];
+            final double bHigh = Double.longBitsToDouble(Double.doubleToRawLongBits(bi) & ((-1L) << 27));
+            final double bLow  = bi - bHigh;
+            prodHigh[i] = ai * bi;
+            final double prodLow = aLow * bLow - (((prodHigh[i] -
+                                                    aHigh * bHigh) -
+                                                   aLow * bHigh) -
+                                                  aHigh * bLow);
+            prodLowSum += prodLow;
+        }
+
+
+        final double prodHighCur = prodHigh[0];
+        double prodHighNext = prodHigh[1];
+        double sHighPrev = prodHighCur + prodHighNext;
+        double sPrime = sHighPrev - prodHighNext;
+        double sLowSum = (prodHighNext - (sHighPrev - sPrime)) + (prodHighCur - sPrime);
+
+        final int lenMinusOne = len - 1;
+        for (int i = 1; i < lenMinusOne; i++) {
+            prodHighNext = prodHigh[i + 1];
+            final double sHighCur = sHighPrev + prodHighNext;
+            sPrime = sHighCur - prodHighNext;
+            sLowSum += (prodHighNext - (sHighCur - sPrime)) + (sHighPrev - sPrime);
+            sHighPrev = sHighCur;
+        }
+
+        double result = sHighPrev + (prodLowSum + sLowSum);
+
+        if (Double.isNaN(result)) {
+            // either we have split infinite numbers or some coefficients were NaNs,
+            // just rely on the naive implementation and let IEEE754 handle this
+            result = 0;
+            for (int i = 0; i < len; ++i) {
+                result += a[i] * b[i];
+            }
+        }
+
+        return result;
+    }
+
+    /**
+     * Compute a linear combination accurately.
+     * 
      
+     * This method computes a1×b1 +
+     * a2×b2 to high accuracy. It does
+     * so by using specific multiplication and addition algorithms to
+     * preserve accuracy and reduce cancellation effects. It is based
+     * on the 2005 paper 
+     * Accurate Sum and Dot Product by Takeshi Ogita,
+     * Siegfried M. Rump, and Shin'ichi Oishi published in SIAM J. Sci. Comput.
+     * 
      
+     * @param a1 first factor of the first term
+     * @param b1 second factor of the first term
+     * @param a2 first factor of the second term
+     * @param b2 second factor of the second term
+     * @return a1×b1 +
+     * a2×b2
+     * @see #linearCombination(double, double, double, double, double, double)
+     * @see #linearCombination(double, double, double, double, double, double, double, double)
+     */
+    public static double linearCombination(final double a1, final double b1,
+                                           final double a2, final double b2) {
+
+        // the code below is split in many additions/subtractions that may
+        // appear redundant. However, they should NOT be simplified, as they
+        // use IEEE754 floating point arithmetic rounding properties.
+        // The variable naming conventions are that xyzHigh contains the most significant
+        // bits of xyz and xyzLow contains its least significant bits. So theoretically
+        // xyz is the sum xyzHigh + xyzLow, but in many cases below, this sum cannot
+        // be represented in only one double precision number so we preserve two numbers
+        // to hold it as long as we can, combining the high and low order bits together
+        // only at the end, after cancellation may have occurred on high order bits
+
+        // split a1 and b1 as one 26 bits number and one 27 bits number
+        final double a1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a1) & ((-1L) << 27));
+        final double a1Low      = a1 - a1High;
+        final double b1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b1) & ((-1L) << 27));
+        final double b1Low      = b1 - b1High;
+
+        // accurate multiplication a1 * b1
+        final double prod1High  = a1 * b1;
+        final double prod1Low   = a1Low * b1Low - (((prod1High - a1High * b1High) - a1Low * b1High) - a1High * b1Low);
+
+        // split a2 and b2 as one 26 bits number and one 27 bits number
+        final double a2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a2) & ((-1L) << 27));
+        final double a2Low      = a2 - a2High;
+        final double b2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b2) & ((-1L) << 27));
+        final double b2Low      = b2 - b2High;
+
+        // accurate multiplication a2 * b2
+        final double prod2High  = a2 * b2;
+        final double prod2Low   = a2Low * b2Low - (((prod2High - a2High * b2High) - a2Low * b2High) - a2High * b2Low);
+
+        // accurate addition a1 * b1 + a2 * b2
+        final double s12High    = prod1High + prod2High;
+        final double s12Prime   = s12High - prod2High;
+        final double s12Low     = (prod2High - (s12High - s12Prime)) + (prod1High - s12Prime);
+
+        // final rounding, s12 may have suffered many cancellations, we try
+        // to recover some bits from the extra words we have saved up to now
+        double result = s12High + (prod1Low + prod2Low + s12Low);
+
+        if (Double.isNaN(result)) {
+            // either we have split infinite numbers or some coefficients were NaNs,
+            // just rely on the naive implementation and let IEEE754 handle this
+            result = a1 * b1 + a2 * b2;
+        }
+
+        return result;
+    }
+
+    /**
+     * Compute a linear combination accurately.
+     * 
      
+     * This method computes a1×b1 +
+     * a2×b2 + a3×b3
+     * to high accuracy. It does so by using specific multiplication and
+     * addition algorithms to preserve accuracy and reduce cancellation effects.
+     * It is based on the 2005 paper 
+     * Accurate Sum and Dot Product by Takeshi Ogita,
+     * Siegfried M. Rump, and Shin'ichi Oishi published in SIAM J. Sci. Comput.
+     * 
      
+     * @param a1 first factor of the first term
+     * @param b1 second factor of the first term
+     * @param a2 first factor of the second term
+     * @param b2 second factor of the second term
+     * @param a3 first factor of the third term
+     * @param b3 second factor of the third term
+     * @return a1×b1 +
+     * a2×b2 + a3×b3
+     * @see #linearCombination(double, double, double, double)
+     * @see #linearCombination(double, double, double, double, double, double, double, double)
+     */
+    public static double linearCombination(final double a1, final double b1,
+                                           final double a2, final double b2,
+                                           final double a3, final double b3) {
+
+        // the code below is split in many additions/subtractions that may
+        // appear redundant. However, they should NOT be simplified, as they
+        // do use IEEE754 floating point arithmetic rounding properties.
+        // The variables naming conventions are that xyzHigh contains the most significant
+        // bits of xyz and xyzLow contains its least significant bits. So theoretically
+        // xyz is the sum xyzHigh + xyzLow, but in many cases below, this sum cannot
+        // be represented in only one double precision number so we preserve two numbers
+        // to hold it as long as we can, combining the high and low order bits together
+        // only at the end, after cancellation may have occurred on high order bits
+
+        // split a1 and b1 as one 26 bits number and one 27 bits number
+        final double a1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a1) & ((-1L) << 27));
+        final double a1Low      = a1 - a1High;
+        final double b1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b1) & ((-1L) << 27));
+        final double b1Low      = b1 - b1High;
+
+        // accurate multiplication a1 * b1
+        final double prod1High  = a1 * b1;
+        final double prod1Low   = a1Low * b1Low - (((prod1High - a1High * b1High) - a1Low * b1High) - a1High * b1Low);
+
+        // split a2 and b2 as one 26 bits number and one 27 bits number
+        final double a2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a2) & ((-1L) << 27));
+        final double a2Low      = a2 - a2High;
+        final double b2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b2) & ((-1L) << 27));
+        final double b2Low      = b2 - b2High;
+
+        // accurate multiplication a2 * b2
+        final double prod2High  = a2 * b2;
+        final double prod2Low   = a2Low * b2Low - (((prod2High - a2High * b2High) - a2Low * b2High) - a2High * b2Low);
+
+        // split a3 and b3 as one 26 bits number and one 27 bits number
+        final double a3High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a3) & ((-1L) << 27));
+        final double a3Low      = a3 - a3High;
+        final double b3High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b3) & ((-1L) << 27));
+        final double b3Low      = b3 - b3High;
+
+        // accurate multiplication a3 * b3
+        final double prod3High  = a3 * b3;
+        final double prod3Low   = a3Low * b3Low - (((prod3High - a3High * b3High) - a3Low * b3High) - a3High * b3Low);
+
+        // accurate addition a1 * b1 + a2 * b2
+        final double s12High    = prod1High + prod2High;
+        final double s12Prime   = s12High - prod2High;
+        final double s12Low     = (prod2High - (s12High - s12Prime)) + (prod1High - s12Prime);
+
+        // accurate addition a1 * b1 + a2 * b2 + a3 * b3
+        final double s123High   = s12High + prod3High;
+        final double s123Prime  = s123High - prod3High;
+        final double s123Low    = (prod3High - (s123High - s123Prime)) + (s12High - s123Prime);
+
+        // final rounding, s123 may have suffered many cancellations, we try
+        // to recover some bits from the extra words we have saved up to now
+        double result = s123High + (prod1Low + prod2Low + prod3Low + s12Low + s123Low);
+
+        if (Double.isNaN(result)) {
+            // either we have split infinite numbers or some coefficients were NaNs,
+            // just rely on the naive implementation and let IEEE754 handle this
+            result = a1 * b1 + a2 * b2 + a3 * b3;
+        }
+
+        return result;
+    }
+
+    /**
+     * Compute a linear combination accurately.
+     * 
      
+     * This method computes a1×b1 +
+     * a2×b2 + a3×b3 +
+     * a4×b4
+     * to high accuracy. It does so by using specific multiplication and
+     * addition algorithms to preserve accuracy and reduce cancellation effects.
+     * It is based on the 2005 paper 
+     * Accurate Sum and Dot Product by Takeshi Ogita,
+     * Siegfried M. Rump, and Shin'ichi Oishi published in SIAM J. Sci. Comput.
+     * 
      
+     * @param a1 first factor of the first term
+     * @param b1 second factor of the first term
+     * @param a2 first factor of the second term
+     * @param b2 second factor of the second term
+     * @param a3 first factor of the third term
+     * @param b3 second factor of the third term
+     * @param a4 first factor of the third term
+     * @param b4 second factor of the third term
+     * @return a1×b1 +
+     * a2×b2 + a3×b3 +
+     * a4×b4
+     * @see #linearCombination(double, double, double, double)
+     * @see #linearCombination(double, double, double, double, double, double)
+     */
+    public static double linearCombination(final double a1, final double b1,
+                                           final double a2, final double b2,
+                                           final double a3, final double b3,
+                                           final double a4, final double b4) {
+
+        // the code below is split in many additions/subtractions that may
+        // appear redundant. However, they should NOT be simplified, as they
+        // do use IEEE754 floating point arithmetic rounding properties.
+        // The variables naming conventions are that xyzHigh contains the most significant
+        // bits of xyz and xyzLow contains its least significant bits. So theoretically
+        // xyz is the sum xyzHigh + xyzLow, but in many cases below, this sum cannot
+        // be represented in only one double precision number so we preserve two numbers
+        // to hold it as long as we can, combining the high and low order bits together
+        // only at the end, after cancellation may have occurred on high order bits
+
+        // split a1 and b1 as one 26 bits number and one 27 bits number
+        final double a1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a1) & ((-1L) << 27));
+        final double a1Low      = a1 - a1High;
+        final double b1High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b1) & ((-1L) << 27));
+        final double b1Low      = b1 - b1High;
+
+        // accurate multiplication a1 * b1
+        final double prod1High  = a1 * b1;
+        final double prod1Low   = a1Low * b1Low - (((prod1High - a1High * b1High) - a1Low * b1High) - a1High * b1Low);
+
+        // split a2 and b2 as one 26 bits number and one 27 bits number
+        final double a2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a2) & ((-1L) << 27));
+        final double a2Low      = a2 - a2High;
+        final double b2High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b2) & ((-1L) << 27));
+        final double b2Low      = b2 - b2High;
+
+        // accurate multiplication a2 * b2
+        final double prod2High  = a2 * b2;
+        final double prod2Low   = a2Low * b2Low - (((prod2High - a2High * b2High) - a2Low * b2High) - a2High * b2Low);
+
+        // split a3 and b3 as one 26 bits number and one 27 bits number
+        final double a3High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a3) & ((-1L) << 27));
+        final double a3Low      = a3 - a3High;
+        final double b3High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b3) & ((-1L) << 27));
+        final double b3Low      = b3 - b3High;
+
+        // accurate multiplication a3 * b3
+        final double prod3High  = a3 * b3;
+        final double prod3Low   = a3Low * b3Low - (((prod3High - a3High * b3High) - a3Low * b3High) - a3High * b3Low);
+
+        // split a4 and b4 as one 26 bits number and one 27 bits number
+        final double a4High     = Double.longBitsToDouble(Double.doubleToRawLongBits(a4) & ((-1L) << 27));
+        final double a4Low      = a4 - a4High;
+        final double b4High     = Double.longBitsToDouble(Double.doubleToRawLongBits(b4) & ((-1L) << 27));
+        final double b4Low      = b4 - b4High;
+
+        // accurate multiplication a4 * b4
+        final double prod4High  = a4 * b4;
+        final double prod4Low   = a4Low * b4Low - (((prod4High - a4High * b4High) - a4Low * b4High) - a4High * b4Low);
+
+        // accurate addition a1 * b1 + a2 * b2
+        final double s12High    = prod1High + prod2High;
+        final double s12Prime   = s12High - prod2High;
+        final double s12Low     = (prod2High - (s12High - s12Prime)) + (prod1High - s12Prime);
+
+        // accurate addition a1 * b1 + a2 * b2 + a3 * b3
+        final double s123High   = s12High + prod3High;
+        final double s123Prime  = s123High - prod3High;
+        final double s123Low    = (prod3High - (s123High - s123Prime)) + (s12High - s123Prime);
+
+        // accurate addition a1 * b1 + a2 * b2 + a3 * b3 + a4 * b4
+        final double s1234High  = s123High + prod4High;
+        final double s1234Prime = s1234High - prod4High;
+        final double s1234Low   = (prod4High - (s1234High - s1234Prime)) + (s123High - s1234Prime);
+
+        // final rounding, s1234 may have suffered many cancellations, we try
+        // to recover some bits from the extra words we have saved up to now
+        double result = s1234High + (prod1Low + prod2Low + prod3Low + prod4Low + s12Low + s123Low + s1234Low);
+
+        if (Double.isNaN(result)) {
+            // either we have split infinite numbers or some coefficients were NaNs,
+            // just rely on the naive implementation and let IEEE754 handle this
+            result = a1 * b1 + a2 * b2 + a3 * b3 + a4 * b4;
+        }
+
+        return result;
+    }
+
+    /**
+     * Returns true iff both arguments are null or have same dimensions and all
+     * their elements are equal as defined by
+     * {@link Precision#equals(float,float)}.
+     *
+     * @param x first array
+     * @param y second array
+     * @return true if the values are both null or have same dimension
+     * and equal elements.
+     */
+    public static boolean equals(float[] x, float[] y) {
+        if ((x == null) || (y == null)) {
+            return !((x == null) ^ (y == null));
+        }
+        if (x.length != y.length) {
+            return false;
+        }
+        for (int i = 0; i < x.length; ++i) {
+            if (!Precision.equals(x[i], y[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Returns true iff both arguments are null or have same dimensions and all
+     * their elements are equal as defined by
+     * {@link Precision#equalsIncludingNaN(double,double) this method}.
+     *
+     * @param x first array
+     * @param y second array
+     * @return true if the values are both null or have same dimension and
+     * equal elements
+     * @since 2.2
+     */
+    public static boolean equalsIncludingNaN(float[] x, float[] y) {
+        if ((x == null) || (y == null)) {
+            return !((x == null) ^ (y == null));
+        }
+        if (x.length != y.length) {
+            return false;
+        }
+        for (int i = 0; i < x.length; ++i) {
+            if (!Precision.equalsIncludingNaN(x[i], y[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Returns {@code true} iff both arguments are {@code null} or have same
+     * dimensions and all their elements are equal as defined by
+     * {@link Precision#equals(double,double)}.
+     *
+     * @param x First array.
+     * @param y Second array.
+     * @return {@code true} if the values are both {@code null} or have same
+     * dimension and equal elements.
+     */
+    public static boolean equals(double[] x, double[] y) {
+        if ((x == null) || (y == null)) {
+            return !((x == null) ^ (y == null));
+        }
+        if (x.length != y.length) {
+            return false;
+        }
+        for (int i = 0; i < x.length; ++i) {
+            if (!Precision.equals(x[i], y[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Returns {@code true} iff both arguments are {@code null} or have same
+     * dimensions and all their elements are equal as defined by
+     * {@link Precision#equalsIncludingNaN(double,double) this method}.
+     *
+     * @param x First array.
+     * @param y Second array.
+     * @return {@code true} if the values are both {@code null} or have same
+     * dimension and equal elements.
+     * @since 2.2
+     */
+    public static boolean equalsIncludingNaN(double[] x, double[] y) {
+        if ((x == null) || (y == null)) {
+            return !((x == null) ^ (y == null));
+        }
+        if (x.length != y.length) {
+            return false;
+        }
+        for (int i = 0; i < x.length; ++i) {
+            if (!Precision.equalsIncludingNaN(x[i], y[i])) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Normalizes an array to make it sum to a specified value.
+     * Returns the result of the transformation
+     * 
      +     *    x |-> x * normalizedSum / sum
+     * 
      
+     * applied to each non-NaN element x of the input array, where sum is the
+     * sum of the non-NaN entries in the input array.
+     * 
      
+     * Throws IllegalArgumentException if {@code normalizedSum} is infinite
+     * or NaN and ArithmeticException if the input array contains any infinite elements
+     * or sums to 0.
+     * 
      
+     * Ignores (i.e., copies unchanged to the output array) NaNs in the input array.
+     *
+     * @param values Input array to be normalized
+     * @param normalizedSum Target sum for the normalized array
+     * @return the normalized array.
+     * @throws MathArithmeticException if the input array contains infinite
+     * elements or sums to zero.
+     * @throws MathIllegalArgumentException if the target sum is infinite or {@code NaN}.
+     * @since 2.1
+     */
+    public static double[] normalizeArray(double[] values, double normalizedSum)
+        throws MathIllegalArgumentException, MathArithmeticException {
+        if (Double.isInfinite(normalizedSum)) {
+            throw new MathIllegalArgumentException(LocalizedFormats.NORMALIZE_INFINITE);
+        }
+        if (Double.isNaN(normalizedSum)) {
+            throw new MathIllegalArgumentException(LocalizedFormats.NORMALIZE_NAN);
+        }
+        double sum = 0d;
+        final int len = values.length;
+        double[] out = new double[len];
+        for (int i = 0; i < len; i++) {
+            if (Double.isInfinite(values[i])) {
+                throw new MathIllegalArgumentException(LocalizedFormats.INFINITE_ARRAY_ELEMENT, values[i], i);
+            }
+            if (!Double.isNaN(values[i])) {
+                sum += values[i];
+            }
+        }
+        if (sum == 0) {
+            throw new MathArithmeticException(LocalizedFormats.ARRAY_SUMS_TO_ZERO);
+        }
+        for (int i = 0; i < len; i++) {
+            if (Double.isNaN(values[i])) {
+                out[i] = Double.NaN;
+            } else {
+                out[i] = values[i] * normalizedSum / sum;
+            }
+        }
+        return out;
+    }
+
+    /** Build an array of elements.
+     * 
      
+     * Arrays are filled with field.getZero()
+     *
+     * @param  the type of the field elements
+     * @param field field to which array elements belong
+     * @param length of the array
+     * @return a new array
+     * @since 3.2
+     */
+    public static  T[] buildArray(final Field field, final int length) {
+        @SuppressWarnings("unchecked") // OK because field must be correct class
+        T[] array = (T[]) Array.newInstance(field.getRuntimeClass(), length);
+        Arrays.fill(array, field.getZero());
+        return array;
+    }
+
+    /** Build a double dimension  array of elements.
+     * 
      
+     * Arrays are filled with field.getZero()
+     *
+     * @param  the type of the field elements
+     * @param field field to which array elements belong
+     * @param rows number of rows in the array
+     * @param columns number of columns (may be negative to build partial
+     * arrays in the same way new Field[rows][] works)
+     * @return a new array
+     * @since 3.2
+     */
+    @SuppressWarnings("unchecked")
+    public static  T[][] buildArray(final Field field, final int rows, final int columns) {
+        final T[][] array;
+        if (columns < 0) {
+            T[] dummyRow = buildArray(field, 0);
+            array = (T[][]) Array.newInstance(dummyRow.getClass(), rows);
+        } else {
+            array = (T[][]) Array.newInstance(field.getRuntimeClass(),
+                                              new int[] {
+                                                  rows, columns
+                                              });
+            for (int i = 0; i < rows; ++i) {
+                Arrays.fill(array[i], field.getZero());
+            }
+        }
+        return array;
+    }
+
+    /**
+     * Calculates the 
+     * convolution between two sequences.
+     * 
      
+     * The solution is obtained via straightforward computation of the
+     * convolution sum (and not via FFT). Whenever the computation needs
+     * an element that would be located at an index outside the input arrays,
+     * the value is assumed to be zero.
+     *
+     * @param x First sequence.
+     * Typically, this sequence will represent an input signal to a system.
+     * @param h Second sequence.
+     * Typically, this sequence will represent the impulse response of the system.
+     * @return the convolution of {@code x} and {@code h}.
+     * This array's length will be {@code x.length + h.length - 1}.
+     * @throws NullArgumentException if either {@code x} or {@code h} is {@code null}.
+     * @throws NoDataException if either {@code x} or {@code h} is empty.
+     *
+     * @since 3.3
+     */
+    public static double[] convolve(double[] x, double[] h)
+        throws NullArgumentException,
+               NoDataException {
+        MathUtils.checkNotNull(x);
+        MathUtils.checkNotNull(h);
+
+        final int xLen = x.length;
+        final int hLen = h.length;
+
+        if (xLen == 0 || hLen == 0) {
+            throw new NoDataException();
+        }
+
+        // initialize the output array
+        final int totalLength = xLen + hLen - 1;
+        final double[] y = new double[totalLength];
+
+        // straightforward implementation of the convolution sum
+        for (int n = 0; n < totalLength; n++) {
+            double yn = 0;
+            int k = FastMath.max(0, n + 1 - xLen);
+            int j = n - k;
+            while (k < hLen && j >= 0) {
+                yn += x[j--] * h[k++];
+            }
+            y[n] = yn;
+        }
+
+        return y;
+    }
+
+    /**
+     * Specification for indicating that some operation applies
+     * before or after a given index.
+     */
+    public enum Position {
+        /** Designates the beginning of the array (near index 0). */
+        HEAD,
+        /** Designates the end of the array. */
+        TAIL
+    }
+
+    /**
+     * Shuffle the entries of the given array.
+     * The {@code start} and {@code pos} parameters select which portion
+     * of the array is randomized and which is left untouched.
+     *
+     * @see #shuffle(int[],int,Position,RandomGenerator)
+     *
+     * @param list Array whose entries will be shuffled (in-place).
+     * @param start Index at which shuffling begins.
+     * @param pos Shuffling is performed for index positions between
+     * {@code start} and either the end (if {@link Position#TAIL})
+     * or the beginning (if {@link Position#HEAD}) of the array.
+     */
+    public static void shuffle(int[] list,
+                               int start,
+                               Position pos) {
+        shuffle(list, start, pos, new Well19937c());
+    }
+
+    /**
+     * Shuffle the entries of the given array, using the
+     * 
+     * Fisher–Yates algorithm.
+     * The {@code start} and {@code pos} parameters select which portion
+     * of the array is randomized and which is left untouched.
+     *
+     * @param list Array whose entries will be shuffled (in-place).
+     * @param start Index at which shuffling begins.
+     * @param pos Shuffling is performed for index positions between
+     * {@code start} and either the end (if {@link Position#TAIL})
+     * or the beginning (if {@link Position#HEAD}) of the array.
+     * @param rng Random number generator.
+     */
+    public static void shuffle(int[] list,
+                               int start,
+                               Position pos,
+                               RandomGenerator rng) {
+        switch (pos) {
+        case TAIL: {
+            for (int i = list.length - 1; i >= start; i--) {
+                final int target;
+                if (i == start) {
+                    target = start;
+                } else {
+                    // NumberIsTooLargeException cannot occur.
+                    target = new UniformIntegerDistribution(rng, start, i).sample();
+                }
+                final int temp = list[target];
+                list[target] = list[i];
+                list[i] = temp;
+            }
+        }
+            break;
+        case HEAD: {
+            for (int i = 0; i <= start; i++) {
+                final int target;
+                if (i == start) {
+                    target = start;
+                } else {
+                    // NumberIsTooLargeException cannot occur.
+                    target = new UniformIntegerDistribution(rng, i, start).sample();
+                }
+                final int temp = list[target];
+                list[target] = list[i];
+                list[i] = temp;
+            }
+        }
+            break;
+        default:
+            throw new MathInternalError(); // Should never happen.
+        }
+    }
+
+    /**
+     * Shuffle the entries of the given array.
+     *
+     * @see #shuffle(int[],int,Position,RandomGenerator)
+     *
+     * @param list Array whose entries will be shuffled (in-place).
+     * @param rng Random number generator.
+     */
+    public static void shuffle(int[] list,
+                               RandomGenerator rng) {
+        shuffle(list, 0, Position.TAIL, rng);
+    }
+
+    /**
+     * Shuffle the entries of the given array.
+     *
+     * @see #shuffle(int[],int,Position,RandomGenerator)
+     *
+     * @param list Array whose entries will be shuffled (in-place).
+     */
+    public static void shuffle(int[] list) {
+        shuffle(list, new Well19937c());
+    }
+
+    /**
+     * Returns an array representing the natural number {@code n}.
+     *
+     * @param n Natural number.
+     * @return an array whose entries are the numbers 0, 1, ..., {@code n}-1.
+     * If {@code n == 0}, the returned array is empty.
+     */
+    public static int[] natural(int n) {
+        return sequence(n, 0, 1);
+    }
+    /**
+     * Returns an array of {@code size} integers starting at {@code start},
+     * skipping {@code stride} numbers.
+     *
+     * @param size Natural number.
+     * @param start Natural number.
+     * @param stride Natural number.
+     * @return an array whose entries are the numbers
+     * {@code start, start + stride, ..., start + (size - 1) * stride}.
+     * If {@code size == 0}, the returned array is empty.
+     *
+     * @since 3.4
+     */
+    public static int[] sequence(int size,
+                                 int start,
+                                 int stride) {
+        final int[] a = new int[size];
+        for (int i = 0; i < size; i++) {
+            a[i] = start + i * stride;
+        }
+        return a;
+    }
+    /**
+     * This method is used
+     * to verify that the input parameters designate a subarray of positive length.
+     * 
      
+     * 
        
+     * 
      • returns true iff the parameters designate a subarray of
+     * positive length
        • 
+     * 
      • throws MathIllegalArgumentException if the array is null or
+     * or the indices are invalid
        • 
+     * 
      • returns false
        •  if the array is non-null, but
+     * length is 0.
+     * 
      
+     *
+     * @param values the input array
+     * @param begin index of the first array element to include
+     * @param length the number of elements to include
+     * @return true if the parameters are valid and designate a subarray of positive length
+     * @throws MathIllegalArgumentException if the indices are invalid or the array is null
+     * @since 3.3
+     */
+    public static boolean verifyValues(final double[] values, final int begin, final int length)
+            throws MathIllegalArgumentException {
+        return verifyValues(values, begin, length, false);
+    }
+
+    /**
+     * This method is used
+     * to verify that the input parameters designate a subarray of positive length.
+     * 
      
+     * 
        
+     * 
      • returns true iff the parameters designate a subarray of
+     * non-negative length
        • 
+     * 
      • throws IllegalArgumentException if the array is null or
+     * or the indices are invalid
        • 
+     * 
      • returns false
        •  if the array is non-null, but
+     * length is 0 unless allowEmpty is true
+     * 
      
+     *
+     * @param values the input array
+     * @param begin index of the first array element to include
+     * @param length the number of elements to include
+     * @param allowEmpty if true then zero length arrays are allowed
+     * @return true if the parameters are valid
+     * @throws MathIllegalArgumentException if the indices are invalid or the array is null
+     * @since 3.3
+     */
+    public static boolean verifyValues(final double[] values, final int begin,
+            final int length, final boolean allowEmpty) throws MathIllegalArgumentException {
+
+        if (values == null) {
+            throw new NullArgumentException(LocalizedFormats.INPUT_ARRAY);
+        }
+
+        if (begin < 0) {
+            throw new NotPositiveException(LocalizedFormats.START_POSITION, Integer.valueOf(begin));
+        }
+
+        if (length < 0) {
+            throw new NotPositiveException(LocalizedFormats.LENGTH, Integer.valueOf(length));
+        }
+
+        if (begin + length > values.length) {
+            throw new NumberIsTooLargeException(LocalizedFormats.SUBARRAY_ENDS_AFTER_ARRAY_END,
+                    Integer.valueOf(begin + length), Integer.valueOf(values.length), true);
+        }
+
+        if (length == 0 && !allowEmpty) {
+            return false;
+        }
+
+        return true;
+
+    }
+
+    /**
+     * This method is used
+     * to verify that the begin and length parameters designate a subarray of positive length
+     * and the weights are all non-negative, non-NaN, finite, and not all zero.
+     * 
      
+     * 
        
+     * 
      • returns true iff the parameters designate a subarray of
+     * positive length and the weights array contains legitimate values.
        • 
+     * 
      • throws IllegalArgumentException if any of the following are true:
+     * 
        • the values array is null
          • 
+     *     
        • the weights array is null
          • 
+     *     
        • the weights array does not have the same length as the values array
          • 
+     *     
        • the weights array contains one or more infinite values
          • 
+     *     
        • the weights array contains one or more NaN values
          • 
+     *     
        • the weights array contains negative values
          • 
+     *     
        • the start and length arguments do not determine a valid array
        
+     * 
        • 
+     * 
      • returns false
        •  if the array is non-null, but
+     * length is 0.
+     * 
      
+     *
+     * @param values the input array
+     * @param weights the weights array
+     * @param begin index of the first array element to include
+     * @param length the number of elements to include
+     * @return true if the parameters are valid and designate a subarray of positive length
+     * @throws MathIllegalArgumentException if the indices are invalid or the array is null
+     * @since 3.3
+     */
+    public static boolean verifyValues(
+        final double[] values,
+        final double[] weights,
+        final int begin,
+        final int length) throws MathIllegalArgumentException {
+        return verifyValues(values, weights, begin, length, false);
+    }
+
+    /**
+     * This method is used
+     * to verify that the begin and length parameters designate a subarray of positive length
+     * and the weights are all non-negative, non-NaN, finite, and not all zero.
+     * 
      
+     * 
        
+     * 
      • returns true iff the parameters designate a subarray of
+     * non-negative length and the weights array contains legitimate values.
        • 
+     * 
      • throws MathIllegalArgumentException if any of the following are true:
+     * 
        • the values array is null
          • 
+     *     
        • the weights array is null
          • 
+     *     
        • the weights array does not have the same length as the values array
          • 
+     *     
        • the weights array contains one or more infinite values
          • 
+     *     
        • the weights array contains one or more NaN values
          • 
+     *     
        • the weights array contains negative values
          • 
+     *     
        • the start and length arguments do not determine a valid array
        
+     * 
        • 
+     * 
      • returns false
        •  if the array is non-null, but
+     * length is 0 unless allowEmpty is true.
+     * 
      
+     *
+     * @param values the input array.
+     * @param weights the weights array.
+     * @param begin index of the first array element to include.
+     * @param length the number of elements to include.
+     * @param allowEmpty if {@code true} than allow zero length arrays to pass.
+     * @return {@code true} if the parameters are valid.
+     * @throws NullArgumentException if either of the arrays are null
+     * @throws MathIllegalArgumentException if the array indices are not valid,
+     * the weights array contains NaN, infinite or negative elements, or there
+     * are no positive weights.
+     * @since 3.3
+     */
+    public static boolean verifyValues(final double[] values, final double[] weights,
+            final int begin, final int length, final boolean allowEmpty) throws MathIllegalArgumentException {
+
+        if (weights == null || values == null) {
+            throw new NullArgumentException(LocalizedFormats.INPUT_ARRAY);
+        }
+
+        checkEqualLength(weights, values);
+
+        boolean containsPositiveWeight = false;
+        for (int i = begin; i < begin + length; i++) {
+            final double weight = weights[i];
+            if (Double.isNaN(weight)) {
+                throw new MathIllegalArgumentException(LocalizedFormats.NAN_ELEMENT_AT_INDEX, Integer.valueOf(i));
+            }
+            if (Double.isInfinite(weight)) {
+                throw new MathIllegalArgumentException(LocalizedFormats.INFINITE_ARRAY_ELEMENT, Double.valueOf(weight), Integer.valueOf(i));
+            }
+            if (weight < 0) {
+                throw new MathIllegalArgumentException(LocalizedFormats.NEGATIVE_ELEMENT_AT_INDEX, Integer.valueOf(i), Double.valueOf(weight));
+            }
+            if (!containsPositiveWeight && weight > 0.0) {
+                containsPositiveWeight = true;
+            }
+        }
+
+        if (!containsPositiveWeight) {
+            throw new MathIllegalArgumentException(LocalizedFormats.WEIGHT_AT_LEAST_ONE_NON_ZERO);
+        }
+
+        return verifyValues(values, begin, length, allowEmpty);
+    }
+
+    /**
+     * Concatenates a sequence of arrays. The return array consists of the
+     * entries of the input arrays concatenated in the order they appear in
+     * the argument list.  Null arrays cause NullPointerExceptions; zero
+     * length arrays are allowed (contributing nothing to the output array).
+     *
+     * @param x list of double[] arrays to concatenate
+     * @return a new array consisting of the entries of the argument arrays
+     * @throws NullPointerException if any of the arrays are null
+     * @since 3.6
+     */
+    public static double[] concatenate(double[] ...x) {
+        int combinedLength = 0;
+        for (double[] a : x) {
+            combinedLength += a.length;
+        }
+        int offset = 0;
+        int curLength = 0;
+        final double[] combined = new double[combinedLength];
+        for (int i = 0; i < x.length; i++) {
+            curLength = x[i].length;
+            System.arraycopy(x[i], 0, combined, offset, curLength);
+            offset += curLength;
+        }
+        return combined;
+    }
+
+    /**
+     * Returns an array consisting of the unique values in {@code data}.
+     * The return array is sorted in descending order.  Empty arrays
+     * are allowed, but null arrays result in NullPointerException.
+     * Infinities are allowed.  NaN values are allowed with maximum
+     * sort order - i.e., if there are NaN values in {@code data},
+     * {@code Double.NaN} will be the first element of the output array,
+     * even if the array also contains {@code Double.POSITIVE_INFINITY}.
+     *
+     * @param data array to scan
+     * @return descending list of values included in the input array
+     * @throws NullPointerException if data is null
+     * @since 3.6
+     */
+    public static double[] unique(double[] data) {
+        TreeSet values = new TreeSet();
+        for (int i = 0; i < data.length; i++) {
+            values.add(data[i]);
+        }
+        final int count = values.size();
+        final double[] out = new double[count];
+        Iterator iterator = values.iterator();
+        int i = 0;
+        while (iterator.hasNext()) {
+            out[count - ++i] = iterator.next();
+        }
+        return out;
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/MathUtils.java b/java/source/infodynamics/utils/commonsmath3/util/MathUtils.java
new file mode 100644
index 0000000..5509abe
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/MathUtils.java
@@ -0,0 +1,338 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package infodynamics.utils.commonsmath3.util;
+
+import java.util.Arrays;
+
+import infodynamics.utils.commonsmath3.RealFieldElement;
+import infodynamics.utils.commonsmath3.exception.MathArithmeticException;
+import infodynamics.utils.commonsmath3.exception.NotFiniteNumberException;
+import infodynamics.utils.commonsmath3.exception.NullArgumentException;
+import infodynamics.utils.commonsmath3.exception.util.Localizable;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
+/**
+ * Miscellaneous utility functions.
+ *
+ * @see ArithmeticUtils
+ * @see Precision
+ * @see MathArrays
+ *
+ */
+public final class MathUtils {
+    /**
+     * \(2\pi\)
+     * @since 2.1
+     */
+    public static final double TWO_PI = 2 * FastMath.PI;
+
+    /**
+     * \(\pi^2\)
+     * @since 3.4
+     */
+    public static final double PI_SQUARED = FastMath.PI * FastMath.PI;
+
+
+    /**
+     * Class contains only static methods.
+     */
+    private MathUtils() {}
+
+
+    /**
+     * Returns an integer hash code representing the given double value.
+     *
+     * @param value the value to be hashed
+     * @return the hash code
+     */
+    public static int hash(double value) {
+        return new Double(value).hashCode();
+    }
+
+    /**
+     * Returns {@code true} if the values are equal according to semantics of
+     * {@link Double#equals(Object)}.
+     *
+     * @param x Value
+     * @param y Value
+     * @return {@code new Double(x).equals(new Double(y))}
+     */
+    public static boolean equals(double x, double y) {
+        return new Double(x).equals(new Double(y));
+    }
+
+    /**
+     * Returns an integer hash code representing the given double array.
+     *
+     * @param value the value to be hashed (may be null)
+     * @return the hash code
+     * @since 1.2
+     */
+    public static int hash(double[] value) {
+        return Arrays.hashCode(value);
+    }
+
+    /**
+     * Normalize an angle in a 2π wide interval around a center value.
+     * 
      This method has three main uses:
      
+     * 
        
+     *   
      • normalize an angle between 0 and 2π:
        
+     *       {@code a = MathUtils.normalizeAngle(a, FastMath.PI);}
        • 
+     *   
      • normalize an angle between -π and +π
        
+     *       {@code a = MathUtils.normalizeAngle(a, 0.0);}
        • 
+     *   
      • compute the angle between two defining angular positions:
        
+     *       {@code angle = MathUtils.normalizeAngle(end, start) - start;}
        • 
+     * 
      
+     * 
      Note that due to numerical accuracy and since π cannot be represented
+     * exactly, the result interval is closed, it cannot be half-closed
+     * as would be more satisfactory in a purely mathematical view.
      
+     * @param a angle to normalize
+     * @param center center of the desired 2π interval for the result
+     * @return a-2kπ with integer k and center-π <= a-2kπ <= center+π
+     * @since 1.2
+     */
+     public static double normalizeAngle(double a, double center) {
+         return a - TWO_PI * FastMath.floor((a + FastMath.PI - center) / TWO_PI);
+     }
+
+     /** Find the maximum of two field elements.
+      * @param  the type of the field elements
+      * @param e1 first element
+      * @param e2 second element
+      * @return max(a1, e2)
+      * @since 3.6
+      */
+     public static > T max(final T e1, final T e2) {
+         return e1.subtract(e2).getReal() >= 0 ? e1 : e2;
+     }
+
+     /** Find the minimum of two field elements.
+      * @param  the type of the field elements
+      * @param e1 first element
+      * @param e2 second element
+      * @return min(a1, e2)
+      * @since 3.6
+      */
+     public static > T min(final T e1, final T e2) {
+         return e1.subtract(e2).getReal() >= 0 ? e2 : e1;
+     }
+
+    /**
+     * 
      Reduce {@code |a - offset|} to the primary interval
+     * {@code [0, |period|)}.
      
+     *
+     * 
      Specifically, the value returned is 
      
+     * {@code a - |period| * floor((a - offset) / |period|) - offset}.
      
+     *
+     * 
      If any of the parameters are {@code NaN} or infinite, the result is
+     * {@code NaN}.
      
+     *
+     * @param a Value to reduce.
+     * @param period Period.
+     * @param offset Value that will be mapped to {@code 0}.
+     * @return the value, within the interval {@code [0 |period|)},
+     * that corresponds to {@code a}.
+     */
+    public static double reduce(double a,
+                                double period,
+                                double offset) {
+        final double p = FastMath.abs(period);
+        return a - p * FastMath.floor((a - offset) / p) - offset;
+    }
+
+    /**
+     * Returns the first argument with the sign of the second argument.
+     *
+     * @param magnitude Magnitude of the returned value.
+     * @param sign Sign of the returned value.
+     * @return a value with magnitude equal to {@code magnitude} and with the
+     * same sign as the {@code sign} argument.
+     * @throws MathArithmeticException if {@code magnitude == Byte.MIN_VALUE}
+     * and {@code sign >= 0}.
+     */
+    public static byte copySign(byte magnitude, byte sign)
+        throws MathArithmeticException {
+        if ((magnitude >= 0 && sign >= 0) ||
+            (magnitude < 0 && sign < 0)) { // Sign is OK.
+            return magnitude;
+        } else if (sign >= 0 &&
+                   magnitude == Byte.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW);
+        } else {
+            return (byte) -magnitude; // Flip sign.
+        }
+    }
+
+    /**
+     * Returns the first argument with the sign of the second argument.
+     *
+     * @param magnitude Magnitude of the returned value.
+     * @param sign Sign of the returned value.
+     * @return a value with magnitude equal to {@code magnitude} and with the
+     * same sign as the {@code sign} argument.
+     * @throws MathArithmeticException if {@code magnitude == Short.MIN_VALUE}
+     * and {@code sign >= 0}.
+     */
+    public static short copySign(short magnitude, short sign)
+            throws MathArithmeticException {
+        if ((magnitude >= 0 && sign >= 0) ||
+            (magnitude < 0 && sign < 0)) { // Sign is OK.
+            return magnitude;
+        } else if (sign >= 0 &&
+                   magnitude == Short.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW);
+        } else {
+            return (short) -magnitude; // Flip sign.
+        }
+    }
+
+    /**
+     * Returns the first argument with the sign of the second argument.
+     *
+     * @param magnitude Magnitude of the returned value.
+     * @param sign Sign of the returned value.
+     * @return a value with magnitude equal to {@code magnitude} and with the
+     * same sign as the {@code sign} argument.
+     * @throws MathArithmeticException if {@code magnitude == Integer.MIN_VALUE}
+     * and {@code sign >= 0}.
+     */
+    public static int copySign(int magnitude, int sign)
+            throws MathArithmeticException {
+        if ((magnitude >= 0 && sign >= 0) ||
+            (magnitude < 0 && sign < 0)) { // Sign is OK.
+            return magnitude;
+        } else if (sign >= 0 &&
+                   magnitude == Integer.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW);
+        } else {
+            return -magnitude; // Flip sign.
+        }
+    }
+
+    /**
+     * Returns the first argument with the sign of the second argument.
+     *
+     * @param magnitude Magnitude of the returned value.
+     * @param sign Sign of the returned value.
+     * @return a value with magnitude equal to {@code magnitude} and with the
+     * same sign as the {@code sign} argument.
+     * @throws MathArithmeticException if {@code magnitude == Long.MIN_VALUE}
+     * and {@code sign >= 0}.
+     */
+    public static long copySign(long magnitude, long sign)
+        throws MathArithmeticException {
+        if ((magnitude >= 0 && sign >= 0) ||
+            (magnitude < 0 && sign < 0)) { // Sign is OK.
+            return magnitude;
+        } else if (sign >= 0 &&
+                   magnitude == Long.MIN_VALUE) {
+            throw new MathArithmeticException(LocalizedFormats.OVERFLOW);
+        } else {
+            return -magnitude; // Flip sign.
+        }
+    }
+    /**
+     * Check that the argument is a real number.
+     *
+     * @param x Argument.
+     * @throws NotFiniteNumberException if {@code x} is not a
+     * finite real number.
+     */
+    public static void checkFinite(final double x)
+        throws NotFiniteNumberException {
+        if (Double.isInfinite(x) || Double.isNaN(x)) {
+            throw new NotFiniteNumberException(x);
+        }
+    }
+
+    /**
+     * Check that all the elements are real numbers.
+     *
+     * @param val Arguments.
+     * @throws NotFiniteNumberException if any values of the array is not a
+     * finite real number.
+     */
+    public static void checkFinite(final double[] val)
+        throws NotFiniteNumberException {
+        for (int i = 0; i < val.length; i++) {
+            final double x = val[i];
+            if (Double.isInfinite(x) || Double.isNaN(x)) {
+                throw new NotFiniteNumberException(LocalizedFormats.ARRAY_ELEMENT, x, i);
+            }
+        }
+    }
+
+    /**
+     * Checks that an object is not null.
+     *
+     * @param o Object to be checked.
+     * @param pattern Message pattern.
+     * @param args Arguments to replace the placeholders in {@code pattern}.
+     * @throws NullArgumentException if {@code o} is {@code null}.
+     */
+    public static void checkNotNull(Object o,
+                                    Localizable pattern,
+                                    Object ... args)
+        throws NullArgumentException {
+        if (o == null) {
+            throw new NullArgumentException(pattern, args);
+        }
+    }
+
+    /**
+     * Checks that an object is not null.
+     *
+     * @param o Object to be checked.
+     * @throws NullArgumentException if {@code o} is {@code null}.
+     */
+    public static void checkNotNull(Object o)
+        throws NullArgumentException {
+        if (o == null) {
+            throw new NullArgumentException();
+        }
+    }
+}
diff --git a/java/source/infodynamics/utils/commonsmath3/util/Precision.java b/java/source/infodynamics/utils/commonsmath3/util/Precision.java
old mode 100755
new mode 100644
index dc8c294..5e3fcb5
--- a/java/source/infodynamics/utils/commonsmath3/util/Precision.java
+++ b/java/source/infodynamics/utils/commonsmath3/util/Precision.java
@@ -1,6 +1,6 @@
 /*
  *  Java Information Dynamics Toolkit (JIDT)
- *  Copyright (C) 2012, Joseph T. Lizier
+ *  Copyright (C) 2017, Joseph T. Lizier
  *  
  *  This program is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -18,7 +18,7 @@
 
 /*
  * This class was originally distributed as part of the Apache Commons
- *  Math3 library, under the Apache License Version 2.0, which is 
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
  *  copied below. This Apache 2 software is now included as a derivative
  *  work in the GPLv3 licensed JIDT project, as per:
  *  http://www.apache.org/licenses/GPL-compatibility.html
@@ -56,7 +56,6 @@ import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
  * Utilities for comparing numbers.
  *
  * @since 3.0
- * @version $Id$
  */
 public class Precision {
     /**
@@ -129,7 +128,8 @@ public class Precision {
      * @param eps the amount of error to allow when checking for equality
      * @return 
      • 0 if  {@link #equals(double, double, double) equals(x, y, eps)}
        • 
      *       
      • < 0 if !{@link #equals(double, double, double) equals(x, y, eps)} && x < y
        • 
-     *       
      • > 0 if !{@link #equals(double, double, double) equals(x, y, eps)} && x > y
      
+     *       
    6. > 0 if !{@link #equals(double, double, double) equals(x, y, eps)} && x > y or
+     *       either argument is NaN
      7. 
      */
     public static int compareTo(double x, double y, double eps) {
         if (equals(x, y, eps)) {
@@ -147,7 +147,7 @@ public class Precision {
      * point numbers are considered equal.
      * Adapted from 
-     * Bruce Dawson
+     * Bruce Dawson. Returns {@code false} if either of the arguments is NaN.
      *
      * @param x first value
      * @param y second value
@@ -155,7 +155,8 @@ public class Precision {
      * values between {@code x} and {@code y}.
      * @return 
      • 0 if  {@link #equals(double, double, int) equals(x, y, maxUlps)}
        • 
      *       
      • < 0 if !{@link #equals(double, double, int) equals(x, y, maxUlps)} && x < y
        • 
-     *       
      • > 0 if !{@link #equals(double, double, int) equals(x, y, maxUlps)} && x > y
      
+     *       
    7. > 0 if !{@link #equals(double, double, int) equals(x, y, maxUlps)} && x > y
+     *       or either argument is NaN
      8. 
      */
     public static int compareTo(final double x, final double y, final int maxUlps) {
         if (equals(x, y, maxUlps)) {
@@ -179,7 +180,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are NaN or neither is NaN and they are
+     * Returns true if both arguments are NaN or they are
      * equal as defined by {@link #equals(float,float) equals(x, y, 1)}.
      *
      * @param x first value
@@ -192,8 +193,9 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are equal or within the range of allowed
-     * error (inclusive).
+     * Returns true if the arguments are equal or within the range of allowed
+     * error (inclusive).  Returns {@code false} if either of the arguments
+     * is NaN.
      *
      * @param x first value
      * @param y second value
@@ -206,7 +208,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are NaN or are equal or within the range
+     * Returns true if the arguments are both NaN, are equal, or are within the range
      * of allowed error (inclusive).
      *
      * @param x first value
@@ -221,14 +223,14 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are equal or within the range of allowed
+     * Returns true if the arguments are equal or within the range of allowed
      * error (inclusive).
      * Two float numbers are considered equal if there are {@code (maxUlps - 1)}
      * (or fewer) floating point numbers between them, i.e. two adjacent floating
      * point numbers are considered equal.
      * Adapted from 
-     * Bruce Dawson
+     * Bruce Dawson.  Returns {@code false} if either of the arguments is NaN.
      *
      * @param x first value
      * @param y second value
@@ -272,7 +274,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are NaN or if they are equal as defined
+     * Returns true if the arguments are both NaN or if they are equal as defined
      * by {@link #equals(float,float,int) equals(x, y, maxUlps)}.
      *
      * @param x first value
@@ -300,7 +302,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are NaN or neither is NaN and they are
+     * Returns true if the arguments are both NaN or they are
      * equal as defined by {@link #equals(double,double) equals(x, y, 1)}.
      *
      * @param x first value
@@ -315,7 +317,8 @@ public class Precision {
     /**
      * Returns {@code true} if there is no double value strictly between the
      * arguments or the difference between them is within the range of allowed
-     * error (inclusive).
+     * error (inclusive). Returns {@code false} if either of the arguments
+     * is NaN.
      *
      * @param x First value.
      * @param y Second value.
@@ -329,8 +332,9 @@ public class Precision {
 
     /**
      * Returns {@code true} if there is no double value strictly between the
-     * arguments or the relative difference between them is smaller or equal
-     * to the given tolerance.
+     * arguments or the relative difference between them is less than or equal
+     * to the given tolerance. Returns {@code false} if either of the arguments
+     * is NaN.
      *
      * @param x First value.
      * @param y Second value.
@@ -351,7 +355,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are NaN or are equal or within the range
+     * Returns true if the arguments are both NaN, are equal or are within the range
      * of allowed error (inclusive).
      *
      * @param x first value
@@ -366,7 +370,7 @@ public class Precision {
     }
 
     /**
-     * Returns true if both arguments are equal or within the range of allowed
+     * Returns true if the arguments are equal or within the range of allowed
      * error (inclusive).
      * 
      
      * Two float numbers are considered equal if there are {@code (maxUlps - 1)}
@@ -376,7 +380,7 @@ public class Precision {
      * 
      
      * Adapted from 
-     * Bruce Dawson
+     * Bruce Dawson. Returns {@code false} if either of the arguments is NaN.
      * 
      
      *
      * @param x first value
diff --git a/java/source/infodynamics/utils/commonsmath3/util/ResizableDoubleArray.java b/java/source/infodynamics/utils/commonsmath3/util/ResizableDoubleArray.java
new file mode 100644
index 0000000..4e24cf9
--- /dev/null
+++ b/java/source/infodynamics/utils/commonsmath3/util/ResizableDoubleArray.java
@@ -0,0 +1,1239 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+*/
+
+/*
+ * This class was originally distributed as part of the Apache Commons
+ *  Math3 library (3.6.1), under the Apache License Version 2.0, which is 
+ *  copied below. This Apache 2 software is now included as a derivative
+ *  work in the GPLv3 licensed JIDT project, as per:
+ *  http://www.apache.org/licenses/GPL-compatibility.html
+ *  
+ * The original Apache source code has been modified as follows:
+ * -- We have modified package names to sit inside the JIDT structure.
+ */
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package infodynamics.utils.commonsmath3.util;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+import infodynamics.utils.commonsmath3.exception.MathIllegalArgumentException;
+import infodynamics.utils.commonsmath3.exception.MathIllegalStateException;
+import infodynamics.utils.commonsmath3.exception.MathInternalError;
+import infodynamics.utils.commonsmath3.exception.NullArgumentException;
+import infodynamics.utils.commonsmath3.exception.NotStrictlyPositiveException;
+import infodynamics.utils.commonsmath3.exception.NumberIsTooSmallException;
+import infodynamics.utils.commonsmath3.exception.util.LocalizedFormats;
+
+/**
+ * 
      
+ * A variable length {@link DoubleArray} implementation that automatically
+ * handles expanding and contracting its internal storage array as elements
+ * are added and removed.
+ * 
      
+ * 
      Important note: Usage should not assume that this class is thread-safe
+ * even though some of the methods are {@code synchronized}.
+ * This qualifier will be dropped in the next major release (4.0).
      
+ * 
      
+ * The internal storage array starts with capacity determined by the
+ * {@code initialCapacity} property, which can be set by the constructor.
+ * The default initial capacity is 16.  Adding elements using
+ * {@link #addElement(double)} appends elements to the end of the array.
+ * When there are no open entries at the end of the internal storage array,
+ * the array is expanded.  The size of the expanded array depends on the
+ * {@code expansionMode} and {@code expansionFactor} properties.
+ * The {@code expansionMode} determines whether the size of the array is
+ * multiplied by the {@code expansionFactor}
+ * ({@link ExpansionMode#MULTIPLICATIVE}) or if the expansion is additive
+ * ({@link ExpansionMode#ADDITIVE} -- {@code expansionFactor} storage
+ * locations added).
+ * The default {@code expansionMode} is {@code MULTIPLICATIVE} and the default
+ * {@code expansionFactor} is 2.
+ * 
      
+ * 
      
+ * The {@link #addElementRolling(double)} method adds a new element to the end
+ * of the internal storage array and adjusts the "usable window" of the
+ * internal array forward by one position (effectively making what was the
+ * second element the first, and so on).  Repeated activations of this method
+ * (or activation of {@link #discardFrontElements(int)}) will effectively orphan
+ * the storage locations at the beginning of the internal storage array.  To
+ * reclaim this storage, each time one of these methods is activated, the size
+ * of the internal storage array is compared to the number of addressable
+ * elements (the {@code numElements} property) and if the difference
+ * is too large, the internal array is contracted to size
+ * {@code numElements + 1}.  The determination of when the internal
+ * storage array is "too large" depends on the {@code expansionMode} and
+ * {@code contractionFactor} properties.  If  the {@code expansionMode}
+ * is {@code MULTIPLICATIVE}, contraction is triggered when the
+ * ratio between storage array length and {@code numElements} exceeds
+ * {@code contractionFactor.}  If the {@code expansionMode}
+ * is {@code ADDITIVE}, the number of excess storage locations
+ * is compared to {@code contractionFactor}.
+ * 
      
+ * 
      
+ * To avoid cycles of expansions and contractions, the
+ * {@code expansionFactor} must not exceed the {@code contractionFactor}.
+ * Constructors and mutators for both of these properties enforce this
+ * requirement, throwing a {@code MathIllegalArgumentException} if it is
+ * violated.
+ * 
      
+ */
+public class ResizableDoubleArray implements DoubleArray, Serializable {
+    /** Additive expansion mode.
+     * @deprecated As of 3.1. Please use {@link ExpansionMode#ADDITIVE} instead.
+     */
+    @Deprecated
+    public static final int ADDITIVE_MODE = 1;
+    /** Multiplicative expansion mode.
+     * @deprecated As of 3.1. Please use {@link ExpansionMode#MULTIPLICATIVE} instead.
+     */
+    @Deprecated
+    public static final int MULTIPLICATIVE_MODE = 0;
+    /** Serializable version identifier. */
+    private static final long serialVersionUID = -3485529955529426875L;
+
+    /** Default value for initial capacity. */
+    private static final int DEFAULT_INITIAL_CAPACITY = 16;
+    /** Default value for array size modifier. */
+    private static final double DEFAULT_EXPANSION_FACTOR = 2.0;
+    /**
+     * Default value for the difference between {@link #contractionCriterion}
+     * and {@link #expansionFactor}.
+     */
+    private static final double DEFAULT_CONTRACTION_DELTA = 0.5;
+
+    /**
+     * The contraction criteria determines when the internal array will be
+     * contracted to fit the number of elements contained in the element
+     *  array + 1.
+     */
+    private double contractionCriterion = 2.5;
+
+    /**
+     * The expansion factor of the array.  When the array needs to be expanded,
+     * the new array size will be
+     * {@code internalArray.length * expansionFactor}
+     * if {@code expansionMode} is set to MULTIPLICATIVE_MODE, or
+     * {@code internalArray.length + expansionFactor} if
+     * {@code expansionMode} is set to ADDITIVE_MODE.
+     */
+    private double expansionFactor = 2.0;
+
+    /**
+     * Determines whether array expansion by {@code expansionFactor}
+     * is additive or multiplicative.
+     */
+    private ExpansionMode expansionMode = ExpansionMode.MULTIPLICATIVE;
+
+    /**
+     * The internal storage array.
+     */
+    private double[] internalArray;
+
+    /**
+     * The number of addressable elements in the array.  Note that this
+     * has nothing to do with the length of the internal storage array.
+     */
+    private int numElements = 0;
+
+    /**
+     * The position of the first addressable element in the internal storage
+     * array.  The addressable elements in the array are
+     * {@code internalArray[startIndex],...,internalArray[startIndex + numElements - 1]}.
+     */
+    private int startIndex = 0;
+
+    /**
+     * Specification of expansion algorithm.
+     * @since 3.1
+     */
+    public enum ExpansionMode {
+        /** Multiplicative expansion mode. */
+        MULTIPLICATIVE,
+        /** Additive expansion mode. */
+        ADDITIVE
+    }
+
+    /**
+     * Creates an instance with default properties.
+     * 
        
+     *  
      • {@code initialCapacity = 16}
        • 
+     *  
      • {@code expansionMode = MULTIPLICATIVE}
        • 
+     *  
      • {@code expansionFactor = 2.0}
        • 
+     *  
      • {@code contractionCriterion = 2.5}
        • 
+     * 
      
+     */
+    public ResizableDoubleArray() {
+        this(DEFAULT_INITIAL_CAPACITY);
+    }
+
+    /**
+     * Creates an instance with the specified initial capacity.
+     * Other properties take default values:
+     * 
        
+     *  
      • {@code expansionMode = MULTIPLICATIVE}
        • 
+     *  
      • {@code expansionFactor = 2.0}
        • 
+     *  
      • {@code contractionCriterion = 2.5}
        • 
+     * 
      
+     * @param initialCapacity Initial size of the internal storage array.
+     * @throws MathIllegalArgumentException if {@code initialCapacity <= 0}.
+     */
+    public ResizableDoubleArray(int initialCapacity)
+        throws MathIllegalArgumentException {
+        this(initialCapacity, DEFAULT_EXPANSION_FACTOR);
+    }
+
+    /**
+     * Creates an instance from an existing {@code double[]} with the
+     * initial capacity and numElements corresponding to the size of
+     * the supplied {@code double[]} array.
+     * If the supplied array is null, a new empty array with the default
+     * initial capacity will be created.
+     * The input array is copied, not referenced.
+     * Other properties take default values:
+     * 
        
+     *  
      • {@code initialCapacity = 16}
        • 
+     *  
      • {@code expansionMode = MULTIPLICATIVE}
        • 
+     *  
      • {@code expansionFactor = 2.0}
        • 
+     *  
      • {@code contractionCriterion = 2.5}
        • 
+     * 
      
+     *
+     * @param initialArray initial array
+     * @since 2.2
+     */
+    public ResizableDoubleArray(double[] initialArray) {
+        this(DEFAULT_INITIAL_CAPACITY,
+             DEFAULT_EXPANSION_FACTOR,
+             DEFAULT_CONTRACTION_DELTA + DEFAULT_EXPANSION_FACTOR,
+             ExpansionMode.MULTIPLICATIVE,
+             initialArray);
+    }
+
+    /**
+     * Creates an instance with the specified initial capacity
+     * and expansion factor.
+     * The remaining properties take default values:
+     * 
        
+     *  
      • {@code expansionMode = MULTIPLICATIVE}
        • 
+     *  
      • {@code contractionCriterion = 0.5 + expansionFactor}
        • 
+     * 
      
+     * 
      
+     * Throws IllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     *  
      • {@code initialCapacity > 0}
        • 
+     *  
      • {@code expansionFactor > 1}
        • 
+     * 
      
+     *
+     * @param initialCapacity Initial size of the internal storage array.
+     * @param expansionFactor The array will be expanded based on this
+     * parameter.
+     * @throws MathIllegalArgumentException if parameters are not valid.
+     * @deprecated As of 3.1. Please use
+     * {@link #ResizableDoubleArray(int,double)} instead.
+     */
+    @Deprecated
+    public ResizableDoubleArray(int initialCapacity,
+                                float expansionFactor)
+        throws MathIllegalArgumentException {
+        this(initialCapacity,
+             (double) expansionFactor);
+    }
+
+    /**
+     * Creates an instance with the specified initial capacity
+     * and expansion factor.
+     * The remaining properties take default values:
+     * 
        
+     *  
      • {@code expansionMode = MULTIPLICATIVE}
        • 
+     *  
      • {@code contractionCriterion = 0.5 + expansionFactor}
        • 
+     * 
      
+     * 
      
+     * Throws IllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     *  
      • {@code initialCapacity > 0}
        • 
+     *  
      • {@code expansionFactor > 1}
        • 
+     * 
      
+     *
+     * @param initialCapacity Initial size of the internal storage array.
+     * @param expansionFactor The array will be expanded based on this
+     * parameter.
+     * @throws MathIllegalArgumentException if parameters are not valid.
+     * @since 3.1
+     */
+    public ResizableDoubleArray(int initialCapacity,
+                                double expansionFactor)
+        throws MathIllegalArgumentException {
+        this(initialCapacity,
+             expansionFactor,
+             DEFAULT_CONTRACTION_DELTA + expansionFactor);
+    }
+
+    /**
+     * Creates an instance with the specified initialCapacity,
+     * expansionFactor, and contractionCriterion.
+     * The expansion mode will default to {@code MULTIPLICATIVE}.
+     * 
      
+     * Throws IllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     *  
      • {@code initialCapacity > 0}
        • 
+     *  
      • {@code expansionFactor > 1}
        • 
+     *  
      • {@code contractionCriterion >= expansionFactor}
        • 
+     * 
      
+     *
+     * @param initialCapacity Initial size of the internal storage array..
+     * @param expansionFactor The array will be expanded based on this
+     * parameter.
+     * @param contractionCriteria Contraction criteria.
+     * @throws MathIllegalArgumentException if parameters are not valid.
+     * @deprecated As of 3.1. Please use
+     * {@link #ResizableDoubleArray(int,double,double)} instead.
+     */
+    @Deprecated
+    public ResizableDoubleArray(int initialCapacity,
+                                float expansionFactor,
+                                float contractionCriteria)
+        throws MathIllegalArgumentException {
+        this(initialCapacity,
+             (double) expansionFactor,
+             (double) contractionCriteria);
+    }
+
+    /**
+     * Creates an instance with the specified initial capacity,
+     * expansion factor, and contraction criteria.
+     * The expansion mode will default to {@code MULTIPLICATIVE}.
+     * 
      
+     * Throws IllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     *  
      • {@code initialCapacity > 0}
        • 
+     *  
      • {@code expansionFactor > 1}
        • 
+     *  
      • {@code contractionCriterion >= expansionFactor}
        • 
+     * 
      
+     *
+     * @param initialCapacity Initial size of the internal storage array..
+     * @param expansionFactor The array will be expanded based on this
+     * parameter.
+     * @param contractionCriterion Contraction criterion.
+     * @throws MathIllegalArgumentException if the parameters are not valid.
+     * @since 3.1
+     */
+    public ResizableDoubleArray(int initialCapacity,
+                                double expansionFactor,
+                                double contractionCriterion)
+        throws MathIllegalArgumentException {
+        this(initialCapacity,
+             expansionFactor,
+             contractionCriterion,
+             ExpansionMode.MULTIPLICATIVE,
+             null);
+    }
+
+    /**
+     * 
      
+     * Create a ResizableArray with the specified properties.
      
+     * 
      
+     * Throws IllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     * 
      • initialCapacity > 0
        • 
+     * 
      • expansionFactor > 1
        • 
+     * 
      • contractionFactor >= expansionFactor
        • 
+     * 
      • expansionMode in {MULTIPLICATIVE_MODE, ADDITIVE_MODE}
+     * 
        • 
+     * 
      
+     *
+     * @param initialCapacity the initial size of the internal storage array
+     * @param expansionFactor the array will be expanded based on this
+     *                        parameter
+     * @param contractionCriteria the contraction Criteria
+     * @param expansionMode  the expansion mode
+     * @throws MathIllegalArgumentException if parameters are not valid
+     * @deprecated As of 3.1. Please use
+     * {@link #ResizableDoubleArray(int,double,double,ExpansionMode,double[])}
+     * instead.
+     */
+    @Deprecated
+    public ResizableDoubleArray(int initialCapacity, float expansionFactor,
+            float contractionCriteria, int expansionMode) throws MathIllegalArgumentException {
+        this(initialCapacity,
+             expansionFactor,
+             contractionCriteria,
+             expansionMode == ADDITIVE_MODE ?
+             ExpansionMode.ADDITIVE :
+             ExpansionMode.MULTIPLICATIVE,
+             null);
+        // XXX Just ot retain the expected failure in a unit test.
+        // With the new "enum", that test will become obsolete.
+        setExpansionMode(expansionMode);
+    }
+
+    /**
+     * Creates an instance with the specified properties.
+     * 
      
+     * Throws MathIllegalArgumentException if the following conditions are
+     * not met:
+     * 
        
+     *  
      • {@code initialCapacity > 0}
        • 
+     *  
      • {@code expansionFactor > 1}
        • 
+     *  
      • {@code contractionCriterion >= expansionFactor}
        • 
+     * 
      
+     *
+     * @param initialCapacity Initial size of the internal storage array.
+     * @param expansionFactor The array will be expanded based on this
+     * parameter.
+     * @param contractionCriterion Contraction criteria.
+     * @param expansionMode Expansion mode.
+     * @param data Initial contents of the array.
+     * @throws MathIllegalArgumentException if the parameters are not valid.
+     */
+    public ResizableDoubleArray(int initialCapacity,
+                                double expansionFactor,
+                                double contractionCriterion,
+                                ExpansionMode expansionMode,
+                                double ... data)
+        throws MathIllegalArgumentException {
+        if (initialCapacity <= 0) {
+            throw new NotStrictlyPositiveException(LocalizedFormats.INITIAL_CAPACITY_NOT_POSITIVE,
+                                                   initialCapacity);
+        }
+        checkContractExpand(contractionCriterion, expansionFactor);
+
+        this.expansionFactor = expansionFactor;
+        this.contractionCriterion = contractionCriterion;
+        this.expansionMode = expansionMode;
+        internalArray = new double[initialCapacity];
+        numElements = 0;
+        startIndex = 0;
+
+        if (data != null && data.length > 0) {
+            addElements(data);
+        }
+    }
+
+    /**
+     * Copy constructor.  Creates a new ResizableDoubleArray that is a deep,
+     * fresh copy of the original. Needs to acquire synchronization lock
+     * on original.  Original may not be null; otherwise a {@link NullArgumentException}
+     * is thrown.
+     *
+     * @param original array to copy
+     * @exception NullArgumentException if original is null
+     * @since 2.0
+     */
+    public ResizableDoubleArray(ResizableDoubleArray original)
+        throws NullArgumentException {
+        MathUtils.checkNotNull(original);
+        copy(original, this);
+    }
+
+    /**
+     * Adds an element to the end of this expandable array.
+     *
+     * @param value Value to be added to end of array.
+     */
+    public synchronized void addElement(double value) {
+        if (internalArray.length <= startIndex + numElements) {
+            expand();
+        }
+        internalArray[startIndex + numElements++] = value;
+    }
+
+    /**
+     * Adds several element to the end of this expandable array.
+     *
+     * @param values Values to be added to end of array.
+     * @since 2.2
+     */
+    public synchronized void addElements(double[] values) {
+        final double[] tempArray = new double[numElements + values.length + 1];
+        System.arraycopy(internalArray, startIndex, tempArray, 0, numElements);
+        System.arraycopy(values, 0, tempArray, numElements, values.length);
+        internalArray = tempArray;
+        startIndex = 0;
+        numElements += values.length;
+    }
+
+    /**
+     * 
      
+     * Adds an element to the end of the array and removes the first
+     * element in the array.  Returns the discarded first element.
+     * The effect is similar to a push operation in a FIFO queue.
+     * 
      
+     * 
      
+     * Example: If the array contains the elements 1, 2, 3, 4 (in that order)
+     * and addElementRolling(5) is invoked, the result is an array containing
+     * the entries 2, 3, 4, 5 and the value returned is 1.
+     * 
      
+     *
+     * @param value Value to be added to the array.
+     * @return the value which has been discarded or "pushed" out of the array
+     * by this rolling insert.
+     */
+    public synchronized double addElementRolling(double value) {
+        double discarded = internalArray[startIndex];
+
+        if ((startIndex + (numElements + 1)) > internalArray.length) {
+            expand();
+        }
+        // Increment the start index
+        startIndex += 1;
+
+        // Add the new value
+        internalArray[startIndex + (numElements - 1)] = value;
+
+        // Check the contraction criterion.
+        if (shouldContract()) {
+            contract();
+        }
+        return discarded;
+    }
+
+    /**
+     * Substitutes value for the most recently added value.
+     * Returns the value that has been replaced. If the array is empty (i.e.
+     * if {@link #numElements} is zero), an IllegalStateException is thrown.
+     *
+     * @param value New value to substitute for the most recently added value
+     * @return the value that has been replaced in the array.
+     * @throws MathIllegalStateException if the array is empty
+     * @since 2.0
+     */
+    public synchronized double substituteMostRecentElement(double value)
+        throws MathIllegalStateException {
+        if (numElements < 1) {
+            throw new MathIllegalStateException(
+                    LocalizedFormats.CANNOT_SUBSTITUTE_ELEMENT_FROM_EMPTY_ARRAY);
+        }
+
+        final int substIndex = startIndex + (numElements - 1);
+        final double discarded = internalArray[substIndex];
+
+        internalArray[substIndex] = value;
+
+        return discarded;
+    }
+
+    /**
+     * Checks the expansion factor and the contraction criterion and throws an
+     * IllegalArgumentException if the contractionCriteria is less than the
+     * expansionCriteria
+     *
+     * @param expansion factor to be checked
+     * @param contraction criteria to be checked
+     * @throws MathIllegalArgumentException if the contractionCriteria is less than
+     * the expansionCriteria.
+     * @deprecated As of 3.1. Please use
+     * {@link #checkContractExpand(double,double)} instead.
+     */
+    @Deprecated
+    protected void checkContractExpand(float contraction, float expansion)
+        throws MathIllegalArgumentException {
+        checkContractExpand((double) contraction,
+                            (double) expansion);
+    }
+
+    /**
+     * Checks the expansion factor and the contraction criterion and raises
+     * an exception if the contraction criterion is smaller than the
+     * expansion criterion.
+     *
+     * @param contraction Criterion to be checked.
+     * @param expansion Factor to be checked.
+     * @throws NumberIsTooSmallException if {@code contraction < expansion}.
+     * @throws NumberIsTooSmallException if {@code contraction <= 1}.
+     * @throws NumberIsTooSmallException if {@code expansion <= 1 }.
+     * @since 3.1
+     */
+    protected void checkContractExpand(double contraction,
+                                       double expansion)
+        throws NumberIsTooSmallException {
+        if (contraction < expansion) {
+            final NumberIsTooSmallException e = new NumberIsTooSmallException(contraction, 1, true);
+            e.getContext().addMessage(LocalizedFormats.CONTRACTION_CRITERIA_SMALLER_THAN_EXPANSION_FACTOR,
+                                      contraction, expansion);
+            throw e;
+        }
+
+        if (contraction <= 1) {
+            final NumberIsTooSmallException e = new NumberIsTooSmallException(contraction, 1, false);
+            e.getContext().addMessage(LocalizedFormats.CONTRACTION_CRITERIA_SMALLER_THAN_ONE,
+                                      contraction);
+            throw e;
+        }
+
+        if (expansion <= 1) {
+            final NumberIsTooSmallException e = new NumberIsTooSmallException(contraction, 1, false);
+            e.getContext().addMessage(LocalizedFormats.EXPANSION_FACTOR_SMALLER_THAN_ONE,
+                                      expansion);
+            throw e;
+        }
+    }
+
+    /**
+     * Clear the array contents, resetting the number of elements to zero.
+     */
+    public synchronized void clear() {
+        numElements = 0;
+        startIndex = 0;
+    }
+
+    /**
+     * Contracts the storage array to the (size of the element set) + 1 - to
+     * avoid a zero length array. This function also resets the startIndex to
+     * zero.
+     */
+    public synchronized void contract() {
+        final double[] tempArray = new double[numElements + 1];
+
+        // Copy and swap - copy only the element array from the src array.
+        System.arraycopy(internalArray, startIndex, tempArray, 0, numElements);
+        internalArray = tempArray;
+
+        // Reset the start index to zero
+        startIndex = 0;
+    }
+
+    /**
+     * Discards the i initial elements of the array.  For example,
+     * if the array contains the elements 1,2,3,4, invoking
+     * discardFrontElements(2) will cause the first two elements
+     * to be discarded, leaving 3,4 in the array.  Throws illegalArgumentException
+     * if i exceeds numElements.
+     *
+     * @param i  the number of elements to discard from the front of the array
+     * @throws MathIllegalArgumentException if i is greater than numElements.
+     * @since 2.0
+     */
+    public synchronized void discardFrontElements(int i)
+        throws MathIllegalArgumentException {
+        discardExtremeElements(i,true);
+    }
+
+    /**
+     * Discards the i last elements of the array.  For example,
+     * if the array contains the elements 1,2,3,4, invoking
+     * discardMostRecentElements(2) will cause the last two elements
+     * to be discarded, leaving 1,2 in the array.  Throws illegalArgumentException
+     * if i exceeds numElements.
+     *
+     * @param i  the number of elements to discard from the end of the array
+     * @throws MathIllegalArgumentException if i is greater than numElements.
+     * @since 2.0
+     */
+    public synchronized void discardMostRecentElements(int i)
+        throws MathIllegalArgumentException {
+        discardExtremeElements(i,false);
+    }
+
+    /**
+     * Discards the i first or last elements of the array,
+     * depending on the value of front.
+     * For example, if the array contains the elements 1,2,3,4, invoking
+     * discardExtremeElements(2,false) will cause the last two elements
+     * to be discarded, leaving 1,2 in the array.
+     * For example, if the array contains the elements 1,2,3,4, invoking
+     * discardExtremeElements(2,true) will cause the first two elements
+     * to be discarded, leaving 3,4 in the array.
+     * Throws illegalArgumentException
+     * if i exceeds numElements.
+     *
+     * @param i  the number of elements to discard from the front/end of the array
+     * @param front true if elements are to be discarded from the front
+     * of the array, false if elements are to be discarded from the end
+     * of the array
+     * @throws MathIllegalArgumentException if i is greater than numElements.
+     * @since 2.0
+     */
+    private synchronized void discardExtremeElements(int i,
+                                                     boolean front)
+        throws MathIllegalArgumentException {
+        if (i > numElements) {
+            throw new MathIllegalArgumentException(
+                    LocalizedFormats.TOO_MANY_ELEMENTS_TO_DISCARD_FROM_ARRAY,
+                    i, numElements);
+       } else if (i < 0) {
+           throw new MathIllegalArgumentException(
+                   LocalizedFormats.CANNOT_DISCARD_NEGATIVE_NUMBER_OF_ELEMENTS,
+                   i);
+        } else {
+            // "Subtract" this number of discarded from numElements
+            numElements -= i;
+            if (front) {
+                startIndex += i;
+            }
+        }
+        if (shouldContract()) {
+            contract();
+        }
+    }
+
+    /**
+     * Expands the internal storage array using the expansion factor.
+     * 
      
+     * if expansionMode is set to MULTIPLICATIVE_MODE,
+     * the new array size will be internalArray.length * expansionFactor.
+     * If expansionMode is set to ADDITIVE_MODE,  the length
+     * after expansion will be internalArray.length + expansionFactor
+     * 
      
+     */
+    protected synchronized void expand() {
+        // notice the use of FastMath.ceil(), this guarantees that we will always
+        // have an array of at least currentSize + 1.   Assume that the
+        // current initial capacity is 1 and the expansion factor
+        // is 1.000000000000000001.  The newly calculated size will be
+        // rounded up to 2 after the multiplication is performed.
+        int newSize = 0;
+        if (expansionMode == ExpansionMode.MULTIPLICATIVE) {
+            newSize = (int) FastMath.ceil(internalArray.length * expansionFactor);
+        } else {
+            newSize = (int) (internalArray.length + FastMath.round(expansionFactor));
+        }
+        final double[] tempArray = new double[newSize];
+
+        // Copy and swap
+        System.arraycopy(internalArray, 0, tempArray, 0, internalArray.length);
+        internalArray = tempArray;
+    }
+
+    /**
+     * Expands the internal storage array to the specified size.
+     *
+     * @param size Size of the new internal storage array.
+     */
+    private synchronized void expandTo(int size) {
+        final double[] tempArray = new double[size];
+        // Copy and swap
+        System.arraycopy(internalArray, 0, tempArray, 0, internalArray.length);
+        internalArray = tempArray;
+    }
+
+    /**
+     * The contraction criteria defines when the internal array will contract
+     * to store only the number of elements in the element array.
+     * If  the expansionMode is MULTIPLICATIVE_MODE,
+     * contraction is triggered when the ratio between storage array length
+     * and numElements exceeds contractionFactor.
+     * If the expansionMode is ADDITIVE_MODE, the
+     * number of excess storage locations is compared to
+     * contractionFactor.
+     *
+     * @return the contraction criteria used to reclaim memory.
+     * @deprecated As of 3.1. Please use {@link #getContractionCriterion()}
+     * instead.
+     */
+    @Deprecated
+    public float getContractionCriteria() {
+        return (float) getContractionCriterion();
+    }
+
+    /**
+     * The contraction criterion defines when the internal array will contract
+     * to store only the number of elements in the element array.
+     * If  the expansionMode is MULTIPLICATIVE_MODE,
+     * contraction is triggered when the ratio between storage array length
+     * and numElements exceeds contractionFactor.
+     * If the expansionMode is ADDITIVE_MODE, the
+     * number of excess storage locations is compared to
+     * contractionFactor.
+     *
+     * @return the contraction criterion used to reclaim memory.
+     * @since 3.1
+     */
+    public double getContractionCriterion() {
+        return contractionCriterion;
+    }
+
+    /**
+     * Returns the element at the specified index
+     *
+     * @param index index to fetch a value from
+     * @return value stored at the specified index
+     * @throws ArrayIndexOutOfBoundsException if index is less than
+     * zero or is greater than getNumElements() - 1.
+     */
+    public synchronized double getElement(int index) {
+        if (index >= numElements) {
+            throw new ArrayIndexOutOfBoundsException(index);
+        } else if (index >= 0) {
+            return internalArray[startIndex + index];
+        } else {
+            throw new ArrayIndexOutOfBoundsException(index);
+        }
+    }
+
+     /**
+     * Returns a double array containing the elements of this
+     * ResizableArray.  This method returns a copy, not a
+     * reference to the underlying array, so that changes made to the returned
+     *  array have no effect on this ResizableArray.
+     * @return the double array.
+     */
+    public synchronized double[] getElements() {
+        final double[] elementArray = new double[numElements];
+        System.arraycopy(internalArray, startIndex, elementArray, 0, numElements);
+        return elementArray;
+    }
+
+    /**
+     * The expansion factor controls the size of a new array when an array
+     * needs to be expanded.  The expansionMode
+     * determines whether the size of the array is multiplied by the
+     * expansionFactor (MULTIPLICATIVE_MODE) or if
+     * the expansion is additive (ADDITIVE_MODE -- expansionFactor
+     * storage locations added).  The default expansionMode is
+     * MULTIPLICATIVE_MODE and the default expansionFactor
+     * is 2.0.
+     *
+     * @return the expansion factor of this expandable double array
+     * @deprecated As of 3.1. Return type will be changed to "double" in 4.0.
+     */
+    @Deprecated
+    public float getExpansionFactor() {
+        return (float) expansionFactor;
+    }
+
+    /**
+     * The expansion mode determines whether the internal storage
+     * array grows additively or multiplicatively when it is expanded.
+     *
+     * @return the expansion mode.
+     * @deprecated As of 3.1. Return value to be changed to
+     * {@link ExpansionMode} in 4.0.
+     */
+    @Deprecated
+    public int getExpansionMode() {
+        synchronized (this) {
+            switch (expansionMode) {
+                case MULTIPLICATIVE:
+                    return MULTIPLICATIVE_MODE;
+                case ADDITIVE:
+                    return ADDITIVE_MODE;
+                default:
+                    throw new MathInternalError(); // Should never happen.
+            }
+        }
+    }
+
+    /**
+     * Notice the package scope on this method.   This method is simply here
+     * for the JUnit test, it allows us check if the expansion is working
+     * properly after a number of expansions.  This is not meant to be a part
+     * of the public interface of this class.
+     *
+     * @return the length of the internal storage array.
+     * @deprecated As of 3.1. Please use {@link #getCapacity()} instead.
+     */
+    @Deprecated
+    synchronized int getInternalLength() {
+        return internalArray.length;
+    }
+
+    /**
+     * Gets the currently allocated size of the internal data structure used
+     * for storing elements.
+     * This is not to be confused with {@link #getNumElements() the number of
+     * elements actually stored}.
+     *
+     * @return the length of the internal array.
+     * @since 3.1
+     */
+    public int getCapacity() {
+        return internalArray.length;
+    }
+
+    /**
+     * Returns the number of elements currently in the array.  Please note
+     * that this is different from the length of the internal storage array.
+     *
+     * @return the number of elements.
+     */
+    public synchronized int getNumElements() {
+        return numElements;
+    }
+
+    /**
+     * Returns the internal storage array.  Note that this method returns
+     * a reference to the internal storage array, not a copy, and to correctly
+     * address elements of the array, the startIndex is
+     * required (available via the {@link #start} method).  This method should
+     * only be used in cases where copying the internal array is not practical.
+     * The {@link #getElements} method should be used in all other cases.
+     *
+     *
+     * @return the internal storage array used by this object
+     * @since 2.0
+     * @deprecated As of 3.1.
+     */
+    @Deprecated
+    public synchronized double[] getInternalValues() {
+        return internalArray;
+    }
+
+    /**
+     * Provides direct access to the internal storage array.
+     * Please note that this method returns a reference to this object's
+     * storage array, not a copy.
+     * 
      
+     * To correctly address elements of the array, the "start index" is
+     * required (available via the {@link #getStartIndex() getStartIndex}
+     * method.
+     * 
      
+     * This method should only be used to avoid copying the internal array.
+     * The returned value must be used for reading only; other
+     * uses could lead to this object becoming inconsistent.
+     * 
      
+     * The {@link #getElements} method has no such limitation since it
+     * returns a copy of this array's addressable elements.
+     *
+     * @return the internal storage array used by this object.
+     * @since 3.1
+     */
+    protected double[] getArrayRef() {
+        return internalArray;
+    }
+
+    /**
+     * Returns the "start index" of the internal array.
+     * This index is the position of the first addressable element in the
+     * internal storage array.
+     * The addressable elements in the array are at indices contained in
+     * the interval [{@link #getStartIndex()},
+     *               {@link #getStartIndex()} + {@link #getNumElements()} - 1].
+     *
+     * @return the start index.
+     * @since 3.1
+     */
+    protected int getStartIndex() {
+        return startIndex;
+    }
+
+    /**
+     * Sets the contraction criteria.
+     *
+     * @param contractionCriteria contraction criteria
+     * @throws MathIllegalArgumentException if the contractionCriteria is less than
+     *         the expansionCriteria.
+     * @deprecated As of 3.1 (to be removed in 4.0 as field will become "final").
+     */
+    @Deprecated
+    public void setContractionCriteria(float contractionCriteria)
+        throws MathIllegalArgumentException {
+        checkContractExpand(contractionCriteria, getExpansionFactor());
+        synchronized(this) {
+            this.contractionCriterion = contractionCriteria;
+        }
+    }
+
+    /**
+     * Performs an operation on the addressable elements of the array.
+     *
+     * @param f Function to be applied on this array.
+     * @return the result.
+     * @since 3.1
+     */
+    public double compute(MathArrays.Function f) {
+        final double[] array;
+        final int start;
+        final int num;
+        synchronized(this) {
+            array = internalArray;
+            start = startIndex;
+            num   = numElements;
+        }
+        return f.evaluate(array, start, num);
+    }
+
+    /**
+     * Sets the element at the specified index.  If the specified index is greater than
+     * getNumElements() - 1, the numElements property
+     * is increased to index +1 and additional storage is allocated
+     * (if necessary) for the new element and all  (uninitialized) elements
+     * between the new element and the previous end of the array).
+     *
+     * @param index index to store a value in
+     * @param value value to store at the specified index
+     * @throws ArrayIndexOutOfBoundsException if {@code index < 0}.
+     */
+    public synchronized void setElement(int index, double value) {
+        if (index < 0) {
+            throw new ArrayIndexOutOfBoundsException(index);
+        }
+        if (index + 1 > numElements) {
+            numElements = index + 1;
+        }
+        if ((startIndex + index) >= internalArray.length) {
+            expandTo(startIndex + (index + 1));
+        }
+        internalArray[startIndex + index] = value;
+    }
+
+    /**
+     * Sets the expansionFactor.  Throws IllegalArgumentException if the
+     * the following conditions are not met:
+     * 
        
+     * 
      • expansionFactor > 1
        • 
+     * 
      • contractionFactor >= expansionFactor
        • 
+     * 
      
+     * @param expansionFactor the new expansion factor value.
+     * @throws MathIllegalArgumentException if expansionFactor is <= 1 or greater
+     * than contractionFactor
+     * @deprecated As of 3.1 (to be removed in 4.0 as field will become "final").
+     */
+    @Deprecated
+    public void setExpansionFactor(float expansionFactor) throws MathIllegalArgumentException {
+        checkContractExpand(getContractionCriterion(), expansionFactor);
+        // The check above verifies that the expansion factor is > 1.0;
+        synchronized(this) {
+            this.expansionFactor = expansionFactor;
+        }
+    }
+
+    /**
+     * Sets the expansionMode. The specified value must be one of
+     * ADDITIVE_MODE, MULTIPLICATIVE_MODE.
+     *
+     * @param expansionMode The expansionMode to set.
+     * @throws MathIllegalArgumentException if the specified mode value is not valid.
+     * @deprecated As of 3.1. Please use {@link #setExpansionMode(ExpansionMode)} instead.
+     */
+    @Deprecated
+    public void setExpansionMode(int expansionMode)
+        throws MathIllegalArgumentException {
+        if (expansionMode != MULTIPLICATIVE_MODE &&
+            expansionMode != ADDITIVE_MODE) {
+            throw new MathIllegalArgumentException(LocalizedFormats.UNSUPPORTED_EXPANSION_MODE, expansionMode,
+                                                   MULTIPLICATIVE_MODE, "MULTIPLICATIVE_MODE",
+                                                   ADDITIVE_MODE, "ADDITIVE_MODE");
+        }
+        synchronized(this) {
+            if (expansionMode == MULTIPLICATIVE_MODE) {
+                setExpansionMode(ExpansionMode.MULTIPLICATIVE);
+            } else if (expansionMode == ADDITIVE_MODE) {
+                setExpansionMode(ExpansionMode.ADDITIVE);
+            }
+        }
+    }
+
+    /**
+     * Sets the {@link ExpansionMode expansion mode}.
+     *
+     * @param expansionMode Expansion mode to use for resizing the array.
+     * @deprecated As of 3.1 (to be removed in 4.0 as field will become "final").
+     */
+    @Deprecated
+    public void setExpansionMode(ExpansionMode expansionMode) {
+        synchronized(this) {
+            this.expansionMode = expansionMode;
+        }
+    }
+
+    /**
+     * Sets the initial capacity.  Should only be invoked by constructors.
+     *
+     * @param initialCapacity of the array
+     * @throws MathIllegalArgumentException if initialCapacity is not
+     * positive.
+     * @deprecated As of 3.1, this is a no-op.
+     */
+    @Deprecated
+    protected void setInitialCapacity(int initialCapacity)
+        throws MathIllegalArgumentException {
+        // Body removed in 3.1.
+    }
+
+    /**
+     * This function allows you to control the number of elements contained
+     * in this array, and can be used to "throw out" the last n values in an
+     * array. This function will also expand the internal array as needed.
+     *
+     * @param i a new number of elements
+     * @throws MathIllegalArgumentException if i is negative.
+     */
+    public synchronized void setNumElements(int i)
+        throws MathIllegalArgumentException {
+        // If index is negative thrown an error.
+        if (i < 0) {
+            throw new MathIllegalArgumentException(
+                    LocalizedFormats.INDEX_NOT_POSITIVE,
+                    i);
+        }
+
+        // Test the new num elements, check to see if the array needs to be
+        // expanded to accommodate this new number of elements.
+        final int newSize = startIndex + i;
+        if (newSize > internalArray.length) {
+            expandTo(newSize);
+        }
+
+        // Set the new number of elements to new value.
+        numElements = i;
+    }
+
+    /**
+     * Returns true if the internal storage array has too many unused
+     * storage positions.
+     *
+     * @return true if array satisfies the contraction criteria
+     */
+    private synchronized boolean shouldContract() {
+        if (expansionMode == ExpansionMode.MULTIPLICATIVE) {
+            return (internalArray.length / ((float) numElements)) > contractionCriterion;
+        } else {
+            return (internalArray.length - numElements) > contractionCriterion;
+        }
+    }
+
+    /**
+     * Returns the starting index of the internal array.  The starting index is
+     * the position of the first addressable element in the internal storage
+     * array.  The addressable elements in the array are 
+     * internalArray[startIndex],...,internalArray[startIndex + numElements -1]
+     * 
+     *
+     * @return the starting index.
+     * @deprecated As of 3.1.
+     */
+    @Deprecated
+    public synchronized int start() {
+        return startIndex;
+    }
+
+    /**
+     * 
      Copies source to dest, copying the underlying data, so dest is
+     * a new, independent copy of source.  Does not contract before
+     * the copy.
      
+     *
+     * 
      Obtains synchronization locks on both source and dest
+     * (in that order) before performing the copy.
      
+     *
+     * 
      Neither source nor dest may be null; otherwise a {@link NullArgumentException}
+     * is thrown
      
+     *
+     * @param source ResizableDoubleArray to copy
+     * @param dest ResizableArray to replace with a copy of the source array
+     * @exception NullArgumentException if either source or dest is null
+     * @since 2.0
+     *
+     */
+    public static void copy(ResizableDoubleArray source,
+                            ResizableDoubleArray dest)
+        throws NullArgumentException {
+        MathUtils.checkNotNull(source);
+        MathUtils.checkNotNull(dest);
+        synchronized(source) {
+           synchronized(dest) {
+               dest.contractionCriterion = source.contractionCriterion;
+               dest.expansionFactor = source.expansionFactor;
+               dest.expansionMode = source.expansionMode;
+               dest.internalArray = new double[source.internalArray.length];
+               System.arraycopy(source.internalArray, 0, dest.internalArray,
+                       0, dest.internalArray.length);
+               dest.numElements = source.numElements;
+               dest.startIndex = source.startIndex;
+           }
+       }
+    }
+
+    /**
+     * Returns a copy of the ResizableDoubleArray.  Does not contract before
+     * the copy, so the returned object is an exact copy of this.
+     *
+     * @return a new ResizableDoubleArray with the same data and configuration
+     * properties as this
+     * @since 2.0
+     */
+    public synchronized ResizableDoubleArray copy() {
+        final ResizableDoubleArray result = new ResizableDoubleArray();
+        copy(this, result);
+        return result;
+    }
+
+    /**
+     * Returns true iff object is a ResizableDoubleArray with the same properties
+     * as this and an identical internal storage array.
+     *
+     * @param object object to be compared for equality with this
+     * @return true iff object is a ResizableDoubleArray with the same data and
+     * properties as this
+     * @since 2.0
+     */
+    @Override
+    public boolean equals(Object object) {
+        if (object == this ) {
+            return true;
+        }
+        if (object instanceof ResizableDoubleArray == false) {
+            return false;
+        }
+        synchronized(this) {
+            synchronized(object) {
+                boolean result = true;
+                final ResizableDoubleArray other = (ResizableDoubleArray) object;
+                result = result && (other.contractionCriterion == contractionCriterion);
+                result = result && (other.expansionFactor == expansionFactor);
+                result = result && (other.expansionMode == expansionMode);
+                result = result && (other.numElements == numElements);
+                result = result && (other.startIndex == startIndex);
+                if (!result) {
+                    return false;
+                } else {
+                    return Arrays.equals(internalArray, other.internalArray);
+                }
+            }
+        }
+    }
+
+    /**
+     * Returns a hash code consistent with equals.
+     *
+     * @return the hash code representing this {@code ResizableDoubleArray}.
+     * @since 2.0
+     */
+    @Override
+    public synchronized int hashCode() {
+        final int[] hashData = new int[6];
+        hashData[0] = Double.valueOf(expansionFactor).hashCode();
+        hashData[1] = Double.valueOf(contractionCriterion).hashCode();
+        hashData[2] = expansionMode.hashCode();
+        hashData[3] = Arrays.hashCode(internalArray);
+        hashData[4] = numElements;
+        hashData[5] = startIndex;
+        return Arrays.hashCode(hashData);
+    }
+
+}
diff --git a/java/unittests/infodynamics/utils/ChiSquareMeasurementDistributionTest.java b/java/unittests/infodynamics/utils/ChiSquareMeasurementDistributionTest.java
new file mode 100644
index 0000000..e01e262
--- /dev/null
+++ b/java/unittests/infodynamics/utils/ChiSquareMeasurementDistributionTest.java
@@ -0,0 +1,52 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.utils;
+
+import junit.framework.TestCase;
+
+/**
+ * Test functionality of the utility functions in ChiSquareMeasurementDistribution
+ * 
+ * @author Joseph Lizier.
+ *
+ */
+public class ChiSquareMeasurementDistributionTest extends TestCase {
+
+	public void testPValueAndEstimateCorrespondence() {
+		double[] actualValues = new double[] {0.000001, 0.000002, 0.000005, 0.00001, 0.00002, 0.00005,
+					0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5};
+		int[] numObs = new int[] {100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000};
+		int[] degFreedom = new int[] {1, 2, 3, 5, 10, 20};
+		for (int a = 0; a < actualValues.length; a++) {
+			for (int d = 0; d < degFreedom.length; d++) {
+				for (int n = 0; n < numObs.length; n++) {
+					ChiSquareMeasurementDistribution csmDist = new ChiSquareMeasurementDistribution(actualValues[a], numObs[n], degFreedom[d]);
+					System.out.printf("N=%d, degFree=%d, measuredValue=%.6f: pValue=%.6f\n", numObs[n], degFreedom[d], actualValues[a], csmDist.pValue);
+					// Check that we get the same p value if we stick this estimate in afterwards:
+					assertEquals(csmDist.pValue, csmDist.computePValueForGivenEstimate(actualValues[a]), 0.000001);
+					// Check that if we ask for an inverse on this pValue we get the same estimate back:
+					// Can easily get rounding errors here when the pValue approaches 1 or 0, so we won't test these for now:
+					if ((csmDist.pValue < 0.99) && (csmDist.pValue > 0.000001)) {
+						assertEquals(actualValues[a], csmDist.computeEstimateForGivenPValue(csmDist.pValue), 0.000001);
+					}
+				}
+			}
+		}
+	}
+}
diff --git a/java/unittests/infodynamics/utils/MathsUtilsTest.java b/java/unittests/infodynamics/utils/MathsUtilsTest.java
index e114860..6019240 100755
--- a/java/unittests/infodynamics/utils/MathsUtilsTest.java
+++ b/java/unittests/infodynamics/utils/MathsUtilsTest.java
@@ -153,6 +153,74 @@ public class MathsUtilsTest extends TestCase {
 		}
 	}
 	
+	/** 
+	 * Confirm that our chiinv() function is correct to 5 decimal places
+	 */
+	public void testChiInv() {
+		// Compare against the results supplied by octave
+		
+		// First for 1 degree of freedom
+		double[] octaveResults1degFree = {
+				0.00000, 0.00063, 0.00252, 0.00567, 0.01009, 0.01579, 0.02279, 0.03111,
+				0.04076, 0.05178, 0.06418, 0.07802, 0.09332, 0.11013, 0.12849, 0.14847,
+				0.17013, 0.19352, 0.21874, 0.24587, 0.27500, 0.30623, 0.33970, 0.37554,
+				0.41389, 0.45494, 0.49886, 0.54589, 0.59628, 0.65032, 0.70833, 0.77070,
+				0.83789, 0.91043, 0.98895, 1.07419, 1.16709, 1.26876, 1.38059, 1.50437,
+				1.64237, 1.79762, 1.97423, 2.17796, 2.41732, 2.70554, 3.06490, 3.53738,
+				4.21788, 5.41189
+		};
+		for (int n=0; n<50; n++) {
+			assertEquals(octaveResults1degFree[n],
+					MathsUtils.chiSquareInv(n*0.02, 1), 0.00001);
+		}		
+
+		// Then for 2 degrees of freedom
+		double[] octaveResults2degFree = {
+				0.00000, 0.04041, 0.08164, 0.12375, 0.16676, 0.21072, 0.25567, 0.30165,
+				0.34871, 0.39690, 0.44629, 0.49692, 0.54887, 0.60221, 0.65701, 0.71335,
+				0.77132, 0.83103, 0.89257, 0.95607, 1.02165, 1.08945, 1.15964, 1.23237,
+				1.30785, 1.38629, 1.46794, 1.55306, 1.64196, 1.73500, 1.83258, 1.93517,
+				2.04330, 2.15762, 2.27887, 2.40795, 2.54593, 2.69415, 2.85423, 3.02826,
+				3.21888, 3.42960, 3.66516, 3.93223, 4.24053, 4.60517, 5.05146, 5.62682,
+				6.43775, 7.82405
+		};
+		for (int n=0; n<50; n++) {
+			assertEquals(octaveResults2degFree[n],
+					MathsUtils.chiSquareInv(n*0.02, 2), 0.00001);
+		}		
+
+		// Then for 3 degrees of freedom
+		double[] octaveResults3degFree = {
+				0.00000, 0.18483, 0.30015, 0.40117, 0.49495, 0.58437, 0.67101, 0.75583,
+				0.83949, 0.92248, 1.00517, 1.08788, 1.17087, 1.25435, 1.33855, 1.42365,
+				1.50984, 1.59731, 1.68623, 1.77678, 1.86917, 1.96358, 2.06023, 2.15935,
+				2.26117, 2.36597, 2.47404, 2.58571, 2.70134, 2.82133, 2.94617, 3.07637,
+				3.21255, 3.35544, 3.50588, 3.66487, 3.83362, 4.01359, 4.20662, 4.41498,
+				4.64163, 4.89041, 5.16655, 5.47734, 5.83346, 6.25139, 6.75869, 7.40688,
+				8.31117, 9.83741
+		};
+		for (int n=0; n<50; n++) {
+			assertEquals(octaveResults3degFree[n],
+					MathsUtils.chiSquareInv(n*0.02, 3), 0.00001);
+		}		
+
+		// Then for 10 degrees of freedom
+		double[] octaveResults10degFree = {
+				0.00000,  3.05905,  3.69654,  4.15672,  4.53505,  4.86518,  5.16338,
+				5.43890,  5.69757,  5.94335,  6.17908,  6.40688,  6.62838,  6.84492,
+				7.05756,  7.26722,  7.47468,  7.68063,  7.88571,  8.09047,  8.29547,
+				8.50122,  8.70822,  8.91698,  9.12801,  9.34182,  9.55896,  9.78002,
+				10.00562, 10.23644, 10.47324, 10.71684, 10.96821, 11.22844, 11.49878,
+				11.78072, 12.07604, 12.38685, 12.71578, 13.06609, 13.44196, 13.84881,
+				14.29397, 14.78759, 15.34443, 15.98718, 16.75348, 17.71312, 19.02074,
+				21.16077
+		};
+		for (int n=0; n<50; n++) {
+			assertEquals(octaveResults10degFree[n],
+					MathsUtils.chiSquareInv(n*0.02, 10), 0.00001);
+		}		
+	}
+	
 	public void testNormalPdf() throws Exception {
 		// Check values for x = -4:0.1:4 generated by octave
 		double[] expectedPdfMu0Std1 = {0.00013, 0.00020, 0.00029, 0.00042,

From e25364b84c3886bb891a9c5ae583a2c1b633b6b5 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 16 Jun 2017 13:38:11 +1000
Subject: [PATCH 09/78] Added computeSignificance() to AIS discrete calculator

---
 .../ActiveInformationCalculatorDiscrete.java  | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
index 2987e94..665226e 100755
--- a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
@@ -18,6 +18,9 @@
 
 package infodynamics.measures.discrete;
 
+import infodynamics.utils.AnalyticMeasurementDistribution;
+import infodynamics.utils.AnalyticNullDistributionComputer;
+import infodynamics.utils.ChiSquareMeasurementDistribution;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.RandomGenerator;
@@ -68,8 +71,11 @@ import infodynamics.utils.RandomGenerator;
  * @author Joseph Lizier (email,
  * www)
  */
-public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscreteInContextOfPastCalculator {
+public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscreteInContextOfPastCalculator
+	implements AnalyticNullDistributionComputer {
 
+	protected boolean aisComputed = false;
+	
 	/**
 	 * User was formerly forced to create new instances through this factory method.
 	 * Retained for backwards compatibility.
@@ -95,6 +101,12 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 		super(base, history);
 	}
 
+	@Override
+	public void initialise() {
+		super.initialise();
+		aisComputed = false;
+	}
+
 	@Override
 	public void addObservations(int states[]) {
 		int timeSteps = states.length;
@@ -297,6 +309,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 		}
 		
 		average = mi;
+		aisComputed = true;
 		return mi;
 	}
 	
@@ -721,6 +734,17 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 		return measDistribution;
 	}
 
+	@Override
+	public AnalyticMeasurementDistribution computeSignificance()
+			throws Exception {
+		if (!aisComputed) {
+			computeAverageLocalOfObservations();
+		}
+		return new ChiSquareMeasurementDistribution(average,
+				observations,
+				(base - 1) * (base_power_k - 1));
+	}
+
 	/**
 	 * Debug method to write the current probability distribution functions 
 	 *  

From bd94172d60df0be34dcc5acfef5b1e0842bd30fe Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 16 Jun 2017 13:38:45 +1000
Subject: [PATCH 10/78] Removed setting the miComputed flag to true in a
 computeLocal method, since this could be called with new observations.

---
 .../measures/discrete/MutualInformationCalculatorDiscrete.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
index d7a5c19..1613125 100755
--- a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
@@ -387,7 +387,6 @@ public class MutualInformationCalculatorDiscrete extends InfoMeasureCalculatorDi
 			}
 		}
 		average = average/(double) observations;
-		miComputed = true;
 
 		return localMI;
 	}

From 550bbae5c1ed239d1a30b6562c3fd4fe4cd04770 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 16 Jun 2017 13:39:43 +1000
Subject: [PATCH 11/78] Patched computeSignificance() for discrete TE
 calculator, since this was not returning the correct degrees of freedom for
 the Chi square distribution (it multiplied the history lengths by the base,
 instead of raising them to the power)

---
 .../measures/discrete/TransferEntropyCalculatorDiscrete.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
index 0697d96..52c8382 100755
--- a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
@@ -1289,7 +1289,7 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 		}
 		return new ChiSquareMeasurementDistribution(average,
 				observations,
-				(sourceHistoryEmbedLength*base - 1)*(base - 1)*(k*base));
+				(base_power_l - 1)*(base - 1)*(base_power_k));
 	}
 
 	/**

From 7b0f8b3d340e78170b618e9c288921f9edd9b1d1 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 8 Aug 2017 11:25:02 +1000
Subject: [PATCH 12/78] Making all demo data files readable by Matlab (changing
 comment character to a % rather than #)

---
 demos/data/2coupledBinaryColsUseK2.txt | 2 +-
 demos/data/2randomCols-1.txt           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/demos/data/2coupledBinaryColsUseK2.txt b/demos/data/2coupledBinaryColsUseK2.txt
index 54876c6..fc5f658 100644
--- a/demos/data/2coupledBinaryColsUseK2.txt
+++ b/demos/data/2coupledBinaryColsUseK2.txt
@@ -1,4 +1,4 @@
-# First column is random; Second column is parity of its two previous values and previous value of first column
+% First column is random; Second column is parity of its two previous values and previous value of first column
  1 0
  0 1
  1 1
diff --git a/demos/data/2randomCols-1.txt b/demos/data/2randomCols-1.txt
index 3a41807..b19e244 100755
--- a/demos/data/2randomCols-1.txt
+++ b/demos/data/2randomCols-1.txt
@@ -1,4 +1,4 @@
-# Two random columns of data in [0..1) each
+% Two random columns of data in [0..1) each
  2.97438473e-01 1.58412125e-01
  3.72029104e-02 7.49930621e-01
  4.30819357e-01 4.23417397e-01

From 26137d12b1019891e419cdb809d34906bc309217 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Tue, 15 Aug 2017 20:46:07 +0100
Subject: [PATCH 13/78] Added precomputed digammas in KSG mixed calculator,
 fixed bug in locals.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 45 +++++++++++++++----
 1 file changed, 37 insertions(+), 8 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 8260376..f643aec 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -79,9 +79,30 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * Number of dimenions of the joint continuous variable
 	 */
 	protected int dimensions;
+  /**
+   * Number of observations added to the calculator
+   */
+  protected int totalObservations;
+  /**
+   * Whether to print debug information
+   */
 	protected boolean debug;
+  /**
+   * Last computed value of MI
+   */
 	protected double mi;
+  /**
+   * Whether MI has been computed with the current observations
+   */
 	protected boolean miComputed;
+  /**
+   * Saved digammaN for average and local computations
+   */
+  protected double digammaN;
+  /**
+   * Saved digammaK for average and local computations
+   */
+  protected double digammaK;
 	
 	protected EuclideanUtils normCalculator;
 	// Storage for the norms from each observation to each other one
@@ -122,6 +143,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	public void initialise(int dimensions, int base) {
 		mi = 0.0;
 		miComputed = false;
+    totalObservations = 0;
 		xNorms = null;
 		continuousData = null;
 		means = null;
@@ -194,8 +216,11 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		if (continuousObservations[0].length != dimensions) {
 			throw new Exception("The continuous observations do not have the expected number of variables (" + dimensions + ")");
 		}
+
 		continuousData = continuousObservations;
 		discreteData = discreteObservations;
+    totalObservations = discreteObservations.length;
+
 		if (normalise) {
 			// Take a copy since we're going to normalise it
 			// And we need to keep the means/stds ready to normalise local values
@@ -204,6 +229,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			stds = MatrixUtils.stdDevs(continuousObservations, means);
 			continuousData = MatrixUtils.normaliseIntoNewArray(continuousObservations, means, stds);
 		}
+
 		// count the discrete states:
 		counts = new int[base];
 		for (int t = 0; t < discreteData.length; t++) {
@@ -214,6 +240,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 				throw new RuntimeException("This implementation assumes there are at least k items in each discrete bin");
 			}
 		}
+
+    digammaN = MathsUtils.digamma(totalObservations);
+    digammaK = MathsUtils.digamma(k);
 	}
 
 	/**
@@ -308,7 +337,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		//  for an entropy over m subspaces.
 		// mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
 		// Instead do:
-		mi = MathsUtils.digamma(k) - averageDiGammas + MathsUtils.digamma(N);
+		mi = digammaK + digammaN - averageDiGammas;
 		miComputed = true;
 		return mi;
 	}
@@ -364,7 +393,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 				//  for an entropy over m subspaces.
 				// double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
 				// Instead do:
-				double localValue = MathsUtils.digamma(k) - localSum + MathsUtils.digamma(N);
+				double localValue = digammaK + digammaN - localSum;
 				testSum += localValue;
 				if (dimensions == 1) {
 					System.out.printf("t=%d: x=%.3f, eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
@@ -383,7 +412,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 					avNx, MathsUtils.digamma((int) avNx), MathsUtils.digamma((int) avNx - 1), avNy, MathsUtils.digamma((int) avNy)));
 			System.out.printf("Independent average num in joint box is %.3f\n", (avNx * avNy / (double) N));
 			System.out.println(String.format("digamma(k)=%.3f - averageDiGammas=%.3f + digamma(N)=%.3f\n",
-					MathsUtils.digamma(k), averageDiGammas, MathsUtils.digamma(N)));
+					digammaK, averageDiGammas, digammaN));
 		}
 		
 		// Don't need the 1/k correction here because the conditional entropy term
@@ -391,7 +420,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		//  for an entropy over m subspaces.
 		// mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
 		// Instead, do:
-		mi = MathsUtils.digamma(k) - averageDiGammas + MathsUtils.digamma(N);
+		mi = digammaK + digammaN - averageDiGammas;
 		miComputed = true;
 		return mi;
 	}
@@ -455,7 +484,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 				//  for an entropy over m subspaces.
 				// double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
 				// Instead do:
-				double localValue = MathsUtils.digamma(k) - localSum + MathsUtils.digamma(N);
+				double localValue = digammaK + digammaN - localSum;
 				testSum += localValue;
 				if (dimensions == 1) {
 					System.out.printf("t=%d: x=%.3f, eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
@@ -478,7 +507,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		//  for an entropy over m subspaces.
 		// mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
 		// Instead do:
-		mi = MathsUtils.digamma(k) - averageDiGammas + MathsUtils.digamma(N);
+		mi = digammaK + digammaN - averageDiGammas;
 		miComputed = true;
 		return mi;
 	}
@@ -597,11 +626,11 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		// double fixedPartOfLocals = MathsUtils.digamma(k) - 1.0/(double)k +
 		//							MathsUtils.digamma(N);
 		// Instead do:
-		double fixedPartOfLocals = MathsUtils.digamma(k) + MathsUtils.digamma(N_samplesForPdfs); // Use N_samplesForPdfs here because that's what would be in denominator of probability functions
+		double fixedPartOfLocals = digammaK + digammaN; // Use N_samplesForPdfs here because that's what would be in denominator of probability functions
 		double testSum = 0.0;
 		if (debug) {
 			System.out.printf("digamma(k)=%.3f + digamma(N)=%.3f\n",
-					MathsUtils.digamma(k), MathsUtils.digamma(N_samplesForPdfs));
+					digammaK, digammaN);
 		}
 		double avNx = 0;
 		double avNy = 0;

From abeb15cd6e433399ffbebfd3236bf1115f50ebf6 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Tue, 15 Aug 2017 21:25:03 +0100
Subject: [PATCH 14/78] Added standard start/finaliseAddObservations to mixed
 KSG calc.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 107 +++++++++++++-----
 1 file changed, 80 insertions(+), 27 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index f643aec..455fb75 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -26,6 +26,9 @@ import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
 
+import java.util.Iterator;
+import java.util.Vector;
+
 /**
  * 
      Compute the Mutual Information between a vector of continuous variables and discrete
  *  variable using the Kraskov estimation method.
      
@@ -103,6 +106,16 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
    * Saved digammaK for average and local computations
    */
   protected double digammaK;
+	/**
+	 * Storage for continuous observations supplied via {@link #addObservations(double[][], int[])}
+	 * type calls
+	 */
+	protected Vector vectorOfContinuousObservations;
+	/**
+	 * Storage for discrete observations supplied via {@link #addObservations(double[][], int[])}
+	 * type calls
+	 */
+	protected Vector vectorOfDiscreteObservations;
 	
 	protected EuclideanUtils normCalculator;
 	// Storage for the norms from each observation to each other one
@@ -128,6 +141,14 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * Track the std devs of the joint variables if we are normalising them
 	 */
 	protected double[] stds;
+	/**
+	 * Time difference from the source to the destination observations
+	 *  (ie destination lags the source by this time:
+	 *  	we compute I(source_{n}; dest_{n+timeDiff}).
+	 * (Note that our internal sourceObservations and destObservations
+	 *  are adjusted so that there is no timeDiff between them).
+	 */
+	protected int timeDiff = 0;
 
 	public MutualInfoCalculatorMultiVariateWithDiscreteKraskov() {
 		super();
@@ -179,8 +200,22 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		}
 	}
 
-	public void addObservations(double[][] source, double[][] destination) throws Exception {
-		throw new RuntimeException("Not implemented yet");
+	public void addObservations(double[][] continuousObservations,
+      int[] discreteObservations) throws Exception {
+		if (vectorOfContinuousObservations == null) {
+			// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if (continuousObservations.length != discreteObservations.length) {
+			throw new Exception("Time steps for observations2 " +
+					discreteObservations.length + " does not match the length " +
+					"of observations1 " + continuousObservations.length);
+		}
+		if (continuousObservations[0].length != dimensions) {
+			throw new Exception("The continuous observations do not have the expected number of variables (" + dimensions + ")");
+		}
+		vectorOfContinuousObservations.add(continuousObservations);
+		vectorOfDiscreteObservations.add(discreteObservations);
 	}
 
 	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {
@@ -196,38 +231,47 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	}
 
 	public void startAddObservations() {
-		throw new RuntimeException("Not implemented yet");
+		vectorOfContinuousObservations = new Vector();
+		vectorOfDiscreteObservations   = new Vector();
 	}
 
-	public void finaliseAddObservations() {
-		throw new RuntimeException("Not implemented yet");
-	}
-
-	public void setObservations(double[][] continuousObservations,
-			int[] discreteObservations) throws Exception {
-		if (continuousObservations.length != discreteObservations.length) {
-			throw new Exception("Time steps for observations2 " +
-					discreteObservations.length + " does not match the length " +
-					"of observations1 " + continuousObservations.length);
-		}
-		if (continuousObservations[0].length == 0) {
-			throw new Exception("Computing MI with a null set of data");
-		}
-		if (continuousObservations[0].length != dimensions) {
-			throw new Exception("The continuous observations do not have the expected number of variables (" + dimensions + ")");
+	public void finaliseAddObservations() throws Exception {
+		if (vectorOfContinuousObservations.size() < 1) {
+			throw new Exception("Cannot compute MI with a null set of data");
 		}
 
-		continuousData = continuousObservations;
-		discreteData = discreteObservations;
-    totalObservations = discreteObservations.length;
+		// First work out the size to allocate the joint vectors, and do the allocation:
+		totalObservations = 0;
+		for (double[][] destination : vectorOfContinuousObservations) {
+			totalObservations += destination.length - timeDiff;
+		}
+		continuousData = new double[totalObservations][dimensions];
+		discreteData   = new int[totalObservations];
+		
+		// Construct the joint vectors from the given observations
+		//  (removing redundant data which is outside any timeDiff)
+		int startObservation = 0;
+		Iterator iterator = vectorOfContinuousObservations.iterator();
+		for (int[] dct : vectorOfDiscreteObservations) {
+			double[][] cnt = iterator.next();
+			// Copy the data from these given observations into our master 
+			//  array, aligning them incorporating the timeDiff:
+			MatrixUtils.arrayCopy(cnt, 0, 0,
+					continuousData, startObservation, 0,
+					cnt.length - timeDiff, dimensions);
+      System.arraycopy(dct, timeDiff, discreteData, startObservation, dct.length - timeDiff);
+			startObservation += cnt.length - timeDiff;
+		}
+		// We don't need to keep the vectors of observation sets anymore:
+		vectorOfContinuousObservations = null;
+		vectorOfDiscreteObservations   = null;
 
 		if (normalise) {
-			// Take a copy since we're going to normalise it
-			// And we need to keep the means/stds ready to normalise local values
+			// We need to keep the means/stds ready to normalise local values
 			//  that are supplied later:
-			means = MatrixUtils.means(continuousObservations);
-			stds = MatrixUtils.stdDevs(continuousObservations, means);
-			continuousData = MatrixUtils.normaliseIntoNewArray(continuousObservations, means, stds);
+			means = MatrixUtils.means(continuousData);
+			stds = MatrixUtils.stdDevs(continuousData, means);
+			MatrixUtils.normaliseIntoNewArray(continuousData, means, stds);
 		}
 
 		// count the discrete states:
@@ -245,6 +289,15 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
     digammaK = MathsUtils.digamma(k);
 	}
 
+
+	public void setObservations(double[][] continuousObservations,
+			int[] discreteObservations) throws Exception {
+    startAddObservations();
+    addObservations(continuousObservations, discreteObservations);
+    finaliseAddObservations();
+	}
+
+
 	/**
 	 * Compute the norms for each marginal time series
 	 *

From 39ad14d855fba8d152fa04118b5b8db35290efd9 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Tue, 15 Aug 2017 21:41:52 +0100
Subject: [PATCH 15/78] Added function to set timeDiff property.

---
 ...MutualInfoCalculatorMultiVariateWithDiscrete.java |  3 ++-
 ...nfoCalculatorMultiVariateWithDiscreteKraskov.java | 12 +++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/MutualInfoCalculatorMultiVariateWithDiscrete.java b/java/source/infodynamics/measures/mixed/MutualInfoCalculatorMultiVariateWithDiscrete.java
index 424647a..3a54114 100755
--- a/java/source/infodynamics/measures/mixed/MutualInfoCalculatorMultiVariateWithDiscrete.java
+++ b/java/source/infodynamics/measures/mixed/MutualInfoCalculatorMultiVariateWithDiscrete.java
@@ -77,7 +77,8 @@ public interface MutualInfoCalculatorMultiVariateWithDiscrete {
 	 * @param propertyName name of property
 	 * @param propertyValue value of property
 	 */
-	public void setProperty(String propertyName, String propertyValue);
+	public void setProperty(String propertyName, String propertyValue)
+      throws Exception;
 	
 	/**
 	 * Set the properties from which the mutual information should be computed.
diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 455fb75..d271cc5 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -128,6 +128,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	public final static String PROP_K = "k";
 	public final static String PROP_NORM_TYPE = "NORM_TYPE";	
 	public static final String PROP_NORMALISE = "NORMALISE";
+	public static final String PROP_TIME_DIFF = "TIME_DIFF";
 	
 	/**
 	 * Track whether we're going to normalise the joint variables individually
@@ -185,18 +186,27 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * 		default is {@link EuclideanUtils#NORM_MAX_NORM}.
 	 *  
    8. {@link #PROP_NORMALISE} - whether to normalise the individual
 	 *      variables (true by default)
      9. 
+	 * 	
    9. {@link #PROP_TIME_DIFF} - Time difference between source and
+   *     	destination (0 by default). Must be >= 0.
      10. 
 	 * 
 	 * 
 	 * @param propertyName name of the property to set
 	 * @param propertyValue value to set on that property
 	 */
-	public void setProperty(String propertyName, String propertyValue) {
+	public void setProperty(String propertyName, String propertyValue)
+      throws Exception {
 		if (propertyName.equalsIgnoreCase(PROP_K)) {
 			k = Integer.parseInt(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
 			normCalculator.setNormToUse(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
 			normalise = Boolean.parseBoolean(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
+			int val = Integer.parseInt(propertyValue);
+			if (val < 0) {
+				throw new Exception("Time difference must be >= 0. Flip data1 and data2 around if required.");
+			}
+      timeDiff = val;
 		}
 	}
 

From 437e606233680236c00081f1cd16ff116e8323d2 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 00:07:51 +0100
Subject: [PATCH 16/78] getNumObservations in KSG mixed now returns
 totalObservations.

Before it returned continuousData.length, which more more prone
to null pointer errors, and therefore riskier.
---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index d271cc5..38c983c 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -755,6 +755,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	}
 	
 	public int getNumObservations() {
-		return continuousData.length;
+		return totalObservations;
 	}
 }

From dcd6314082c7e3d92856fec28af018d887ef63a9 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 00:26:53 +0100
Subject: [PATCH 17/78] Add a few unit tests for KSG mixed calculator.

---
 ...MultiVariateWithDiscreteKraskovTester.java | 159 ++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100755 java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
new file mode 100755
index 0000000..4de42ed
--- /dev/null
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -0,0 +1,159 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.measures.mixed.kraskov;
+
+import junit.framework.TestCase;
+import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.RandomGenerator;
+import infodynamics.utils.MatrixUtils;
+
+public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
+
+  int DATA_LENGTH = 100;
+  int TEST_DIMENSIONS = 2;
+  int TEST_BASE = 2;
+
+  public void testSetObservationsDataIntegrity() throws Exception { 
+    
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+    
+    // Generate proper and incorrect data sets
+    RandomGenerator rg = new RandomGenerator();
+    double[][] contDataTooManyVariables = rg.generateNormalData(DATA_LENGTH,
+        TEST_DIMENSIONS + 1, 0, 1);
+    double[][] contDataCorrect = rg.generateNormalData(DATA_LENGTH, TEST_DIMENSIONS,
+        0, 1);
+    int[] discDataCorrect = rg.generateRandomInts(DATA_LENGTH, TEST_BASE);
+    int[] discDataValuesOutsideRange = rg.generateRandomInts(100, TEST_BASE + 1);
+    int[] discDataWrongLength = rg.generateRandomInts(DATA_LENGTH - 1, TEST_BASE);
+
+    // Check that we catch continuous data which doesn't have
+    //  the right number of variables
+    boolean caughtException = false;
+    try {
+      miCalc.setObservations(contDataTooManyVariables, discDataCorrect);
+    } catch (Exception e) {
+      caughtException = true;
+    }
+    assertTrue(caughtException);
+    
+    // Check that we catch discrete data outside the range:
+    caughtException = false;
+    try {
+      miCalc.setObservations(contDataCorrect, discDataValuesOutsideRange);
+    } catch (Exception e) {
+      caughtException = true;
+    }
+    assertTrue(caughtException);
+    
+    // Check that we catch mismatched data lengths:
+    caughtException = false;
+    try {
+      miCalc.setObservations(contDataCorrect, discDataWrongLength);
+    } catch (Exception e) {
+      caughtException = true;
+    }
+    assertTrue(caughtException);
+
+    // No observations should have been set yet
+    assertEquals(0, miCalc.getNumObservations());
+    
+    // Check that no exception is thrown for ok data:
+    caughtException = false;
+    try {
+      miCalc.setObservations(contDataCorrect, discDataCorrect);
+    } catch (Exception e) {
+      caughtException = true;
+      e.printStackTrace();
+    }
+    assertFalse(caughtException);
+    
+    // and that the observations have now been set:
+    assertEquals(DATA_LENGTH, miCalc.getNumObservations());
+  }
+    
+  public void testComputeSignificanceDoesntAlterAverage() throws Exception {
+    
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+    
+    // Generate proper data sets
+    RandomGenerator rg = new RandomGenerator();
+    double[][] contDataCorrect = rg.generateNormalData(DATA_LENGTH, TEST_DIMENSIONS,
+        0, 1);
+    int[] discDataCorrect = rg.generateRandomInts(DATA_LENGTH, TEST_BASE);
+
+    miCalc.setObservations(contDataCorrect, discDataCorrect);
+
+    double mi = miCalc.computeAverageLocalOfObservations();
+    
+    System.out.printf("Average was %.5f\n", mi);
+    
+    EmpiricalMeasurementDistribution measDist =
+        miCalc.computeSignificance(100);
+    
+    System.out.printf("pValue of sig test was %.3f\n", measDist.pValue);
+    
+    // And compute the average value again to check that it's consistent:
+    for (int i = 0; i < 10; i++) {
+      double averageCheck1 = miCalc.computeAverageLocalOfObservations();
+      assertEquals(mi, averageCheck1);
+    }
+  }
+  
+  public void testObservationsRequiredBeforeStatTest() throws Exception {
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+
+    boolean caughtException = false;
+    try {
+      miCalc.computeSignificance(100);
+    } catch (Exception e) {
+      caughtException = true;
+    }
+    assertTrue(caughtException);
+  }
+
+  public void testLocals() throws Exception {
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+
+    // Generate proper data sets
+    RandomGenerator rg = new RandomGenerator();
+    double[][] contData = rg.generateNormalData(DATA_LENGTH, TEST_DIMENSIONS,
+        0, 1);
+    int[] discData = rg.generateRandomInts(DATA_LENGTH, TEST_BASE);
+
+    double[][] contDataNew = rg.generateNormalData(10, TEST_DIMENSIONS, 0, 1);
+    int[] discDataNew = rg.generateRandomInts(10, TEST_BASE);
+
+    // Check that result of locals does not depend on other points in the same query
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+    miCalc.setObservations(contData, discData);
+    double[] locals1 = miCalc.computeLocalUsingPreviousObservations(MatrixUtils.selectRows(contDataNew, 0, 5), MatrixUtils.select(discDataNew, 0, 5));
+    double[] locals2 = miCalc.computeLocalUsingPreviousObservations(contDataNew, discDataNew);
+    for (int i = 0; i < 5; i++) {
+      assertEquals(locals1[i], locals2[i]);
+    }
+  }
+}

From 9b42911c602e4770d30cc6ba94b5a3f75612a24a Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 01:01:31 +0100
Subject: [PATCH 18/78] Add KSG mixed unit test comparing against analytical
 value.

---
 ...MultiVariateWithDiscreteKraskovTester.java | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index 4de42ed..a2eb2bc 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -156,4 +156,28 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
       assertEquals(locals1[i], locals2[i]);
     }
   }
+
+  public void testCompareAnalyticalValue() throws Exception {
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+
+    double SEP = 4.0;
+    double true_val = 0.6327;
+    int N = 10000;
+
+    // Generate data with unit Gaussians separated by fixed distance
+    RandomGenerator rg = new RandomGenerator();
+    double[][] contData = rg.generateNormalData(N, 1, 0, 1);
+    int[] discData = rg.generateRandomInts(N, 2);
+    for (int i = 0; i < N; i++) {
+      if (discData[i] > 0) {
+        contData[i][0] += SEP;
+      }
+    }
+
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res = miCalc.computeAverageLocalOfObservations();
+    assertEquals(true_val, res, 0.05);
+  }
 }

From 17e8ee6104ffcb5d76c878a1b952620211f30234 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 16 Aug 2017 16:41:58 +1000
Subject: [PATCH 19/78] Adding a new demo data set for discrete variables,
 where each variable is of range 0..3, but share 1 bit over a lag 1.

---
 demos/data/2coupledDiscreteCols-1.txt | 10007 ++++++++++++++++++++++++
 1 file changed, 10007 insertions(+)
 create mode 100644 demos/data/2coupledDiscreteCols-1.txt

diff --git a/demos/data/2coupledDiscreteCols-1.txt b/demos/data/2coupledDiscreteCols-1.txt
new file mode 100644
index 0000000..1cef45e
--- /dev/null
+++ b/demos/data/2coupledDiscreteCols-1.txt
@@ -0,0 +1,10007 @@
+% Variable 1 is randomly distributed on 0..3.
+% Variable 2 takes the highest bit of the previous value of Variable 1
+%  (if we take a 2 bit representation of variable 1)
+%  as its own lowest bit, then assigns its highest bit at random.
+% So, the two should have ~1 bit of mutual information, if measured
+%  over a lag of 1 time step.
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 1 2
+ 3 2
+ 3 1
+ 2 3
+ 0 1
+ 0 2
+ 1 2
+ 1 2
+ 2 2
+ 3 1
+ 2 1
+ 0 3
+ 0 2
+ 1 0
+ 1 0
+ 0 2
+ 1 0
+ 1 0
+ 2 2
+ 2 1
+ 2 1
+ 0 1
+ 2 0
+ 1 1
+ 3 2
+ 1 3
+ 2 2
+ 3 1
+ 1 3
+ 1 0
+ 2 2
+ 0 1
+ 2 0
+ 2 3
+ 1 1
+ 2 0
+ 1 3
+ 2 2
+ 1 3
+ 2 2
+ 2 3
+ 1 1
+ 3 2
+ 3 1
+ 3 3
+ 2 3
+ 0 1
+ 2 0
+ 0 1
+ 2 0
+ 1 1
+ 1 0
+ 1 0
+ 2 2
+ 2 1
+ 2 1
+ 2 1
+ 2 3
+ 3 3
+ 1 1
+ 2 0
+ 3 3
+ 3 3
+ 3 3
+ 0 3
+ 3 0
+ 1 3
+ 1 0
+ 2 0
+ 1 3
+ 1 0
+ 0 2
+ 1 2
+ 2 0
+ 2 1
+ 2 1
+ 2 1
+ 1 1
+ 3 0
+ 2 3
+ 0 1
+ 1 2
+ 0 0
+ 0 0
+ 1 0
+ 2 0
+ 0 1
+ 0 0
+ 1 0
+ 3 0
+ 1 1
+ 0 0
+ 0 2
+ 1 2
+ 1 2
+ 0 0
+ 1 2
+ 1 0
+ 3 0
+ 2 3
+ 3 3
+ 1 1
+ 1 2
+ 3 2
+ 2 1
+ 0 3
+ 2 2
+ 1 1
+ 1 0
+ 2 2
+ 2 3
+ 1 1
+ 2 0
+ 0 3
+ 2 0
+ 3 1
+ 2 1
+ 3 3
+ 1 1
+ 0 0
+ 1 2
+ 0 0
+ 0 2
+ 2 0
+ 2 1
+ 3 1
+ 3 1
+ 3 3
+ 0 1
+ 3 0
+ 3 1
+ 2 1
+ 2 3
+ 0 1
+ 1 0
+ 0 2
+ 0 2
+ 0 0
+ 3 0
+ 2 3
+ 3 3
+ 2 1
+ 1 3
+ 3 0
+ 2 3
+ 1 1
+ 2 0
+ 3 1
+ 0 1
+ 2 0
+ 0 3
+ 1 0
+ 1 2
+ 1 0
+ 0 2
+ 0 0
+ 1 0
+ 0 0
+ 0 2
+ 2 0
+ 0 1
+ 0 2
+ 3 2
+ 0 3
+ 1 0
+ 1 2
+ 3 0
+ 1 3
+ 1 2
+ 0 2
+ 0 2
+ 0 2
+ 2 2
+ 1 3
+ 0 0
+ 2 0
+ 2 1
+ 2 1
+ 0 3
+ 0 2
+ 3 2
+ 3 3
+ 2 1
+ 1 1
+ 3 0
+ 1 3
+ 2 2
+ 1 1
+ 3 0
+ 2 1
+ 2 3
+ 0 1
+ 0 2
+ 3 2
+ 0 1
+ 1 2
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 3 2
+ 1 1
+ 2 0
+ 1 3
+ 3 0
+ 1 1
+ 0 2
+ 3 2
+ 2 1
+ 1 3
+ 0 2
+ 2 2
+ 2 3
+ 2 3
+ 0 1
+ 3 2
+ 2 1
+ 2 1
+ 1 3
+ 2 0
+ 2 3
+ 2 1
+ 2 1
+ 0 1
+ 0 0
+ 2 0
+ 2 1
+ 2 1
+ 1 3
+ 3 0
+ 0 3
+ 3 2
+ 0 1
+ 0 2
+ 3 0
+ 1 3
+ 0 2
+ 0 0
+ 1 0
+ 3 0
+ 2 3
+ 0 1
+ 1 2
+ 1 0
+ 1 0
+ 2 0
+ 2 3
+ 2 3
+ 1 3
+ 3 0
+ 2 3
+ 2 3
+ 1 1
+ 3 0
+ 1 1
+ 2 0
+ 1 1
+ 1 0
+ 2 0
+ 0 1
+ 1 0
+ 0 2
+ 1 2
+ 0 0
+ 1 2
+ 1 0
+ 2 0
+ 0 3
+ 3 0
+ 1 3
+ 0 0
+ 2 2
+ 0 1
+ 1 2
+ 3 0
+ 2 3
+ 3 1
+ 1 3
+ 1 0
+ 2 2
+ 0 3
+ 0 2
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 2 1
+ 1 3
+ 3 0
+ 0 3
+ 0 0
+ 1 2
+ 3 2
+ 0 3
+ 1 2
+ 3 2
+ 1 1
+ 3 2
+ 1 1
+ 1 2
+ 1 2
+ 1 0
+ 0 2
+ 3 2
+ 0 3
+ 1 0
+ 0 0
+ 2 2
+ 2 3
+ 3 3
+ 0 1
+ 2 0
+ 2 1
+ 3 1
+ 1 3
+ 0 0
+ 0 0
+ 1 0
+ 2 2
+ 1 3
+ 0 2
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 2 2
+ 3 3
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 3 0
+ 0 3
+ 3 0
+ 0 3
+ 0 2
+ 3 2
+ 1 1
+ 0 0
+ 1 0
+ 2 0
+ 1 3
+ 0 0
+ 1 2
+ 2 2
+ 1 1
+ 0 2
+ 2 0
+ 2 1
+ 2 1
+ 1 3
+ 2 0
+ 1 3
+ 3 2
+ 3 1
+ 0 3
+ 3 2
+ 0 1
+ 1 2
+ 2 0
+ 3 1
+ 1 3
+ 1 2
+ 0 0
+ 2 0
+ 1 3
+ 3 0
+ 2 3
+ 0 1
+ 0 0
+ 1 2
+ 3 0
+ 0 1
+ 3 0
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 3 1
+ 1 1
+ 3 2
+ 0 3
+ 3 0
+ 2 3
+ 1 3
+ 1 0
+ 2 0
+ 2 3
+ 3 3
+ 0 3
+ 1 2
+ 0 2
+ 3 0
+ 2 3
+ 0 3
+ 0 2
+ 3 0
+ 0 3
+ 2 2
+ 0 1
+ 0 2
+ 3 0
+ 2 1
+ 2 1
+ 3 1
+ 2 1
+ 1 3
+ 3 2
+ 3 1
+ 3 1
+ 1 3
+ 1 0
+ 2 2
+ 3 1
+ 0 3
+ 1 2
+ 0 2
+ 2 0
+ 1 1
+ 3 2
+ 1 1
+ 3 2
+ 3 1
+ 1 3
+ 3 0
+ 0 3
+ 1 2
+ 1 0
+ 1 0
+ 3 0
+ 2 1
+ 1 3
+ 0 0
+ 0 2
+ 3 0
+ 2 1
+ 3 3
+ 2 3
+ 0 3
+ 1 0
+ 2 2
+ 3 3
+ 2 3
+ 2 3
+ 2 3
+ 3 3
+ 0 1
+ 1 2
+ 0 2
+ 3 2
+ 2 3
+ 3 3
+ 1 3
+ 1 2
+ 0 0
+ 0 2
+ 0 0
+ 1 2
+ 2 2
+ 0 3
+ 0 0
+ 2 2
+ 1 1
+ 3 0
+ 3 3
+ 0 3
+ 2 0
+ 2 3
+ 2 3
+ 1 1
+ 3 0
+ 0 3
+ 3 2
+ 3 1
+ 3 3
+ 3 1
+ 1 1
+ 2 0
+ 2 1
+ 3 3
+ 1 3
+ 2 2
+ 3 3
+ 0 1
+ 2 2
+ 0 3
+ 1 0
+ 1 2
+ 1 2
+ 0 0
+ 3 0
+ 3 1
+ 1 1
+ 2 2
+ 3 1
+ 0 1
+ 1 0
+ 2 0
+ 2 1
+ 0 1
+ 3 0
+ 3 1
+ 1 1
+ 1 2
+ 0 2
+ 2 2
+ 1 3
+ 0 0
+ 2 2
+ 1 3
+ 1 2
+ 1 2
+ 2 0
+ 0 1
+ 0 0
+ 3 2
+ 0 3
+ 2 0
+ 0 3
+ 2 0
+ 2 3
+ 1 3
+ 3 2
+ 0 3
+ 1 0
+ 2 2
+ 1 3
+ 0 0
+ 1 2
+ 3 2
+ 3 1
+ 0 3
+ 2 2
+ 2 3
+ 3 1
+ 1 1
+ 1 0
+ 2 0
+ 2 3
+ 1 1
+ 1 0
+ 3 2
+ 0 3
+ 2 0
+ 2 3
+ 1 3
+ 0 2
+ 0 2
+ 3 2
+ 0 1
+ 1 0
+ 3 0
+ 0 3
+ 1 0
+ 0 0
+ 0 2
+ 2 2
+ 1 3
+ 1 2
+ 0 0
+ 1 0
+ 2 2
+ 1 1
+ 2 2
+ 2 3
+ 0 3
+ 3 2
+ 3 3
+ 2 1
+ 1 1
+ 3 2
+ 2 1
+ 0 1
+ 0 2
+ 0 0
+ 2 0
+ 3 3
+ 2 1
+ 2 3
+ 1 3
+ 1 0
+ 1 0
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 2 0
+ 1 1
+ 1 2
+ 2 0
+ 3 3
+ 1 3
+ 3 0
+ 0 3
+ 0 2
+ 1 0
+ 1 2
+ 1 0
+ 1 2
+ 2 0
+ 2 3
+ 3 1
+ 2 3
+ 1 1
+ 2 0
+ 1 3
+ 1 2
+ 2 0
+ 1 1
+ 3 0
+ 0 1
+ 0 0
+ 0 2
+ 1 0
+ 1 2
+ 3 0
+ 2 1
+ 2 3
+ 1 3
+ 3 2
+ 0 1
+ 0 2
+ 2 0
+ 1 1
+ 3 0
+ 3 1
+ 3 1
+ 0 1
+ 3 0
+ 3 1
+ 1 1
+ 3 0
+ 3 3
+ 0 1
+ 0 2
+ 3 2
+ 0 3
+ 1 0
+ 0 0
+ 3 0
+ 1 1
+ 3 0
+ 0 1
+ 0 2
+ 3 2
+ 1 1
+ 3 0
+ 2 3
+ 3 1
+ 2 1
+ 3 1
+ 3 1
+ 0 1
+ 3 0
+ 0 3
+ 1 0
+ 1 2
+ 0 0
+ 0 2
+ 2 2
+ 3 3
+ 3 3
+ 1 1
+ 1 0
+ 1 0
+ 3 2
+ 1 1
+ 1 2
+ 1 2
+ 0 0
+ 0 0
+ 3 0
+ 1 1
+ 0 0
+ 3 0
+ 2 3
+ 2 3
+ 2 1
+ 2 3
+ 2 3
+ 1 1
+ 1 0
+ 3 0
+ 3 3
+ 3 3
+ 2 3
+ 0 1
+ 3 2
+ 0 3
+ 1 0
+ 3 2
+ 3 1
+ 0 3
+ 3 0
+ 3 1
+ 2 1
+ 2 1
+ 3 3
+ 3 3
+ 2 3
+ 2 3
+ 1 1
+ 3 0
+ 0 1
+ 0 0
+ 2 2
+ 2 1
+ 0 3
+ 1 0
+ 2 0
+ 0 1
+ 0 0
+ 2 0
+ 3 1
+ 2 3
+ 2 3
+ 2 1
+ 2 1
+ 2 3
+ 2 1
+ 2 3
+ 0 1
+ 2 2
+ 2 3
+ 3 1
+ 0 1
+ 3 0
+ 3 3
+ 1 1
+ 3 2
+ 1 3
+ 1 0
+ 3 0
+ 2 3
+ 1 1
+ 3 0
+ 2 1
+ 3 1
+ 2 3
+ 2 3
+ 3 1
+ 3 1
+ 3 1
+ 1 3
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 1 1
+ 1 0
+ 2 0
+ 3 3
+ 1 1
+ 3 0
+ 3 3
+ 2 3
+ 3 3
+ 2 1
+ 1 1
+ 2 2
+ 1 3
+ 2 0
+ 3 3
+ 1 3
+ 0 2
+ 1 2
+ 0 2
+ 0 2
+ 1 2
+ 1 2
+ 1 2
+ 2 2
+ 3 1
+ 0 1
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 0 0
+ 3 0
+ 3 3
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 1 1
+ 1 0
+ 2 0
+ 3 1
+ 2 1
+ 0 1
+ 2 2
+ 1 1
+ 3 2
+ 0 1
+ 1 0
+ 1 2
+ 3 0
+ 0 3
+ 1 2
+ 0 2
+ 0 2
+ 2 2
+ 0 3
+ 1 2
+ 2 2
+ 1 1
+ 3 2
+ 0 3
+ 0 0
+ 2 2
+ 2 3
+ 0 3
+ 2 0
+ 3 1
+ 1 3
+ 3 2
+ 1 3
+ 1 0
+ 2 2
+ 1 3
+ 0 0
+ 2 0
+ 3 1
+ 1 1
+ 1 0
+ 3 2
+ 1 1
+ 1 0
+ 1 0
+ 3 2
+ 2 3
+ 1 3
+ 3 0
+ 1 3
+ 2 0
+ 1 1
+ 2 2
+ 3 1
+ 3 1
+ 3 1
+ 0 3
+ 3 2
+ 2 3
+ 0 1
+ 1 2
+ 2 2
+ 1 1
+ 3 2
+ 2 1
+ 1 3
+ 0 0
+ 3 0
+ 0 1
+ 3 0
+ 0 3
+ 3 0
+ 3 1
+ 2 3
+ 2 1
+ 2 3
+ 0 1
+ 2 0
+ 3 3
+ 1 1
+ 1 2
+ 1 0
+ 2 0
+ 1 3
+ 3 2
+ 3 1
+ 1 3
+ 0 0
+ 0 0
+ 3 0
+ 1 3
+ 2 0
+ 0 3
+ 1 2
+ 3 0
+ 3 1
+ 0 3
+ 1 0
+ 0 2
+ 0 2
+ 0 2
+ 0 0
+ 3 2
+ 1 3
+ 0 0
+ 2 2
+ 1 3
+ 0 0
+ 2 2
+ 0 3
+ 0 2
+ 2 2
+ 1 3
+ 1 2
+ 3 2
+ 3 1
+ 3 3
+ 3 3
+ 2 1
+ 0 1
+ 3 0
+ 2 1
+ 2 1
+ 0 1
+ 3 0
+ 0 3
+ 3 0
+ 2 1
+ 2 3
+ 2 1
+ 3 1
+ 3 3
+ 1 1
+ 3 0
+ 0 3
+ 0 2
+ 2 2
+ 2 3
+ 3 1
+ 2 1
+ 0 3
+ 3 0
+ 0 3
+ 0 0
+ 1 2
+ 3 0
+ 0 1
+ 2 2
+ 0 1
+ 2 2
+ 0 3
+ 0 2
+ 2 0
+ 2 3
+ 0 1
+ 0 2
+ 0 2
+ 0 0
+ 2 2
+ 3 3
+ 1 3
+ 0 0
+ 2 2
+ 0 3
+ 3 2
+ 2 1
+ 3 1
+ 3 1
+ 0 3
+ 2 2
+ 0 3
+ 3 0
+ 3 1
+ 0 3
+ 1 2
+ 0 0
+ 1 2
+ 3 0
+ 0 3
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 1 2
+ 3 0
+ 0 1
+ 0 2
+ 0 2
+ 3 0
+ 3 3
+ 3 1
+ 2 3
+ 2 1
+ 3 1
+ 1 3
+ 2 2
+ 1 1
+ 2 0
+ 0 3
+ 0 0
+ 0 2
+ 1 0
+ 2 0
+ 1 1
+ 0 0
+ 0 0
+ 0 0
+ 2 2
+ 2 3
+ 2 3
+ 1 1
+ 2 0
+ 3 3
+ 2 1
+ 2 3
+ 1 3
+ 0 0
+ 3 2
+ 3 3
+ 0 3
+ 0 2
+ 1 2
+ 3 2
+ 2 3
+ 3 3
+ 1 3
+ 1 2
+ 1 2
+ 1 2
+ 1 0
+ 2 0
+ 3 3
+ 1 3
+ 3 2
+ 1 1
+ 2 2
+ 0 3
+ 1 2
+ 3 0
+ 3 1
+ 1 1
+ 2 2
+ 3 1
+ 3 3
+ 1 3
+ 2 0
+ 3 3
+ 3 1
+ 2 3
+ 2 1
+ 2 1
+ 1 1
+ 1 0
+ 0 2
+ 3 0
+ 2 3
+ 2 3
+ 1 3
+ 3 2
+ 1 1
+ 2 2
+ 0 1
+ 1 0
+ 0 0
+ 3 0
+ 2 1
+ 1 3
+ 1 0
+ 0 0
+ 2 0
+ 1 1
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 1 1
+ 0 2
+ 1 2
+ 3 0
+ 0 1
+ 0 2
+ 1 0
+ 1 2
+ 2 0
+ 0 3
+ 0 2
+ 1 0
+ 1 0
+ 1 0
+ 1 2
+ 0 2
+ 2 0
+ 0 3
+ 3 2
+ 3 1
+ 2 3
+ 0 1
+ 2 2
+ 1 1
+ 0 0
+ 3 2
+ 1 1
+ 0 2
+ 2 2
+ 2 3
+ 3 3
+ 2 1
+ 2 3
+ 1 1
+ 1 2
+ 2 2
+ 3 3
+ 2 1
+ 1 1
+ 1 2
+ 0 0
+ 0 0
+ 3 0
+ 1 3
+ 3 0
+ 0 3
+ 3 0
+ 1 3
+ 3 2
+ 0 1
+ 0 0
+ 1 0
+ 0 2
+ 1 2
+ 0 0
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 2 1
+ 0 3
+ 2 0
+ 1 1
+ 3 2
+ 1 1
+ 3 0
+ 1 1
+ 2 0
+ 3 1
+ 0 3
+ 3 2
+ 1 1
+ 0 2
+ 2 2
+ 1 3
+ 3 0
+ 2 3
+ 1 1
+ 0 2
+ 3 2
+ 3 1
+ 0 3
+ 2 0
+ 1 3
+ 2 0
+ 3 1
+ 1 3
+ 1 0
+ 3 0
+ 2 1
+ 3 3
+ 1 1
+ 2 0
+ 0 1
+ 1 2
+ 1 0
+ 3 0
+ 2 1
+ 2 1
+ 0 3
+ 0 2
+ 0 0
+ 3 2
+ 1 1
+ 1 0
+ 0 2
+ 0 2
+ 1 2
+ 2 0
+ 1 3
+ 0 2
+ 3 2
+ 0 1
+ 3 0
+ 0 3
+ 1 2
+ 3 0
+ 3 3
+ 3 3
+ 1 3
+ 1 0
+ 3 0
+ 0 3
+ 0 0
+ 0 0
+ 2 0
+ 0 3
+ 3 0
+ 1 1
+ 0 0
+ 2 0
+ 0 3
+ 2 0
+ 3 1
+ 1 1
+ 0 0
+ 2 2
+ 2 1
+ 2 3
+ 2 3
+ 0 1
+ 2 2
+ 0 1
+ 3 0
+ 2 3
+ 3 3
+ 0 3
+ 2 0
+ 0 1
+ 0 2
+ 0 0
+ 1 2
+ 0 0
+ 3 0
+ 2 1
+ 3 1
+ 0 1
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 1 3
+ 0 2
+ 3 2
+ 0 3
+ 3 0
+ 2 1
+ 0 3
+ 3 2
+ 1 3
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 2 2
+ 0 3
+ 1 2
+ 2 0
+ 3 1
+ 2 1
+ 3 3
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 1 1
+ 0 2
+ 2 2
+ 2 3
+ 2 3
+ 0 1
+ 0 2
+ 0 2
+ 2 2
+ 1 1
+ 1 0
+ 1 2
+ 3 2
+ 3 3
+ 3 3
+ 3 1
+ 2 3
+ 1 3
+ 2 2
+ 3 1
+ 1 1
+ 3 2
+ 2 1
+ 1 3
+ 0 0
+ 1 2
+ 1 0
+ 0 2
+ 1 0
+ 3 0
+ 3 1
+ 3 3
+ 2 3
+ 0 3
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 1 1
+ 2 2
+ 2 3
+ 3 3
+ 3 3
+ 1 3
+ 3 2
+ 3 3
+ 0 3
+ 3 0
+ 2 3
+ 0 3
+ 2 2
+ 1 3
+ 3 2
+ 0 1
+ 0 0
+ 2 2
+ 1 1
+ 2 0
+ 0 1
+ 3 2
+ 2 1
+ 1 1
+ 1 0
+ 0 2
+ 2 0
+ 2 3
+ 2 1
+ 3 3
+ 3 3
+ 1 1
+ 1 2
+ 3 2
+ 0 3
+ 2 2
+ 1 3
+ 3 0
+ 0 3
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 2 3
+ 3 3
+ 1 1
+ 3 2
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 1 1
+ 2 0
+ 1 1
+ 2 0
+ 1 3
+ 1 2
+ 1 0
+ 0 2
+ 3 2
+ 1 1
+ 3 0
+ 3 1
+ 2 3
+ 1 1
+ 3 0
+ 2 1
+ 3 3
+ 0 3
+ 3 0
+ 2 1
+ 2 1
+ 3 1
+ 1 3
+ 3 0
+ 1 1
+ 3 0
+ 3 3
+ 1 1
+ 0 0
+ 3 0
+ 2 1
+ 3 1
+ 2 3
+ 0 1
+ 1 2
+ 3 0
+ 0 3
+ 3 2
+ 2 1
+ 1 3
+ 0 0
+ 3 2
+ 0 1
+ 0 0
+ 2 0
+ 3 3
+ 3 3
+ 2 1
+ 0 3
+ 1 2
+ 0 0
+ 1 0
+ 0 0
+ 2 0
+ 1 3
+ 0 0
+ 1 0
+ 1 0
+ 2 2
+ 2 1
+ 0 3
+ 2 2
+ 1 1
+ 1 2
+ 2 0
+ 3 3
+ 2 3
+ 1 3
+ 1 2
+ 1 2
+ 3 0
+ 0 3
+ 1 0
+ 3 0
+ 3 3
+ 1 3
+ 3 0
+ 2 1
+ 3 1
+ 3 1
+ 3 3
+ 3 1
+ 2 3
+ 1 3
+ 2 2
+ 3 1
+ 1 3
+ 2 0
+ 1 3
+ 1 2
+ 1 0
+ 1 2
+ 3 0
+ 1 1
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 0 3
+ 3 0
+ 2 1
+ 0 3
+ 2 0
+ 3 3
+ 1 3
+ 2 2
+ 1 1
+ 0 0
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 3 0
+ 2 3
+ 3 1
+ 3 3
+ 0 3
+ 1 0
+ 0 2
+ 2 2
+ 1 1
+ 1 0
+ 3 0
+ 1 3
+ 2 2
+ 0 1
+ 3 2
+ 1 3
+ 3 2
+ 2 3
+ 0 3
+ 0 2
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 0 3
+ 3 2
+ 2 1
+ 2 3
+ 3 1
+ 0 3
+ 2 2
+ 1 1
+ 0 0
+ 1 2
+ 2 2
+ 1 3
+ 1 2
+ 3 2
+ 1 3
+ 2 0
+ 1 1
+ 3 0
+ 0 1
+ 3 0
+ 2 3
+ 1 1
+ 0 0
+ 3 2
+ 3 1
+ 3 1
+ 0 3
+ 2 2
+ 2 3
+ 1 3
+ 1 2
+ 0 0
+ 3 2
+ 3 3
+ 1 3
+ 3 0
+ 2 1
+ 2 3
+ 3 1
+ 3 1
+ 0 1
+ 2 2
+ 3 3
+ 1 3
+ 1 2
+ 2 0
+ 1 1
+ 3 0
+ 0 3
+ 2 2
+ 3 3
+ 0 3
+ 1 2
+ 2 2
+ 3 1
+ 0 1
+ 2 0
+ 0 3
+ 3 0
+ 3 3
+ 3 1
+ 3 3
+ 3 3
+ 1 3
+ 1 0
+ 0 0
+ 2 0
+ 1 3
+ 3 0
+ 2 3
+ 1 3
+ 3 2
+ 3 1
+ 2 3
+ 3 1
+ 3 1
+ 2 1
+ 3 3
+ 0 3
+ 2 2
+ 0 1
+ 3 2
+ 0 1
+ 1 0
+ 3 0
+ 3 3
+ 0 1
+ 2 0
+ 1 1
+ 1 2
+ 3 0
+ 2 3
+ 0 3
+ 1 0
+ 0 0
+ 3 2
+ 2 3
+ 2 1
+ 2 1
+ 0 1
+ 1 2
+ 3 2
+ 1 3
+ 3 2
+ 0 1
+ 1 2
+ 3 2
+ 0 3
+ 1 2
+ 3 2
+ 3 3
+ 3 1
+ 2 3
+ 0 3
+ 0 0
+ 2 2
+ 2 3
+ 1 3
+ 3 2
+ 1 3
+ 0 0
+ 3 2
+ 1 1
+ 2 2
+ 2 3
+ 1 3
+ 0 0
+ 3 0
+ 0 3
+ 2 0
+ 3 1
+ 3 1
+ 2 3
+ 2 3
+ 3 1
+ 2 1
+ 2 1
+ 3 1
+ 1 1
+ 3 2
+ 0 3
+ 0 2
+ 3 0
+ 2 1
+ 0 1
+ 0 0
+ 3 2
+ 2 3
+ 3 1
+ 3 3
+ 0 3
+ 1 2
+ 2 2
+ 3 3
+ 2 1
+ 0 1
+ 1 2
+ 3 0
+ 3 3
+ 0 1
+ 3 2
+ 0 1
+ 1 2
+ 3 2
+ 3 1
+ 2 3
+ 3 1
+ 1 1
+ 0 2
+ 2 0
+ 0 3
+ 2 2
+ 0 3
+ 1 2
+ 2 2
+ 2 3
+ 0 1
+ 0 2
+ 2 0
+ 0 3
+ 1 2
+ 2 2
+ 1 1
+ 2 2
+ 2 3
+ 0 3
+ 1 0
+ 1 2
+ 0 2
+ 1 2
+ 1 0
+ 3 2
+ 3 1
+ 1 3
+ 0 0
+ 3 2
+ 3 3
+ 0 3
+ 2 2
+ 3 3
+ 0 3
+ 1 2
+ 2 0
+ 3 3
+ 3 3
+ 3 3
+ 1 3
+ 0 2
+ 1 0
+ 2 0
+ 1 1
+ 3 2
+ 1 3
+ 0 2
+ 2 2
+ 1 1
+ 2 0
+ 1 3
+ 0 0
+ 3 0
+ 2 1
+ 1 1
+ 0 2
+ 3 0
+ 1 3
+ 1 0
+ 2 2
+ 3 3
+ 2 3
+ 0 3
+ 0 0
+ 0 2
+ 2 2
+ 0 3
+ 0 0
+ 1 0
+ 0 0
+ 0 0
+ 3 0
+ 2 1
+ 0 1
+ 1 2
+ 3 0
+ 0 3
+ 3 0
+ 2 1
+ 2 1
+ 2 3
+ 1 3
+ 2 0
+ 2 1
+ 2 1
+ 3 3
+ 1 3
+ 0 0
+ 3 0
+ 2 3
+ 3 3
+ 0 1
+ 3 2
+ 0 1
+ 2 2
+ 2 1
+ 2 1
+ 2 3
+ 2 3
+ 3 1
+ 0 3
+ 1 0
+ 3 0
+ 1 1
+ 0 0
+ 2 0
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 0 0
+ 3 0
+ 3 3
+ 1 3
+ 3 0
+ 1 3
+ 3 2
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 0 1
+ 3 0
+ 1 1
+ 3 0
+ 0 1
+ 2 2
+ 1 3
+ 2 2
+ 2 3
+ 2 1
+ 1 1
+ 0 0
+ 1 2
+ 3 0
+ 3 1
+ 0 1
+ 2 0
+ 1 3
+ 3 0
+ 1 3
+ 1 2
+ 2 2
+ 2 3
+ 2 3
+ 2 3
+ 2 1
+ 1 1
+ 0 0
+ 3 2
+ 3 3
+ 3 3
+ 0 3
+ 2 2
+ 0 1
+ 1 0
+ 2 2
+ 1 1
+ 1 2
+ 0 0
+ 1 2
+ 3 2
+ 1 3
+ 0 2
+ 0 2
+ 1 0
+ 2 0
+ 3 1
+ 1 3
+ 3 2
+ 3 1
+ 0 1
+ 1 2
+ 1 0
+ 0 2
+ 0 0
+ 3 2
+ 1 1
+ 3 2
+ 1 3
+ 3 0
+ 1 3
+ 3 0
+ 2 3
+ 3 1
+ 3 3
+ 3 1
+ 0 1
+ 1 0
+ 1 2
+ 2 2
+ 0 1
+ 1 2
+ 2 0
+ 3 1
+ 1 3
+ 3 0
+ 2 1
+ 1 1
+ 3 0
+ 3 1
+ 1 3
+ 1 0
+ 0 2
+ 0 2
+ 0 0
+ 2 2
+ 2 1
+ 3 3
+ 0 3
+ 2 0
+ 0 1
+ 2 0
+ 2 1
+ 2 1
+ 3 3
+ 3 3
+ 0 1
+ 1 2
+ 0 2
+ 0 0
+ 2 2
+ 3 3
+ 2 1
+ 1 1
+ 2 0
+ 3 3
+ 0 3
+ 2 2
+ 1 1
+ 0 0
+ 1 2
+ 2 0
+ 0 3
+ 3 2
+ 3 1
+ 2 3
+ 3 1
+ 2 1
+ 1 1
+ 2 2
+ 2 3
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 3 3
+ 3 1
+ 1 1
+ 3 2
+ 2 3
+ 2 1
+ 1 3
+ 3 0
+ 3 3
+ 3 3
+ 1 3
+ 1 0
+ 3 0
+ 3 1
+ 2 1
+ 1 3
+ 2 2
+ 0 3
+ 3 2
+ 3 3
+ 2 1
+ 1 3
+ 2 0
+ 3 1
+ 2 3
+ 3 1
+ 2 3
+ 2 3
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 3 2
+ 0 3
+ 0 2
+ 3 0
+ 1 1
+ 1 2
+ 3 0
+ 0 1
+ 3 2
+ 3 3
+ 1 1
+ 0 2
+ 0 2
+ 2 0
+ 2 3
+ 0 3
+ 2 2
+ 1 3
+ 1 0
+ 0 0
+ 2 2
+ 3 1
+ 3 3
+ 2 3
+ 0 3
+ 3 2
+ 3 1
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 0 3
+ 1 0
+ 1 0
+ 1 0
+ 0 2
+ 0 0
+ 2 0
+ 3 1
+ 1 3
+ 0 0
+ 3 2
+ 1 3
+ 3 2
+ 2 3
+ 3 1
+ 0 3
+ 1 2
+ 0 2
+ 2 0
+ 2 3
+ 2 1
+ 2 3
+ 1 3
+ 0 2
+ 3 0
+ 1 3
+ 1 2
+ 2 0
+ 0 1
+ 0 2
+ 1 0
+ 0 0
+ 1 2
+ 0 0
+ 3 2
+ 3 1
+ 2 3
+ 2 1
+ 0 1
+ 3 0
+ 1 1
+ 0 0
+ 0 0
+ 3 2
+ 1 1
+ 2 0
+ 2 3
+ 1 3
+ 1 0
+ 2 2
+ 2 3
+ 0 3
+ 2 0
+ 3 1
+ 0 3
+ 1 0
+ 2 2
+ 1 3
+ 2 2
+ 3 1
+ 3 1
+ 0 3
+ 0 0
+ 2 0
+ 3 1
+ 2 1
+ 0 1
+ 1 0
+ 2 0
+ 2 3
+ 3 1
+ 3 1
+ 2 1
+ 2 1
+ 3 3
+ 0 1
+ 3 0
+ 0 1
+ 2 0
+ 1 3
+ 0 2
+ 2 0
+ 1 1
+ 1 2
+ 1 2
+ 0 0
+ 1 2
+ 2 0
+ 3 3
+ 2 3
+ 3 1
+ 0 1
+ 2 2
+ 2 1
+ 0 1
+ 1 0
+ 2 2
+ 3 3
+ 1 1
+ 3 0
+ 0 3
+ 1 2
+ 0 0
+ 0 2
+ 1 2
+ 3 2
+ 2 3
+ 0 1
+ 3 2
+ 3 3
+ 1 3
+ 3 2
+ 0 3
+ 0 0
+ 2 0
+ 2 1
+ 3 3
+ 1 3
+ 0 0
+ 1 2
+ 0 2
+ 0 0
+ 2 0
+ 1 3
+ 3 2
+ 3 3
+ 3 3
+ 1 1
+ 2 2
+ 2 3
+ 3 1
+ 2 3
+ 3 3
+ 2 1
+ 0 3
+ 2 2
+ 2 1
+ 0 3
+ 0 0
+ 3 2
+ 2 3
+ 3 3
+ 2 3
+ 0 3
+ 1 0
+ 1 0
+ 0 0
+ 0 2
+ 2 2
+ 1 3
+ 1 0
+ 1 0
+ 1 2
+ 0 2
+ 0 0
+ 2 2
+ 3 1
+ 3 1
+ 2 3
+ 3 3
+ 3 1
+ 0 3
+ 2 0
+ 1 3
+ 2 2
+ 1 3
+ 3 0
+ 2 1
+ 0 1
+ 1 0
+ 3 0
+ 3 1
+ 1 1
+ 1 2
+ 2 2
+ 0 3
+ 2 0
+ 1 3
+ 3 2
+ 0 3
+ 2 0
+ 0 1
+ 2 0
+ 1 3
+ 1 0
+ 3 2
+ 0 1
+ 0 0
+ 0 2
+ 1 2
+ 1 2
+ 3 0
+ 0 1
+ 0 0
+ 3 2
+ 2 1
+ 1 3
+ 3 0
+ 3 3
+ 3 1
+ 2 3
+ 1 1
+ 1 0
+ 1 0
+ 0 0
+ 2 0
+ 1 3
+ 2 0
+ 3 3
+ 1 1
+ 2 2
+ 3 1
+ 0 3
+ 2 0
+ 3 1
+ 0 3
+ 3 2
+ 3 3
+ 2 3
+ 1 1
+ 1 2
+ 3 0
+ 0 1
+ 0 0
+ 0 0
+ 2 0
+ 1 1
+ 0 0
+ 1 0
+ 3 2
+ 1 3
+ 1 0
+ 1 0
+ 0 2
+ 2 2
+ 2 1
+ 1 3
+ 0 0
+ 3 0
+ 1 1
+ 0 2
+ 3 2
+ 1 1
+ 2 2
+ 3 3
+ 1 1
+ 3 2
+ 3 3
+ 0 1
+ 2 0
+ 1 3
+ 0 0
+ 1 2
+ 0 2
+ 0 2
+ 1 2
+ 3 0
+ 0 3
+ 0 2
+ 1 0
+ 3 0
+ 0 3
+ 0 2
+ 2 0
+ 0 3
+ 1 0
+ 1 2
+ 0 2
+ 1 0
+ 0 0
+ 0 2
+ 1 2
+ 3 0
+ 0 3
+ 1 0
+ 3 0
+ 1 1
+ 1 0
+ 2 0
+ 0 1
+ 0 2
+ 1 0
+ 3 0
+ 2 3
+ 3 1
+ 0 1
+ 3 2
+ 1 3
+ 3 2
+ 2 3
+ 1 1
+ 0 0
+ 1 2
+ 2 2
+ 1 1
+ 1 0
+ 2 2
+ 0 1
+ 3 0
+ 1 3
+ 2 2
+ 3 1
+ 2 1
+ 1 3
+ 2 2
+ 1 3
+ 3 0
+ 2 3
+ 2 1
+ 0 1
+ 3 0
+ 2 1
+ 0 1
+ 2 2
+ 0 3
+ 1 2
+ 0 0
+ 2 0
+ 1 1
+ 0 2
+ 0 2
+ 0 2
+ 0 0
+ 3 0
+ 1 1
+ 2 0
+ 1 3
+ 1 0
+ 1 0
+ 2 2
+ 0 3
+ 1 0
+ 0 0
+ 2 2
+ 3 3
+ 1 3
+ 1 2
+ 3 2
+ 0 1
+ 1 2
+ 2 2
+ 1 3
+ 0 2
+ 2 0
+ 3 1
+ 0 1
+ 3 0
+ 0 1
+ 1 2
+ 2 0
+ 3 3
+ 2 3
+ 2 1
+ 0 1
+ 2 2
+ 2 3
+ 1 3
+ 3 2
+ 3 1
+ 3 3
+ 2 1
+ 0 1
+ 3 2
+ 1 1
+ 2 2
+ 0 3
+ 3 2
+ 2 3
+ 2 3
+ 2 1
+ 1 3
+ 1 2
+ 3 2
+ 0 3
+ 2 0
+ 1 3
+ 0 0
+ 3 2
+ 3 3
+ 0 1
+ 2 2
+ 1 3
+ 0 0
+ 3 0
+ 3 1
+ 2 1
+ 1 1
+ 3 2
+ 2 1
+ 1 3
+ 0 0
+ 2 2
+ 2 1
+ 0 1
+ 1 0
+ 0 0
+ 0 2
+ 3 0
+ 1 3
+ 1 2
+ 1 0
+ 1 2
+ 2 2
+ 2 1
+ 1 3
+ 0 0
+ 3 0
+ 2 3
+ 1 3
+ 3 0
+ 3 3
+ 0 1
+ 1 0
+ 3 2
+ 2 1
+ 2 1
+ 2 3
+ 0 1
+ 2 0
+ 1 3
+ 0 0
+ 2 2
+ 3 1
+ 2 3
+ 0 1
+ 2 2
+ 1 3
+ 3 2
+ 3 3
+ 0 1
+ 1 0
+ 1 0
+ 0 2
+ 1 0
+ 1 2
+ 2 2
+ 2 3
+ 2 1
+ 3 1
+ 0 3
+ 3 2
+ 3 1
+ 2 3
+ 3 1
+ 1 3
+ 2 0
+ 1 1
+ 3 2
+ 0 3
+ 2 2
+ 1 3
+ 3 0
+ 2 3
+ 3 3
+ 3 3
+ 3 1
+ 0 3
+ 0 0
+ 3 2
+ 2 3
+ 1 3
+ 1 0
+ 1 0
+ 2 2
+ 0 3
+ 0 0
+ 3 2
+ 1 1
+ 1 2
+ 3 0
+ 3 3
+ 2 1
+ 1 1
+ 0 2
+ 0 0
+ 0 0
+ 3 2
+ 0 3
+ 2 0
+ 2 1
+ 1 1
+ 2 0
+ 1 3
+ 3 0
+ 2 1
+ 2 1
+ 2 1
+ 3 3
+ 3 1
+ 2 1
+ 0 3
+ 2 0
+ 3 1
+ 1 1
+ 1 2
+ 0 0
+ 3 0
+ 3 3
+ 3 1
+ 1 3
+ 0 0
+ 0 2
+ 2 0
+ 3 1
+ 2 3
+ 2 1
+ 1 3
+ 1 0
+ 2 2
+ 2 1
+ 2 3
+ 3 1
+ 2 1
+ 0 1
+ 2 0
+ 0 3
+ 0 0
+ 3 2
+ 1 3
+ 1 2
+ 2 0
+ 3 3
+ 3 1
+ 0 3
+ 2 0
+ 1 3
+ 2 2
+ 1 1
+ 1 2
+ 2 2
+ 0 1
+ 1 2
+ 0 2
+ 3 2
+ 0 1
+ 2 2
+ 2 3
+ 1 3
+ 0 0
+ 0 0
+ 1 2
+ 1 0
+ 1 0
+ 2 2
+ 0 1
+ 3 0
+ 0 3
+ 3 0
+ 2 3
+ 1 3
+ 0 2
+ 0 0
+ 3 2
+ 3 3
+ 3 3
+ 0 3
+ 3 0
+ 0 1
+ 1 0
+ 1 0
+ 2 2
+ 0 3
+ 1 0
+ 1 2
+ 0 2
+ 0 0
+ 2 2
+ 3 3
+ 3 1
+ 1 3
+ 0 2
+ 1 2
+ 0 2
+ 2 0
+ 2 1
+ 0 1
+ 2 0
+ 0 1
+ 2 0
+ 1 3
+ 1 0
+ 2 0
+ 0 1
+ 0 0
+ 3 0
+ 2 1
+ 3 1
+ 0 1
+ 1 0
+ 1 2
+ 2 0
+ 0 3
+ 0 2
+ 0 2
+ 3 2
+ 3 3
+ 2 1
+ 1 1
+ 0 0
+ 1 2
+ 2 2
+ 0 1
+ 1 2
+ 0 2
+ 0 2
+ 2 0
+ 2 1
+ 0 3
+ 0 2
+ 2 0
+ 1 1
+ 3 2
+ 3 1
+ 2 1
+ 3 3
+ 3 1
+ 1 1
+ 3 0
+ 2 3
+ 1 1
+ 3 0
+ 0 3
+ 1 0
+ 3 2
+ 3 1
+ 1 3
+ 0 0
+ 1 0
+ 3 0
+ 2 1
+ 3 3
+ 3 1
+ 0 1
+ 0 0
+ 2 0
+ 3 3
+ 2 3
+ 2 3
+ 3 1
+ 1 3
+ 1 2
+ 0 2
+ 2 0
+ 0 3
+ 1 0
+ 2 0
+ 1 3
+ 2 2
+ 0 1
+ 3 0
+ 3 1
+ 3 1
+ 0 1
+ 2 2
+ 0 1
+ 2 0
+ 0 1
+ 3 0
+ 1 1
+ 0 0
+ 1 0
+ 3 0
+ 2 3
+ 1 1
+ 0 2
+ 3 0
+ 0 3
+ 0 2
+ 2 2
+ 0 1
+ 2 0
+ 1 3
+ 0 0
+ 1 2
+ 3 0
+ 2 1
+ 2 1
+ 3 3
+ 2 1
+ 0 1
+ 0 0
+ 3 2
+ 2 3
+ 0 3
+ 1 2
+ 3 0
+ 1 3
+ 2 2
+ 3 1
+ 0 1
+ 1 0
+ 2 2
+ 0 1
+ 0 2
+ 3 0
+ 2 1
+ 3 3
+ 1 1
+ 1 0
+ 1 2
+ 1 2
+ 1 2
+ 1 2
+ 1 0
+ 2 0
+ 3 3
+ 3 3
+ 0 3
+ 0 0
+ 1 2
+ 0 0
+ 1 2
+ 0 0
+ 1 2
+ 3 0
+ 1 3
+ 3 2
+ 1 3
+ 1 0
+ 0 0
+ 0 2
+ 0 0
+ 2 2
+ 0 3
+ 0 2
+ 2 2
+ 0 1
+ 3 0
+ 2 3
+ 2 1
+ 1 1
+ 0 0
+ 1 2
+ 3 0
+ 2 3
+ 0 1
+ 2 2
+ 3 3
+ 0 3
+ 2 0
+ 0 3
+ 1 2
+ 1 0
+ 1 0
+ 0 2
+ 1 0
+ 0 2
+ 3 0
+ 1 3
+ 0 2
+ 0 2
+ 1 2
+ 2 0
+ 3 3
+ 1 3
+ 0 0
+ 3 2
+ 2 3
+ 3 1
+ 1 1
+ 2 2
+ 0 1
+ 0 2
+ 1 2
+ 2 0
+ 0 3
+ 0 0
+ 2 0
+ 3 1
+ 1 1
+ 2 0
+ 1 3
+ 2 2
+ 3 1
+ 3 3
+ 0 1
+ 3 2
+ 0 1
+ 3 2
+ 2 3
+ 2 1
+ 3 1
+ 2 3
+ 3 1
+ 3 1
+ 3 3
+ 1 3
+ 3 2
+ 1 3
+ 1 2
+ 1 0
+ 0 2
+ 2 2
+ 0 3
+ 1 2
+ 0 0
+ 2 2
+ 0 1
+ 0 0
+ 2 0
+ 2 3
+ 3 1
+ 0 1
+ 3 0
+ 1 1
+ 1 2
+ 0 0
+ 1 2
+ 2 0
+ 2 1
+ 0 3
+ 3 0
+ 1 3
+ 2 0
+ 2 3
+ 0 1
+ 2 2
+ 3 3
+ 2 1
+ 3 3
+ 0 1
+ 3 0
+ 3 3
+ 3 3
+ 2 3
+ 2 3
+ 2 3
+ 2 1
+ 3 1
+ 2 3
+ 0 3
+ 1 2
+ 0 2
+ 1 0
+ 2 0
+ 1 1
+ 1 2
+ 2 2
+ 0 1
+ 1 0
+ 2 2
+ 1 1
+ 0 0
+ 1 0
+ 1 0
+ 1 0
+ 0 0
+ 0 2
+ 3 2
+ 2 3
+ 3 3
+ 2 1
+ 3 1
+ 0 1
+ 2 0
+ 1 3
+ 1 0
+ 3 2
+ 2 3
+ 0 3
+ 0 0
+ 1 2
+ 2 2
+ 1 1
+ 3 0
+ 2 3
+ 3 1
+ 0 1
+ 3 2
+ 3 3
+ 0 3
+ 0 0
+ 0 0
+ 1 2
+ 0 0
+ 2 0
+ 3 3
+ 0 3
+ 3 0
+ 3 3
+ 0 3
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 2 1
+ 2 3
+ 3 3
+ 2 3
+ 3 3
+ 2 1
+ 2 1
+ 2 1
+ 2 3
+ 3 1
+ 0 1
+ 2 2
+ 2 3
+ 3 1
+ 3 1
+ 2 3
+ 0 3
+ 3 0
+ 2 3
+ 2 3
+ 3 1
+ 3 1
+ 1 1
+ 1 2
+ 3 2
+ 3 1
+ 1 3
+ 3 2
+ 3 3
+ 0 1
+ 2 2
+ 0 1
+ 3 2
+ 3 1
+ 0 1
+ 1 2
+ 2 0
+ 1 3
+ 1 2
+ 1 0
+ 0 0
+ 0 2
+ 0 2
+ 1 0
+ 3 0
+ 2 1
+ 3 1
+ 1 1
+ 1 2
+ 1 2
+ 2 0
+ 2 3
+ 1 3
+ 2 0
+ 2 3
+ 0 3
+ 0 0
+ 2 2
+ 1 3
+ 3 0
+ 3 1
+ 1 3
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 1 2
+ 3 2
+ 3 3
+ 0 1
+ 0 2
+ 1 0
+ 2 0
+ 0 3
+ 2 0
+ 0 3
+ 1 2
+ 1 0
+ 2 2
+ 1 3
+ 2 0
+ 3 3
+ 0 1
+ 3 2
+ 2 3
+ 2 1
+ 0 1
+ 3 0
+ 2 1
+ 3 1
+ 0 3
+ 0 0
+ 3 0
+ 1 3
+ 2 2
+ 0 1
+ 3 0
+ 0 3
+ 0 0
+ 2 2
+ 1 1
+ 2 2
+ 1 1
+ 2 0
+ 2 1
+ 2 1
+ 2 1
+ 2 1
+ 2 1
+ 0 1
+ 1 0
+ 2 0
+ 2 3
+ 0 3
+ 3 2
+ 0 1
+ 0 0
+ 1 0
+ 0 2
+ 0 2
+ 1 0
+ 1 2
+ 3 2
+ 2 1
+ 1 3
+ 2 0
+ 3 3
+ 2 1
+ 1 1
+ 1 2
+ 3 2
+ 3 3
+ 3 1
+ 2 3
+ 2 3
+ 0 1
+ 2 0
+ 1 3
+ 1 2
+ 0 0
+ 3 0
+ 2 1
+ 0 3
+ 1 0
+ 0 2
+ 2 2
+ 0 1
+ 2 0
+ 3 3
+ 0 3
+ 0 2
+ 0 0
+ 1 2
+ 3 2
+ 1 1
+ 0 2
+ 0 0
+ 3 2
+ 2 1
+ 0 1
+ 1 0
+ 0 0
+ 1 0
+ 2 0
+ 0 3
+ 0 0
+ 1 0
+ 2 2
+ 0 3
+ 2 0
+ 3 1
+ 1 1
+ 2 2
+ 3 1
+ 1 3
+ 1 2
+ 0 2
+ 3 0
+ 2 1
+ 2 3
+ 3 3
+ 3 1
+ 1 3
+ 1 2
+ 2 0
+ 2 3
+ 2 1
+ 0 3
+ 3 2
+ 1 3
+ 1 2
+ 0 0
+ 3 0
+ 1 1
+ 2 0
+ 0 3
+ 1 2
+ 0 0
+ 3 2
+ 1 1
+ 2 0
+ 0 1
+ 0 0
+ 2 0
+ 1 1
+ 1 0
+ 3 0
+ 2 1
+ 3 3
+ 0 1
+ 0 2
+ 0 2
+ 0 2
+ 0 2
+ 3 2
+ 3 1
+ 1 3
+ 2 2
+ 3 1
+ 1 3
+ 3 0
+ 0 3
+ 0 2
+ 2 0
+ 3 3
+ 2 3
+ 2 3
+ 3 1
+ 0 3
+ 2 0
+ 1 3
+ 0 2
+ 2 0
+ 0 1
+ 0 0
+ 0 2
+ 2 0
+ 1 3
+ 3 0
+ 3 1
+ 3 1
+ 2 1
+ 2 1
+ 0 1
+ 3 2
+ 2 3
+ 0 3
+ 0 2
+ 3 0
+ 1 1
+ 0 0
+ 1 0
+ 1 2
+ 2 2
+ 1 3
+ 1 2
+ 2 0
+ 2 1
+ 3 3
+ 1 1
+ 2 0
+ 0 1
+ 2 2
+ 0 3
+ 1 0
+ 3 0
+ 3 3
+ 3 3
+ 3 3
+ 0 3
+ 3 2
+ 1 3
+ 3 2
+ 1 3
+ 3 0
+ 1 3
+ 0 0
+ 1 0
+ 3 0
+ 3 1
+ 1 3
+ 2 0
+ 1 3
+ 2 0
+ 3 1
+ 0 1
+ 1 0
+ 0 0
+ 0 2
+ 2 2
+ 2 3
+ 0 3
+ 3 0
+ 1 1
+ 3 2
+ 1 1
+ 2 0
+ 2 3
+ 1 1
+ 1 2
+ 3 2
+ 1 3
+ 3 0
+ 2 1
+ 0 1
+ 2 0
+ 3 3
+ 3 1
+ 2 1
+ 1 3
+ 2 0
+ 1 1
+ 0 0
+ 3 2
+ 2 1
+ 2 1
+ 2 1
+ 0 3
+ 0 2
+ 2 0
+ 2 3
+ 3 1
+ 3 3
+ 1 1
+ 2 2
+ 3 3
+ 2 1
+ 0 3
+ 3 0
+ 2 1
+ 2 1
+ 0 3
+ 0 2
+ 1 0
+ 2 0
+ 0 1
+ 3 0
+ 2 1
+ 0 3
+ 2 0
+ 0 1
+ 3 2
+ 1 3
+ 1 2
+ 2 0
+ 2 1
+ 3 3
+ 1 1
+ 2 2
+ 1 1
+ 3 2
+ 3 1
+ 3 1
+ 3 1
+ 2 3
+ 2 1
+ 3 3
+ 0 1
+ 3 0
+ 0 3
+ 3 0
+ 0 1
+ 3 0
+ 1 3
+ 3 2
+ 1 3
+ 0 2
+ 2 0
+ 2 3
+ 0 1
+ 0 0
+ 0 0
+ 3 0
+ 2 1
+ 3 3
+ 0 3
+ 0 0
+ 2 2
+ 0 3
+ 3 0
+ 3 3
+ 0 1
+ 0 2
+ 0 2
+ 0 2
+ 3 2
+ 1 1
+ 0 0
+ 1 0
+ 2 0
+ 2 1
+ 1 1
+ 3 0
+ 2 1
+ 1 3
+ 1 2
+ 2 2
+ 0 3
+ 2 0
+ 1 1
+ 2 0
+ 3 1
+ 3 1
+ 1 1
+ 2 0
+ 0 1
+ 0 0
+ 3 2
+ 1 1
+ 3 0
+ 3 1
+ 2 3
+ 2 1
+ 2 3
+ 1 3
+ 0 0
+ 1 2
+ 3 0
+ 2 3
+ 2 3
+ 3 3
+ 0 3
+ 3 0
+ 2 1
+ 3 1
+ 1 1
+ 2 0
+ 1 3
+ 2 0
+ 0 3
+ 1 2
+ 0 2
+ 2 2
+ 2 1
+ 2 1
+ 0 1
+ 0 2
+ 2 0
+ 1 1
+ 1 2
+ 3 2
+ 0 3
+ 1 2
+ 3 2
+ 3 1
+ 0 3
+ 2 2
+ 0 1
+ 2 2
+ 3 3
+ 0 1
+ 0 2
+ 3 2
+ 3 3
+ 0 1
+ 1 0
+ 2 2
+ 2 1
+ 0 1
+ 2 2
+ 2 1
+ 3 1
+ 1 3
+ 3 2
+ 0 3
+ 3 2
+ 3 3
+ 3 1
+ 2 3
+ 3 1
+ 3 3
+ 2 3
+ 3 3
+ 3 3
+ 1 3
+ 2 2
+ 0 3
+ 1 0
+ 1 2
+ 3 0
+ 2 3
+ 0 1
+ 1 0
+ 2 2
+ 2 3
+ 1 3
+ 2 2
+ 3 3
+ 0 1
+ 1 2
+ 3 2
+ 0 1
+ 1 2
+ 3 0
+ 3 3
+ 2 3
+ 2 3
+ 2 3
+ 3 3
+ 1 3
+ 2 2
+ 0 3
+ 1 2
+ 2 2
+ 0 3
+ 1 2
+ 1 0
+ 0 2
+ 1 2
+ 2 0
+ 2 3
+ 3 1
+ 3 3
+ 3 1
+ 0 1
+ 2 2
+ 2 3
+ 3 3
+ 3 3
+ 1 3
+ 0 2
+ 2 0
+ 3 3
+ 2 1
+ 0 3
+ 2 2
+ 2 1
+ 2 3
+ 1 1
+ 3 2
+ 1 3
+ 3 2
+ 0 1
+ 1 0
+ 2 2
+ 0 1
+ 3 2
+ 2 3
+ 1 3
+ 2 0
+ 1 3
+ 1 2
+ 3 0
+ 2 1
+ 0 1
+ 0 0
+ 3 0
+ 0 1
+ 3 2
+ 0 3
+ 2 2
+ 0 3
+ 3 0
+ 0 3
+ 0 0
+ 2 0
+ 1 3
+ 1 2
+ 0 2
+ 0 2
+ 1 0
+ 2 2
+ 1 3
+ 1 2
+ 1 2
+ 3 0
+ 2 1
+ 3 3
+ 1 3
+ 1 2
+ 1 2
+ 0 0
+ 3 0
+ 3 3
+ 2 3
+ 0 3
+ 3 0
+ 3 3
+ 0 3
+ 2 0
+ 0 1
+ 1 0
+ 0 2
+ 3 0
+ 1 1
+ 0 0
+ 0 2
+ 3 0
+ 1 3
+ 2 2
+ 1 1
+ 0 0
+ 2 0
+ 0 3
+ 3 0
+ 2 1
+ 0 1
+ 2 0
+ 0 1
+ 2 2
+ 2 1
+ 3 3
+ 2 3
+ 0 3
+ 0 2
+ 0 2
+ 0 0
+ 2 2
+ 2 1
+ 1 3
+ 3 2
+ 0 1
+ 2 0
+ 0 3
+ 1 0
+ 0 0
+ 3 0
+ 1 3
+ 3 2
+ 3 3
+ 3 1
+ 3 1
+ 3 1
+ 3 1
+ 2 1
+ 0 1
+ 1 2
+ 2 0
+ 1 1
+ 0 0
+ 3 2
+ 1 1
+ 2 2
+ 1 3
+ 2 0
+ 2 1
+ 1 3
+ 2 2
+ 0 1
+ 3 2
+ 2 1
+ 1 1
+ 0 2
+ 2 2
+ 2 3
+ 3 1
+ 0 1
+ 0 0
+ 3 0
+ 3 3
+ 0 1
+ 1 0
+ 3 2
+ 2 3
+ 2 1
+ 1 1
+ 2 2
+ 3 1
+ 0 1
+ 3 2
+ 0 1
+ 3 0
+ 2 1
+ 0 3
+ 1 2
+ 0 0
+ 0 0
+ 0 0
+ 0 2
+ 1 0
+ 3 2
+ 1 1
+ 2 2
+ 0 3
+ 1 0
+ 1 2
+ 3 2
+ 0 3
+ 2 2
+ 0 3
+ 0 0
+ 3 2
+ 2 1
+ 3 1
+ 3 3
+ 3 3
+ 1 1
+ 3 2
+ 1 1
+ 1 0
+ 2 2
+ 1 3
+ 3 0
+ 0 3
+ 1 2
+ 0 0
+ 2 2
+ 1 3
+ 3 2
+ 3 1
+ 3 1
+ 1 1
+ 1 0
+ 0 0
+ 0 2
+ 3 0
+ 1 3
+ 3 0
+ 2 3
+ 2 3
+ 3 3
+ 0 3
+ 3 0
+ 1 3
+ 1 2
+ 1 2
+ 1 0
+ 1 0
+ 2 2
+ 0 3
+ 1 2
+ 0 0
+ 2 0
+ 3 3
+ 0 1
+ 0 2
+ 2 2
+ 0 1
+ 2 2
+ 0 1
+ 3 0
+ 3 1
+ 3 3
+ 3 1
+ 2 1
+ 1 1
+ 1 2
+ 0 2
+ 1 2
+ 1 2
+ 2 2
+ 1 1
+ 1 2
+ 1 2
+ 3 2
+ 0 3
+ 3 0
+ 1 3
+ 1 2
+ 3 2
+ 2 1
+ 1 1
+ 0 2
+ 0 0
+ 1 0
+ 1 2
+ 2 2
+ 0 1
+ 2 0
+ 1 3
+ 2 2
+ 1 1
+ 0 2
+ 2 0
+ 0 1
+ 1 2
+ 2 2
+ 1 3
+ 3 0
+ 2 3
+ 0 1
+ 1 2
+ 2 0
+ 1 3
+ 0 0
+ 1 2
+ 1 0
+ 2 2
+ 1 1
+ 0 2
+ 2 0
+ 3 1
+ 1 3
+ 0 2
+ 2 0
+ 2 1
+ 0 3
+ 3 0
+ 2 1
+ 2 1
+ 3 3
+ 2 3
+ 2 3
+ 3 3
+ 2 1
+ 2 3
+ 2 1
+ 2 1
+ 2 1
+ 3 3
+ 3 1
+ 2 1
+ 3 1
+ 1 1
+ 2 0
+ 2 3
+ 1 3
+ 0 2
+ 3 2
+ 0 1
+ 3 2
+ 2 1
+ 2 1
+ 3 1
+ 3 1
+ 0 1
+ 2 2
+ 2 1
+ 1 1
+ 1 2
+ 2 0
+ 3 3
+ 0 1
+ 1 2
+ 0 0
+ 3 0
+ 1 3
+ 1 2
+ 3 2
+ 1 3
+ 3 2
+ 0 1
+ 0 2
+ 0 2
+ 1 2
+ 1 0
+ 3 2
+ 0 1
+ 2 2
+ 2 1
+ 2 3
+ 3 3
+ 1 1
+ 0 0
+ 2 2
+ 1 1
+ 1 2
+ 3 2
+ 0 1
+ 2 2
+ 3 3
+ 2 1
+ 2 3
+ 2 1
+ 2 1
+ 0 1
+ 0 2
+ 2 0
+ 3 1
+ 0 1
+ 2 2
+ 2 1
+ 2 1
+ 3 3
+ 2 1
+ 2 3
+ 1 1
+ 3 2
+ 0 1
+ 0 2
+ 2 2
+ 2 1
+ 1 3
+ 2 2
+ 2 1
+ 1 1
+ 3 2
+ 0 3
+ 1 0
+ 3 0
+ 2 3
+ 2 3
+ 3 1
+ 3 3
+ 2 3
+ 2 3
+ 2 1
+ 1 1
+ 3 2
+ 0 1
+ 3 2
+ 1 1
+ 3 2
+ 1 3
+ 0 2
+ 0 2
+ 2 0
+ 2 3
+ 2 1
+ 3 1
+ 3 1
+ 0 3
+ 1 0
+ 3 2
+ 0 3
+ 2 2
+ 1 1
+ 2 0
+ 1 1
+ 2 0
+ 3 1
+ 2 1
+ 3 1
+ 3 3
+ 1 1
+ 3 0
+ 1 3
+ 2 0
+ 0 3
+ 2 2
+ 1 3
+ 0 0
+ 0 2
+ 0 0
+ 2 0
+ 1 3
+ 2 0
+ 2 1
+ 3 1
+ 2 3
+ 0 1
+ 3 0
+ 2 1
+ 2 1
+ 1 1
+ 0 2
+ 0 2
+ 3 2
+ 3 3
+ 3 3
+ 2 1
+ 0 1
+ 3 0
+ 0 1
+ 0 2
+ 2 2
+ 3 1
+ 1 3
+ 0 0
+ 0 2
+ 2 2
+ 2 1
+ 0 1
+ 0 0
+ 0 0
+ 3 2
+ 0 3
+ 1 0
+ 1 0
+ 1 2
+ 0 0
+ 0 0
+ 3 2
+ 1 3
+ 3 0
+ 3 1
+ 2 3
+ 2 1
+ 1 1
+ 1 2
+ 0 0
+ 1 2
+ 1 2
+ 3 2
+ 1 1
+ 1 0
+ 2 2
+ 3 1
+ 2 3
+ 3 3
+ 3 3
+ 2 3
+ 3 1
+ 0 1
+ 1 0
+ 3 0
+ 2 1
+ 0 3
+ 0 0
+ 3 2
+ 2 1
+ 2 3
+ 0 3
+ 3 2
+ 3 1
+ 0 1
+ 2 2
+ 2 1
+ 2 1
+ 2 3
+ 0 1
+ 2 0
+ 3 3
+ 1 1
+ 2 0
+ 0 1
+ 2 0
+ 3 3
+ 0 1
+ 2 2
+ 0 1
+ 0 0
+ 1 0
+ 1 2
+ 1 2
+ 2 2
+ 1 3
+ 1 2
+ 1 0
+ 3 2
+ 3 1
+ 2 3
+ 1 1
+ 2 2
+ 3 1
+ 2 3
+ 2 1
+ 0 3
+ 3 2
+ 3 1
+ 0 3
+ 3 0
+ 0 1
+ 2 2
+ 2 1
+ 3 1
+ 1 3
+ 1 2
+ 1 0
+ 0 2
+ 3 0
+ 1 1
+ 1 2
+ 2 2
+ 1 3
+ 0 2
+ 2 0
+ 3 3
+ 1 1
+ 1 0
+ 1 0
+ 1 2
+ 1 2
+ 2 0
+ 3 1
+ 0 1
+ 3 0
+ 1 1
+ 0 0
+ 0 2
+ 3 0
+ 3 1
+ 2 1
+ 1 3
+ 2 0
+ 3 1
+ 0 1
+ 0 0
+ 3 2
+ 0 3
+ 3 2
+ 1 1
+ 0 2
+ 1 2
+ 3 0
+ 1 1
+ 1 0
+ 2 2
+ 2 3
+ 1 1
+ 2 2
+ 1 1
+ 2 2
+ 3 1
+ 3 1
+ 3 3
+ 3 3
+ 2 1
+ 1 3
+ 0 0
+ 1 0
+ 2 0
+ 3 1
+ 3 3
+ 2 1
+ 1 3
+ 2 2
+ 3 1
+ 3 3
+ 3 1
+ 0 3
+ 1 0
+ 1 2
+ 1 2
+ 0 0
+ 0 2
+ 1 2
+ 2 0
+ 0 1
+ 1 0
+ 1 0
+ 0 2
+ 0 0
+ 1 0
+ 2 0
+ 3 3
+ 3 3
+ 2 1
+ 3 3
+ 1 1
+ 3 0
+ 0 3
+ 3 0
+ 0 3
+ 0 2
+ 3 2
+ 2 1
+ 1 1
+ 1 2
+ 2 2
+ 0 1
+ 2 2
+ 3 1
+ 1 1
+ 1 2
+ 0 0
+ 2 0
+ 2 3
+ 1 1
+ 0 2
+ 2 0
+ 0 3
+ 3 0
+ 3 3
+ 2 1
+ 0 3
+ 1 2
+ 1 0
+ 2 2
+ 3 3
+ 2 3
+ 1 1
+ 0 2
+ 0 0
+ 1 2
+ 2 0
+ 1 3
+ 0 2
+ 3 2
+ 3 3
+ 2 3
+ 2 1
+ 2 1
+ 2 1
+ 1 1
+ 0 0
+ 1 2
+ 0 2
+ 3 2
+ 1 1
+ 2 0
+ 2 1
+ 1 3
+ 0 2
+ 3 2
+ 3 1
+ 3 1
+ 2 1
+ 2 3
+ 1 1
+ 2 2
+ 2 3
+ 3 3
+ 2 1
+ 2 3
+ 0 1
+ 1 0
+ 1 0
+ 3 2
+ 0 3
+ 1 0
+ 0 0
+ 2 2
+ 3 3
+ 2 3
+ 0 1
+ 3 2
+ 0 1
+ 1 0
+ 0 0
+ 1 2
+ 3 0
+ 1 1
+ 1 0
+ 2 0
+ 2 3
+ 2 3
+ 3 3
+ 1 3
+ 1 2
+ 1 0
+ 3 2
+ 2 1
+ 1 1
+ 0 0
+ 2 0
+ 3 1
+ 1 1
+ 0 0
+ 3 2
+ 0 3
+ 0 0
+ 3 2
+ 1 3
+ 3 0
+ 1 1
+ 2 0
+ 3 1
+ 3 1
+ 0 1
+ 3 2
+ 1 1
+ 2 2
+ 0 1
+ 3 2
+ 0 3
+ 3 0
+ 3 1
+ 2 3
+ 1 1
+ 1 0
+ 1 2
+ 2 2
+ 1 3
+ 0 0
+ 1 0
+ 2 0
+ 3 3
+ 2 1
+ 1 3
+ 0 2
+ 3 2
+ 0 1
+ 1 0
+ 2 0
+ 2 3
+ 3 3
+ 1 3
+ 2 0
+ 2 1
+ 3 1
+ 0 3
+ 0 0
+ 1 0
+ 3 0
+ 2 3
+ 2 3
+ 0 3
+ 2 0
+ 2 3
+ 0 3
+ 0 2
+ 0 2
+ 0 2
+ 2 2
+ 0 3
+ 0 0
+ 2 0
+ 0 1
+ 2 2
+ 3 1
+ 2 3
+ 2 3
+ 1 1
+ 0 0
+ 1 2
+ 1 2
+ 3 2
+ 2 3
+ 2 1
+ 3 3
+ 3 3
+ 2 3
+ 1 3
+ 2 0
+ 3 1
+ 0 3
+ 2 0
+ 3 1
+ 0 1
+ 0 0
+ 0 2
+ 3 0
+ 1 1
+ 0 2
+ 1 2
+ 1 2
+ 3 0
+ 3 3
+ 1 1
+ 1 0
+ 0 2
+ 2 0
+ 2 1
+ 2 3
+ 1 3
+ 0 0
+ 1 2
+ 3 2
+ 2 3
+ 3 3
+ 0 3
+ 0 2
+ 0 0
+ 1 0
+ 2 2
+ 1 3
+ 3 2
+ 1 1
+ 0 0
+ 2 2
+ 2 3
+ 1 3
+ 0 2
+ 3 0
+ 2 3
+ 0 1
+ 1 0
+ 3 0
+ 3 1
+ 0 1
+ 3 0
+ 0 1
+ 3 2
+ 1 1
+ 2 0
+ 2 1
+ 3 1
+ 2 1
+ 2 3
+ 3 3
+ 3 3
+ 3 3
+ 2 1
+ 2 3
+ 0 1
+ 0 2
+ 3 2
+ 1 3
+ 1 2
+ 1 2
+ 0 0
+ 0 2
+ 3 0
+ 2 1
+ 3 1
+ 1 1
+ 2 2
+ 0 1
+ 1 2
+ 2 0
+ 3 1
+ 0 1
+ 2 0
+ 2 1
+ 0 3
+ 2 2
+ 3 3
+ 1 1
+ 1 0
+ 0 2
+ 2 2
+ 0 1
+ 0 0
+ 3 0
+ 1 1
+ 0 0
+ 0 0
+ 2 2
+ 2 1
+ 3 3
+ 3 3
+ 3 1
+ 3 3
+ 1 3
+ 3 2
+ 1 3
+ 3 2
+ 0 1
+ 3 0
+ 2 1
+ 1 3
+ 3 2
+ 0 1
+ 1 0
+ 0 2
+ 2 0
+ 3 3
+ 2 1
+ 0 1
+ 3 2
+ 1 1
+ 0 2
+ 0 2
+ 2 2
+ 1 3
+ 2 0
+ 0 1
+ 3 0
+ 3 1
+ 3 1
+ 3 3
+ 3 1
+ 1 3
+ 0 0
+ 0 0
+ 1 2
+ 1 0
+ 0 2
+ 3 2
+ 2 1
+ 1 3
+ 1 0
+ 1 0
+ 1 2
+ 0 2
+ 2 0
+ 0 3
+ 0 2
+ 3 0
+ 1 3
+ 2 0
+ 3 1
+ 0 3
+ 1 2
+ 1 0
+ 1 2
+ 3 0
+ 2 1
+ 2 3
+ 0 1
+ 2 0
+ 0 3
+ 2 0
+ 3 1
+ 2 1
+ 1 1
+ 3 2
+ 0 1
+ 3 0
+ 0 1
+ 3 0
+ 1 1
+ 2 2
+ 0 1
+ 0 0
+ 2 2
+ 1 1
+ 1 2
+ 3 0
+ 3 1
+ 0 3
+ 3 2
+ 3 3
+ 0 3
+ 0 2
+ 1 2
+ 0 0
+ 1 0
+ 1 0
+ 0 2
+ 2 0
+ 3 3
+ 3 3
+ 1 1
+ 1 2
+ 1 2
+ 0 0
+ 2 0
+ 0 1
+ 0 0
+ 1 2
+ 0 2
+ 2 0
+ 2 3
+ 3 1
+ 3 3
+ 2 1
+ 1 1
+ 3 0
+ 1 1
+ 3 0
+ 2 3
+ 1 1
+ 2 0
+ 0 3
+ 1 0
+ 0 2
+ 2 2
+ 1 3
+ 2 2
+ 2 3
+ 2 1
+ 3 1
+ 3 1
+ 2 1
+ 1 1
+ 1 0
+ 3 2
+ 1 1
+ 3 2
+ 3 1
+ 3 1
+ 0 1
+ 0 2
+ 1 2
+ 0 0
+ 0 0
+ 1 0
+ 1 0
+ 3 0
+ 2 1
+ 3 3
+ 1 1
+ 2 0
+ 0 3
+ 2 0
+ 1 3
+ 2 0
+ 0 3
+ 3 2
+ 2 3
+ 2 1
+ 1 3
+ 2 2
+ 3 1
+ 3 1
+ 1 3
+ 2 0
+ 0 3
+ 2 2
+ 3 3
+ 1 1
+ 3 2
+ 2 3
+ 0 1
+ 0 2
+ 2 2
+ 3 1
+ 1 1
+ 1 0
+ 2 2
+ 2 3
+ 0 3
+ 0 2
+ 1 0
+ 0 0
+ 0 2
+ 3 0
+ 3 1
+ 2 3
+ 2 3
+ 0 1
+ 3 2
+ 0 3
+ 1 0
+ 3 0
+ 3 3
+ 3 1
+ 2 3
+ 3 1
+ 2 3
+ 1 1
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 2 3
+ 2 3
+ 0 3
+ 3 0
+ 3 1
+ 1 1
+ 2 0
+ 2 1
+ 0 3
+ 1 2
+ 0 0
+ 2 2
+ 3 3
+ 2 3
+ 0 3
+ 3 0
+ 0 1
+ 3 2
+ 2 3
+ 3 1
+ 2 3
+ 3 1
+ 0 3
+ 0 0
+ 2 2
+ 1 1
+ 3 0
+ 1 1
+ 2 0
+ 3 1
+ 3 3
+ 2 1
+ 2 3
+ 0 3
+ 2 2
+ 0 1
+ 2 2
+ 0 3
+ 0 2
+ 0 0
+ 3 2
+ 3 3
+ 1 3
+ 1 0
+ 2 2
+ 1 1
+ 2 2
+ 0 1
+ 0 2
+ 3 0
+ 2 3
+ 0 1
+ 0 2
+ 2 2
+ 0 1
+ 0 0
+ 3 2
+ 1 3
+ 2 2
+ 0 1
+ 1 2
+ 0 0
+ 2 0
+ 2 1
+ 1 3
+ 0 2
+ 3 2
+ 3 3
+ 3 3
+ 0 3
+ 2 2
+ 0 3
+ 3 2
+ 0 3
+ 1 2
+ 2 0
+ 3 1
+ 1 3
+ 3 0
+ 3 1
+ 1 3
+ 2 0
+ 0 3
+ 0 2
+ 3 0
+ 3 3
+ 1 3
+ 3 0
+ 1 3
+ 0 0
+ 3 0
+ 3 3
+ 1 3
+ 2 0
+ 0 1
+ 1 2
+ 1 0
+ 3 0
+ 2 3
+ 1 3
+ 3 2
+ 3 3
+ 1 3
+ 3 2
+ 1 3
+ 0 2
+ 1 0
+ 0 2
+ 2 0
+ 0 3
+ 1 2
+ 2 0
+ 3 3
+ 2 3
+ 0 1
+ 0 2
+ 3 2
+ 0 3
+ 3 2
+ 2 3
+ 1 3
+ 1 2
+ 2 0
+ 0 3
+ 1 0
+ 3 0
+ 0 1
+ 1 2
+ 0 0
+ 0 0
+ 2 2
+ 3 1
+ 2 1
+ 1 3
+ 1 2
+ 0 0
+ 3 2
+ 2 1
+ 0 3
+ 2 0
+ 0 3
+ 1 2
+ 0 0
+ 1 0
+ 2 0
+ 3 1
+ 2 3
+ 2 3
+ 0 3
+ 2 2
+ 3 1
+ 3 3
+ 1 1
+ 3 0
+ 2 3
+ 2 1
+ 1 1
+ 0 0
+ 2 2
+ 1 1
+ 0 2
+ 1 0
+ 1 0
+ 1 0
+ 2 2
+ 1 3
+ 1 2
+ 2 0
+ 3 3
+ 1 1
+ 1 0
+ 0 2
+ 2 2
+ 2 1
+ 0 1
+ 1 0
+ 0 0
+ 2 2
+ 0 3
+ 1 0
+ 3 0
+ 2 3
+ 1 1
+ 2 2
+ 3 3
+ 2 1
+ 2 3
+ 0 1
+ 0 0
+ 2 0
+ 2 1
+ 3 1
+ 1 1
+ 0 0
+ 2 2
+ 1 1
+ 1 2
+ 3 0
+ 1 1
+ 1 2
+ 2 2
+ 2 3
+ 2 3
+ 1 3
+ 2 2
+ 3 1
+ 3 1
+ 3 1
+ 2 1
+ 2 3
+ 0 1
+ 3 0
+ 0 1
+ 3 0
+ 1 1
+ 2 2
+ 0 3
+ 2 0
+ 3 1
+ 1 3
+ 2 2
+ 2 3
+ 2 3
+ 1 3
+ 1 0
+ 0 0
+ 0 0
+ 3 2
+ 3 1
+ 0 1
+ 1 0
+ 2 0
+ 0 1
+ 0 0
+ 3 2
+ 1 3
+ 0 0
+ 0 2
+ 2 2
+ 3 1
+ 3 1
+ 0 1
+ 3 2
+ 1 3
+ 0 0
+ 0 0
+ 1 0
+ 0 2
+ 2 0
+ 0 1
+ 0 0
+ 1 0
+ 2 2
+ 0 1
+ 2 2
+ 1 3
+ 3 0
+ 0 3
+ 2 0
+ 1 3
+ 2 0
+ 3 3
+ 1 3
+ 1 2
+ 1 2
+ 2 0
+ 1 1
+ 0 0
+ 2 2
+ 3 3
+ 0 1
+ 3 0
+ 2 1
+ 3 3
+ 3 3
+ 0 3
+ 1 2
+ 1 0
+ 3 2
+ 1 1
+ 0 0
+ 1 2
+ 0 2
+ 0 2
+ 1 2
+ 3 0
+ 1 3
+ 2 0
+ 1 1
+ 2 2
+ 3 1
+ 3 3
+ 2 3
+ 2 3
+ 2 3
+ 0 1
+ 1 0
+ 2 0
+ 1 1
+ 1 2
+ 1 0
+ 2 2
+ 1 3
+ 2 2
+ 1 1
+ 3 2
+ 1 1
+ 1 2
+ 1 0
+ 0 2
+ 0 0
+ 2 2
+ 2 3
+ 3 3
+ 2 3
+ 3 1
+ 1 3
+ 2 0
+ 1 1
+ 3 0
+ 2 1
+ 2 3
+ 2 1
+ 0 1
+ 0 2
+ 0 0
+ 1 0
+ 1 2
+ 3 0
+ 2 3
+ 1 3
+ 2 0
+ 1 1
+ 2 0
+ 3 3
+ 1 3
+ 2 2
+ 1 3
+ 3 0
+ 0 1
+ 2 0
+ 3 3
+ 0 3
+ 1 2
+ 1 2
+ 0 2
+ 3 2
+ 3 3
+ 0 3
+ 0 0
+ 1 0
+ 0 0
+ 1 0
+ 2 0
+ 3 3
+ 1 3
+ 1 0
+ 1 2
+ 0 2
+ 1 2
+ 2 0
+ 1 1
+ 2 2
+ 0 3
+ 2 2
+ 0 3
+ 1 0
+ 0 2
+ 2 2
+ 0 1
+ 2 0
+ 1 3
+ 2 0
+ 1 1
+ 1 2
+ 1 0
+ 2 2
+ 0 1
+ 1 2
+ 3 0
+ 1 1
+ 2 0
+ 0 3
+ 0 2
+ 2 2
+ 1 3
+ 2 2
+ 1 1
+ 0 2
+ 0 2
+ 2 2
+ 0 1
+ 1 2
+ 1 2
+ 1 0
+ 1 0
+ 0 2
+ 0 2
+ 2 0
+ 1 3
+ 3 2
+ 2 3
+ 2 1
+ 1 1
+ 0 2
+ 3 0
+ 1 3
+ 2 0
+ 2 1
+ 1 3
+ 1 2
+ 0 2
+ 0 0
+ 2 0
+ 3 3
+ 1 3
+ 0 2
+ 3 0
+ 1 3
+ 0 0
+ 2 0
+ 1 3
+ 2 0
+ 3 1
+ 3 3
+ 1 1
+ 3 0
+ 1 1
+ 1 0
+ 3 0
+ 2 1
+ 0 1
+ 0 2
+ 3 2
+ 0 1
+ 3 0
+ 2 3
+ 1 3
+ 2 0
+ 1 3
+ 1 0
+ 0 0
+ 0 2
+ 2 0
+ 1 3
+ 1 0
+ 2 2
+ 1 1
+ 0 0
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 0 0
+ 1 2
+ 1 2
+ 0 0
+ 0 0
+ 2 0
+ 1 1
+ 3 2
+ 0 1
+ 1 0
+ 0 2
+ 1 0
+ 0 2
+ 1 0
+ 1 2
+ 0 0
+ 2 0
+ 0 1
+ 0 2
+ 1 0
+ 1 0
+ 0 0
+ 1 2
+ 3 0
+ 3 3
+ 1 1
+ 3 0
+ 3 1
+ 1 3
+ 0 2
+ 1 0
+ 3 0
+ 2 3
+ 2 1
+ 3 1
+ 3 1
+ 0 1
+ 3 2
+ 2 3
+ 2 1
+ 3 1
+ 0 1
+ 2 2
+ 1 1
+ 0 2
+ 3 0
+ 0 3
+ 0 2
+ 3 2
+ 0 1
+ 2 0
+ 0 3
+ 0 2
+ 1 0
+ 3 0
+ 1 3
+ 1 0
+ 3 0
+ 3 3
+ 3 3
+ 0 1
+ 3 2
+ 2 1
+ 1 1
+ 2 2
+ 1 3
+ 1 2
+ 3 0
+ 2 3
+ 0 1
+ 2 0
+ 2 3
+ 2 1
+ 1 3
+ 0 2
+ 1 0
+ 2 2
+ 1 3
+ 3 0
+ 3 1
+ 3 3
+ 1 3
+ 3 2
+ 2 3
+ 1 1
+ 0 0
+ 0 2
+ 2 0
+ 1 1
+ 3 0
+ 0 1
+ 2 2
+ 1 3
+ 2 2
+ 1 3
+ 0 0
+ 2 0
+ 3 3
+ 3 1
+ 2 3
+ 2 1
+ 3 3
+ 1 3
+ 3 0
+ 3 1
+ 0 3
+ 0 0
+ 0 2
+ 2 0
+ 2 1
+ 1 3
+ 2 2
+ 2 3
+ 2 1
+ 0 3
+ 1 2
+ 2 2
+ 3 1
+ 3 3
+ 3 3
+ 3 3
+ 3 1
+ 2 3
+ 2 3
+ 1 3
+ 1 2
+ 2 0
+ 1 3
+ 0 2
+ 2 2
+ 1 3
+ 2 2
+ 0 1
+ 1 0
+ 2 0
+ 2 1
+ 3 3
+ 2 3
+ 2 3
+ 2 1
+ 0 3
+ 2 0
+ 3 3
+ 0 3
+ 1 0
+ 2 0
+ 2 3
+ 2 1
+ 1 1
+ 3 2
+ 3 1
+ 2 1
+ 0 3
+ 1 2
+ 1 2
+ 3 0
+ 2 1
+ 0 1
+ 3 2
+ 2 3
+ 1 1
+ 0 0
+ 3 0
+ 1 3
+ 1 0
+ 3 2
+ 0 3
+ 0 0
+ 3 2
+ 3 1
+ 0 3
+ 1 2
+ 3 0
+ 0 1
+ 2 2
+ 2 1
+ 2 3
+ 0 3
+ 3 2
+ 3 3
+ 2 3
+ 3 3
+ 2 3
+ 1 1
+ 1 0
+ 2 0
+ 3 3
+ 2 3
+ 1 3
+ 3 2
+ 1 3
+ 1 2
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 1 3
+ 3 0
+ 2 3
+ 2 3
+ 2 3
+ 0 1
+ 0 2
+ 1 2
+ 1 0
+ 2 2
+ 2 1
+ 2 1
+ 1 1
+ 0 0
+ 3 0
+ 0 3
+ 3 2
+ 0 3
+ 0 2
+ 3 2
+ 0 1
+ 0 0
+ 1 0
+ 3 2
+ 1 1
+ 1 0
+ 3 2
+ 3 3
+ 3 1
+ 0 3
+ 3 2
+ 0 1
+ 0 2
+ 2 0
+ 3 1
+ 2 3
+ 1 3
+ 0 2
+ 0 0
+ 1 2
+ 3 2
+ 1 3
+ 0 2
+ 1 2
+ 0 2
+ 1 0
+ 0 2
+ 2 0
+ 1 1
+ 1 0
+ 3 2
+ 3 1
+ 2 1
+ 3 3
+ 3 1
+ 1 3
+ 0 0
+ 1 0
+ 1 2
+ 0 2
+ 0 2
+ 0 2
+ 1 0
+ 2 2
+ 3 1
+ 1 1
+ 3 0
+ 1 1
+ 0 2
+ 0 2
+ 1 0
+ 3 2
+ 0 1
+ 1 0
+ 0 2
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 1 1
+ 0 2
+ 0 2
+ 2 0
+ 1 1
+ 3 2
+ 3 3
+ 1 1
+ 3 0
+ 0 1
+ 0 0
+ 1 0
+ 3 2
+ 1 3
+ 0 2
+ 2 0
+ 1 3
+ 3 0
+ 2 3
+ 1 3
+ 0 2
+ 0 0
+ 0 2
+ 2 0
+ 3 3
+ 0 3
+ 1 0
+ 0 0
+ 2 0
+ 2 1
+ 1 3
+ 3 2
+ 2 3
+ 0 1
+ 2 2
+ 3 3
+ 1 3
+ 2 2
+ 2 1
+ 2 1
+ 1 1
+ 2 0
+ 1 3
+ 2 2
+ 0 1
+ 2 0
+ 3 1
+ 2 1
+ 0 3
+ 0 0
+ 1 0
+ 1 2
+ 0 2
+ 2 0
+ 1 3
+ 3 0
+ 1 3
+ 2 2
+ 3 3
+ 2 1
+ 0 3
+ 3 0
+ 2 1
+ 2 3
+ 3 3
+ 0 3
+ 3 2
+ 0 1
+ 0 2
+ 0 0
+ 3 0
+ 2 3
+ 3 1
+ 0 3
+ 0 2
+ 3 0
+ 2 3
+ 0 3
+ 1 0
+ 0 2
+ 1 2
+ 2 0
+ 1 1
+ 0 2
+ 3 2
+ 1 1
+ 0 0
+ 1 0
+ 3 0
+ 3 3
+ 3 1
+ 1 1
+ 0 0
+ 0 0
+ 2 2
+ 2 1
+ 1 1
+ 2 2
+ 0 1
+ 0 2
+ 2 0
+ 3 1
+ 3 3
+ 3 1
+ 1 3
+ 0 2
+ 3 0
+ 0 1
+ 3 2
+ 3 3
+ 3 3
+ 3 3
+ 3 1
+ 0 1
+ 3 0
+ 2 3
+ 2 3
+ 3 3
+ 1 1
+ 1 0
+ 2 2
+ 1 1
+ 0 2
+ 2 2
+ 2 1
+ 0 3
+ 2 0
+ 1 1
+ 1 0
+ 3 2
+ 3 1
+ 1 1
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 2 3
+ 2 3
+ 3 3
+ 2 1
+ 2 3
+ 3 3
+ 1 1
+ 0 0
+ 1 2
+ 1 2
+ 1 0
+ 0 2
+ 2 2
+ 2 1
+ 2 3
+ 0 1
+ 3 2
+ 3 1
+ 1 3
+ 3 2
+ 0 3
+ 0 2
+ 0 0
+ 1 2
+ 0 0
+ 0 2
+ 1 2
+ 1 0
+ 3 2
+ 2 1
+ 2 1
+ 0 1
+ 1 2
+ 2 0
+ 0 3
+ 0 2
+ 1 2
+ 1 0
+ 2 0
+ 0 1
+ 0 2
+ 2 0
+ 0 1
+ 3 0
+ 1 3
+ 3 2
+ 0 3
+ 0 2
+ 2 0
+ 1 1
+ 3 0
+ 2 1
+ 0 3
+ 0 2
+ 1 0
+ 2 0
+ 3 1
+ 1 1
+ 2 2
+ 1 1
+ 0 0
+ 0 2
+ 2 0
+ 2 1
+ 1 1
+ 0 0
+ 3 0
+ 2 3
+ 2 3
+ 2 1
+ 0 1
+ 2 0
+ 1 1
+ 2 0
+ 3 3
+ 3 3
+ 2 3
+ 0 1
+ 3 2
+ 0 1
+ 0 0
+ 0 0
+ 1 2
+ 0 0
+ 2 2
+ 2 3
+ 1 3
+ 2 0
+ 0 1
+ 1 0
+ 1 2
+ 0 0
+ 2 0
+ 0 3
+ 2 2
+ 0 1
+ 2 0
+ 0 3
+ 2 0
+ 1 1
+ 2 2
+ 3 3
+ 1 3
+ 0 0
+ 2 0
+ 1 3
+ 3 0
+ 2 3
+ 1 3
+ 1 0
+ 1 0
+ 1 2
+ 3 2
+ 1 1
+ 0 0
+ 1 2
+ 2 2
+ 3 3
+ 3 3
+ 2 3
+ 0 1
+ 1 2
+ 0 2
+ 1 0
+ 3 2
+ 0 3
+ 0 0
+ 1 2
+ 0 2
+ 3 2
+ 0 3
+ 3 2
+ 1 3
+ 1 0
+ 0 2
+ 1 0
+ 3 0
+ 2 3
+ 3 3
+ 1 1
+ 0 0
+ 1 2
+ 0 2
+ 1 0
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 0 2
+ 3 0
+ 0 3
+ 2 0
+ 2 1
+ 0 3
+ 3 2
+ 2 3
+ 3 3
+ 2 3
+ 1 1
+ 3 2
+ 3 1
+ 3 3
+ 3 3
+ 0 1
+ 0 2
+ 3 0
+ 3 3
+ 1 3
+ 0 2
+ 2 0
+ 1 1
+ 0 0
+ 1 2
+ 1 2
+ 3 2
+ 2 3
+ 1 3
+ 1 0
+ 2 0
+ 3 3
+ 2 1
+ 1 1
+ 2 0
+ 2 1
+ 3 1
+ 0 1
+ 2 0
+ 0 1
+ 2 2
+ 0 1
+ 3 0
+ 2 1
+ 1 3
+ 2 0
+ 2 1
+ 3 1
+ 0 3
+ 1 0
+ 3 2
+ 0 1
+ 1 2
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 2 0
+ 3 1
+ 2 3
+ 0 1
+ 1 2
+ 2 0
+ 2 1
+ 3 1
+ 3 1
+ 3 3
+ 1 3
+ 3 2
+ 1 3
+ 3 0
+ 0 3
+ 1 0
+ 2 0
+ 3 1
+ 0 1
+ 3 2
+ 1 1
+ 3 2
+ 2 3
+ 3 1
+ 3 1
+ 3 1
+ 3 3
+ 2 1
+ 3 3
+ 2 1
+ 0 3
+ 3 0
+ 3 1
+ 2 1
+ 3 1
+ 0 3
+ 2 0
+ 0 1
+ 2 2
+ 2 1
+ 0 3
+ 1 0
+ 1 2
+ 2 0
+ 1 3
+ 1 2
+ 2 0
+ 1 1
+ 1 0
+ 2 0
+ 3 1
+ 0 1
+ 1 0
+ 2 0
+ 3 3
+ 2 1
+ 0 1
+ 2 2
+ 0 1
+ 1 2
+ 3 0
+ 0 3
+ 3 0
+ 2 3
+ 2 3
+ 1 1
+ 2 0
+ 0 1
+ 2 0
+ 1 1
+ 0 2
+ 2 0
+ 1 3
+ 0 2
+ 2 2
+ 0 1
+ 0 2
+ 1 0
+ 3 2
+ 3 1
+ 0 1
+ 1 2
+ 3 2
+ 0 1
+ 1 2
+ 1 0
+ 2 2
+ 3 1
+ 2 3
+ 3 3
+ 2 1
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 0 1
+ 1 0
+ 1 0
+ 1 0
+ 0 2
+ 1 2
+ 0 2
+ 3 2
+ 2 3
+ 0 1
+ 0 0
+ 0 0
+ 0 2
+ 0 0
+ 3 0
+ 0 1
+ 2 0
+ 1 3
+ 1 2
+ 0 0
+ 2 0
+ 3 1
+ 3 1
+ 2 3
+ 2 3
+ 3 3
+ 0 1
+ 1 0
+ 3 0
+ 3 1
+ 3 1
+ 1 3
+ 3 2
+ 1 3
+ 0 0
+ 0 0
+ 2 2
+ 3 3
+ 1 3
+ 2 2
+ 0 3
+ 2 2
+ 2 3
+ 1 1
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 3 1
+ 0 1
+ 3 2
+ 0 3
+ 0 2
+ 1 0
+ 2 0
+ 2 3
+ 3 3
+ 3 1
+ 2 3
+ 0 3
+ 0 2
+ 0 0
+ 2 0
+ 1 3
+ 1 2
+ 2 2
+ 0 3
+ 2 2
+ 3 3
+ 2 1
+ 0 3
+ 3 2
+ 3 1
+ 3 3
+ 2 3
+ 3 3
+ 3 1
+ 3 1
+ 1 1
+ 0 0
+ 3 0
+ 0 1
+ 1 0
+ 0 2
+ 1 2
+ 1 0
+ 1 0
+ 1 0
+ 1 0
+ 1 0
+ 3 2
+ 0 3
+ 0 0
+ 1 2
+ 2 2
+ 2 3
+ 3 1
+ 0 3
+ 1 2
+ 0 2
+ 0 0
+ 3 2
+ 3 1
+ 3 3
+ 0 1
+ 3 0
+ 1 1
+ 3 2
+ 2 1
+ 3 3
+ 3 3
+ 0 1
+ 1 2
+ 2 2
+ 3 3
+ 1 3
+ 0 2
+ 1 2
+ 3 2
+ 2 1
+ 0 3
+ 0 0
+ 0 0
+ 0 2
+ 0 0
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 0 1
+ 1 2
+ 1 0
+ 1 2
+ 1 0
+ 2 0
+ 3 3
+ 3 3
+ 2 1
+ 0 3
+ 0 2
+ 1 2
+ 1 0
+ 1 2
+ 1 0
+ 2 2
+ 3 3
+ 2 3
+ 2 1
+ 0 1
+ 2 0
+ 0 3
+ 0 0
+ 2 0
+ 1 3
+ 3 2
+ 0 1
+ 1 2
+ 0 2
+ 3 2
+ 0 1
+ 3 2
+ 3 1
+ 3 3
+ 0 1
+ 1 2
+ 3 0
+ 3 1
+ 3 1
+ 0 1
+ 3 0
+ 2 3
+ 1 3
+ 0 2
+ 2 0
+ 0 1
+ 0 2
+ 0 0
+ 0 2
+ 0 0
+ 1 0
+ 0 0
+ 3 0
+ 0 3
+ 3 0
+ 0 1
+ 1 2
+ 2 0
+ 3 1
+ 0 3
+ 2 0
+ 2 1
+ 0 1
+ 1 2
+ 1 0
+ 2 0
+ 2 3
+ 2 3
+ 2 1
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 2 3
+ 1 1
+ 3 0
+ 1 3
+ 2 0
+ 2 1
+ 1 1
+ 2 2
+ 1 3
+ 0 2
+ 0 2
+ 1 0
+ 1 0
+ 2 2
+ 2 1
+ 2 1
+ 1 1
+ 2 0
+ 2 3
+ 0 1
+ 0 0
+ 3 0
+ 2 3
+ 1 1
+ 3 0
+ 2 1
+ 3 1
+ 2 3
+ 0 3
+ 0 2
+ 0 2
+ 3 2
+ 0 3
+ 1 2
+ 0 2
+ 3 2
+ 2 1
+ 2 3
+ 2 3
+ 1 1
+ 2 0
+ 0 1
+ 1 0
+ 0 0
+ 0 2
+ 0 2
+ 2 2
+ 3 3
+ 3 3
+ 1 1
+ 2 2
+ 2 1
+ 3 3
+ 2 3
+ 3 1
+ 2 3
+ 1 3
+ 0 0
+ 2 2
+ 3 1
+ 1 3
+ 0 0
+ 1 2
+ 0 0
+ 1 0
+ 3 0
+ 0 3
+ 0 2
+ 3 0
+ 0 3
+ 3 0
+ 2 1
+ 1 1
+ 3 0
+ 2 1
+ 0 3
+ 0 2
+ 0 2
+ 3 2
+ 0 1
+ 3 2
+ 2 1
+ 1 1
+ 0 0
+ 2 0
+ 3 1
+ 2 1
+ 1 1
+ 2 2
+ 2 3
+ 0 3
+ 2 2
+ 3 3
+ 2 3
+ 1 1
+ 2 0
+ 0 3
+ 0 0
+ 3 2
+ 2 1
+ 1 3
+ 2 0
+ 2 1
+ 2 1
+ 1 3
+ 2 2
+ 1 3
+ 2 2
+ 0 1
+ 3 0
+ 0 1
+ 1 2
+ 0 0
+ 0 2
+ 0 2
+ 3 2
+ 3 3
+ 0 1
+ 1 0
+ 3 2
+ 2 3
+ 2 1
+ 2 3
+ 2 1
+ 1 3
+ 0 0
+ 1 2
+ 1 2
+ 1 0
+ 3 2
+ 1 1
+ 1 0
+ 0 2
+ 1 0
+ 3 2
+ 3 1
+ 2 3
+ 2 1
+ 3 1
+ 0 3
+ 1 2
+ 3 0
+ 2 1
+ 2 1
+ 1 1
+ 2 0
+ 1 3
+ 0 2
+ 0 2
+ 1 0
+ 0 0
+ 3 0
+ 0 3
+ 3 0
+ 3 1
+ 2 3
+ 2 3
+ 2 3
+ 2 3
+ 3 1
+ 3 1
+ 1 3
+ 2 2
+ 3 1
+ 1 1
+ 1 2
+ 0 2
+ 1 2
+ 0 2
+ 0 2
+ 2 0
+ 1 3
+ 0 2
+ 3 2
+ 1 1
+ 3 2
+ 0 3
+ 0 2
+ 0 0
+ 1 2
+ 1 0
+ 1 2
+ 3 2
+ 3 1
+ 0 3
+ 0 0
+ 2 0
+ 0 1
+ 2 2
+ 0 3
+ 3 2
+ 3 3
+ 2 3
+ 2 3
+ 0 1
+ 2 2
+ 2 1
+ 1 3
+ 2 2
+ 0 1
+ 0 0
+ 0 2
+ 2 0
+ 0 1
+ 1 2
+ 2 0
+ 1 1
+ 3 0
+ 0 3
+ 0 0
+ 0 0
+ 2 0
+ 0 1
+ 0 2
+ 2 2
+ 1 1
+ 0 0
+ 3 2
+ 3 1
+ 1 3
+ 3 2
+ 0 3
+ 3 2
+ 0 1
+ 0 2
+ 0 0
+ 3 0
+ 2 1
+ 0 3
+ 0 0
+ 3 2
+ 3 1
+ 1 1
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 0 0
+ 1 2
+ 2 2
+ 3 3
+ 2 3
+ 0 1
+ 2 2
+ 3 3
+ 0 1
+ 3 2
+ 1 3
+ 2 2
+ 0 1
+ 1 2
+ 2 0
+ 3 3
+ 1 3
+ 1 0
+ 1 0
+ 2 0
+ 3 1
+ 1 1
+ 3 2
+ 2 1
+ 3 1
+ 2 3
+ 2 1
+ 3 3
+ 2 3
+ 0 3
+ 3 0
+ 2 3
+ 2 3
+ 0 3
+ 2 2
+ 1 1
+ 0 2
+ 2 0
+ 3 1
+ 2 3
+ 3 3
+ 1 1
+ 1 0
+ 0 0
+ 2 2
+ 1 3
+ 1 2
+ 0 2
+ 1 0
+ 1 0
+ 1 2
+ 0 2
+ 1 2
+ 0 2
+ 3 2
+ 2 3
+ 3 1
+ 0 3
+ 2 0
+ 2 3
+ 0 3
+ 3 2
+ 2 3
+ 1 3
+ 2 0
+ 1 3
+ 0 0
+ 0 2
+ 1 2
+ 1 2
+ 0 0
+ 2 0
+ 2 1
+ 2 3
+ 2 3
+ 0 3
+ 2 2
+ 1 3
+ 1 2
+ 0 0
+ 0 2
+ 1 0
+ 0 2
+ 1 0
+ 2 2
+ 3 3
+ 2 3
+ 1 1
+ 0 0
+ 0 2
+ 3 0
+ 1 1
+ 0 2
+ 0 2
+ 1 0
+ 1 0
+ 0 2
+ 2 0
+ 0 3
+ 2 0
+ 2 3
+ 1 1
+ 3 2
+ 3 1
+ 3 3
+ 1 3
+ 0 2
+ 1 0
+ 0 0
+ 1 2
+ 0 0
+ 3 2
+ 0 3
+ 0 2
+ 0 0
+ 1 2
+ 3 2
+ 1 3
+ 2 2
+ 2 3
+ 0 3
+ 3 0
+ 0 3
+ 1 2
+ 0 2
+ 0 0
+ 0 0
+ 1 2
+ 0 0
+ 0 2
+ 2 0
+ 3 1
+ 1 3
+ 0 0
+ 1 2
+ 1 0
+ 3 2
+ 0 3
+ 2 0
+ 2 1
+ 0 3
+ 2 2
+ 2 3
+ 0 3
+ 1 0
+ 3 2
+ 2 1
+ 0 1
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 2 2
+ 2 1
+ 1 3
+ 2 2
+ 3 1
+ 1 1
+ 1 2
+ 1 0
+ 2 2
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 1 2
+ 2 2
+ 3 3
+ 1 3
+ 3 2
+ 3 3
+ 2 3
+ 2 1
+ 1 3
+ 0 2
+ 3 0
+ 1 3
+ 2 0
+ 0 1
+ 1 0
+ 2 0
+ 3 1
+ 3 3
+ 2 1
+ 0 3
+ 1 2
+ 3 2
+ 1 1
+ 1 2
+ 1 2
+ 1 0
+ 1 0
+ 0 2
+ 3 2
+ 2 1
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 0 2
+ 1 2
+ 3 2
+ 0 1
+ 3 2
+ 0 3
+ 0 0
+ 0 2
+ 3 2
+ 0 1
+ 0 2
+ 0 0
+ 2 0
+ 0 3
+ 0 2
+ 1 0
+ 0 2
+ 2 0
+ 2 1
+ 0 1
+ 3 2
+ 3 1
+ 2 1
+ 3 1
+ 1 1
+ 1 2
+ 2 0
+ 2 3
+ 0 3
+ 0 2
+ 0 0
+ 3 0
+ 1 1
+ 2 0
+ 3 1
+ 0 3
+ 3 0
+ 3 3
+ 3 1
+ 3 1
+ 0 1
+ 3 0
+ 3 1
+ 1 1
+ 0 0
+ 2 2
+ 0 1
+ 1 2
+ 2 2
+ 0 3
+ 0 2
+ 0 0
+ 1 0
+ 0 2
+ 0 0
+ 2 2
+ 2 1
+ 0 1
+ 0 0
+ 3 0
+ 3 1
+ 0 1
+ 1 2
+ 2 2
+ 2 3
+ 1 3
+ 3 2
+ 3 3
+ 1 3
+ 3 0
+ 1 1
+ 2 2
+ 3 3
+ 1 1
+ 3 0
+ 2 3
+ 3 3
+ 2 1
+ 0 1
+ 3 0
+ 0 1
+ 3 2
+ 1 1
+ 0 2
+ 1 2
+ 0 2
+ 2 2
+ 3 1
+ 1 1
+ 0 0
+ 1 0
+ 3 2
+ 1 3
+ 0 2
+ 0 0
+ 1 2
+ 3 0
+ 2 1
+ 1 1
+ 0 0
+ 1 0
+ 0 2
+ 0 0
+ 1 0
+ 2 2
+ 1 3
+ 2 2
+ 0 1
+ 2 0
+ 3 1
+ 3 1
+ 3 1
+ 0 3
+ 2 0
+ 0 1
+ 0 2
+ 3 2
+ 2 1
+ 3 3
+ 1 3
+ 0 0
+ 1 0
+ 1 2
+ 3 0
+ 2 1
+ 0 3
+ 0 2
+ 0 2
+ 1 2
+ 0 0
+ 3 0
+ 3 1
+ 0 1
+ 1 0
+ 1 0
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 3 0
+ 0 1
+ 0 0
+ 3 0
+ 1 1
+ 1 2
+ 3 0
+ 3 1
+ 1 1
+ 0 2
+ 1 2
+ 0 2
+ 0 2
+ 1 2
+ 3 2
+ 1 1
+ 3 0
+ 3 3
+ 3 1
+ 2 1
+ 0 3
+ 0 0
+ 3 2
+ 0 3
+ 1 2
+ 2 2
+ 1 3
+ 0 0
+ 3 0
+ 2 1
+ 1 3
+ 1 2
+ 1 0
+ 2 0
+ 3 1
+ 1 1
+ 3 2
+ 2 1
+ 3 3
+ 0 1
+ 0 0
+ 3 0
+ 1 1
+ 2 2
+ 0 1
+ 0 2
+ 2 2
+ 1 3
+ 1 2
+ 3 2
+ 0 1
+ 2 0
+ 3 3
+ 1 3
+ 3 0
+ 0 3
+ 2 2
+ 1 3
+ 2 0
+ 2 3
+ 3 1
+ 3 1
+ 0 1
+ 3 2
+ 1 3
+ 1 0
+ 1 0
+ 3 0
+ 2 1
+ 3 1
+ 3 1
+ 2 1
+ 0 3
+ 0 2
+ 2 0
+ 3 1
+ 0 1
+ 3 0
+ 0 3
+ 1 0
+ 0 2
+ 3 0
+ 1 3
+ 0 2
+ 2 2
+ 3 1
+ 3 1
+ 2 3
+ 0 3
+ 0 2
+ 2 2
+ 3 1
+ 3 1
+ 3 3
+ 2 3
+ 1 3
+ 0 0
+ 2 0
+ 3 1
+ 3 3
+ 0 3
+ 2 0
+ 1 3
+ 2 0
+ 1 3
+ 2 0
+ 3 1
+ 0 3
+ 1 2
+ 1 2
+ 2 0
+ 2 1
+ 3 1
+ 3 3
+ 2 1
+ 0 3
+ 1 0
+ 2 2
+ 0 1
+ 2 0
+ 3 3
+ 1 3
+ 2 2
+ 1 1
+ 2 2
+ 1 1
+ 1 0
+ 2 2
+ 1 3
+ 2 2
+ 2 1
+ 1 3
+ 2 2
+ 0 3
+ 2 0
+ 3 3
+ 1 3
+ 2 0
+ 0 1
+ 0 0
+ 0 0
+ 0 0
+ 2 0
+ 1 1
+ 0 2
+ 1 0
+ 1 0
+ 3 2
+ 0 1
+ 3 0
+ 3 1
+ 3 1
+ 3 1
+ 2 1
+ 0 3
+ 2 2
+ 0 3
+ 0 2
+ 0 0
+ 0 0
+ 1 0
+ 0 2
+ 3 2
+ 3 1
+ 1 1
+ 3 2
+ 3 3
+ 3 1
+ 1 1
+ 1 0
+ 3 0
+ 3 1
+ 3 3
+ 2 1
+ 2 3
+ 3 1
+ 0 3
+ 2 2
+ 0 3
+ 1 2
+ 0 0
+ 3 2
+ 2 1
+ 1 1
+ 0 0
+ 0 2
+ 3 2
+ 0 1
+ 3 0
+ 1 3
+ 0 0
+ 1 2
+ 3 0
+ 0 1
+ 1 2
+ 0 2
+ 1 0
+ 1 2
+ 1 0
+ 1 2
+ 3 2
+ 2 3
+ 3 1
+ 1 3
+ 3 2
+ 1 3
+ 0 2
+ 2 2
+ 2 1
+ 1 1
+ 1 0
+ 1 2
+ 2 0
+ 3 1
+ 0 3
+ 3 2
+ 1 1
+ 3 0
+ 1 1
+ 1 0
+ 0 0
+ 1 0
+ 3 0
+ 0 1
+ 2 0
+ 2 3
+ 2 1
+ 3 3
+ 3 1
+ 1 1
+ 2 0
+ 1 1
+ 1 2
+ 1 0
+ 1 0
+ 1 0
+ 1 2
+ 1 2
+ 0 2
+ 0 0
+ 3 0
+ 0 3
+ 2 0
+ 3 1
+ 3 1
+ 2 3
+ 0 1
+ 0 2
+ 3 0
+ 1 3
+ 1 2
+ 2 0
+ 3 3
+ 3 1
+ 2 3
+ 2 1
+ 0 1
+ 3 0
+ 1 1
+ 2 0
+ 1 1
+ 0 2
+ 1 0
+ 0 0
+ 1 0
+ 3 0
+ 1 1
+ 0 2
+ 2 2
+ 0 1
+ 3 0
+ 3 1
+ 0 3
+ 3 2
+ 2 3
+ 3 3
+ 1 1
+ 0 0
+ 3 2
+ 3 1
+ 1 1
+ 0 2
+ 3 0
+ 3 1
+ 2 1
+ 1 1
+ 1 2
+ 2 0
+ 1 1
+ 3 2
+ 1 1
+ 2 2
+ 0 1
+ 2 0
+ 2 3
+ 1 1
+ 1 0
+ 2 2
+ 0 3
+ 0 0
+ 1 0
+ 0 2
+ 3 0
+ 3 3
+ 1 3
+ 2 0
+ 1 3
+ 0 0
+ 0 0
+ 1 0
+ 2 2
+ 1 3
+ 3 2
+ 2 3
+ 1 1
+ 2 0
+ 3 1
+ 0 3
+ 0 0
+ 1 0
+ 1 0
+ 0 2
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 1 0
+ 3 0
+ 0 1
+ 3 2
+ 2 3
+ 1 1
+ 2 2
+ 2 1
+ 3 1
+ 1 3
+ 2 0
+ 3 3
+ 0 1
+ 2 0
+ 2 3
+ 1 3
+ 1 2
+ 2 2
+ 2 3
+ 1 3
+ 2 2
+ 0 3
+ 2 0
+ 3 3
+ 2 1
+ 1 1
+ 1 2
+ 3 0
+ 2 3
+ 3 1
+ 3 1
+ 2 1
+ 3 3
+ 2 1
+ 2 3
+ 3 3
+ 3 3
+ 2 1
+ 2 3
+ 1 3
+ 0 0
+ 1 2
+ 2 0
+ 0 1
+ 0 0
+ 1 0
+ 0 2
+ 0 2
+ 1 0
+ 2 2
+ 0 1
+ 2 2
+ 2 3
+ 2 1
+ 1 3
+ 2 2
+ 2 1
+ 1 1
+ 3 0
+ 1 1
+ 0 2
+ 1 0
+ 1 2
+ 3 0
+ 0 1
+ 0 2
+ 0 2
+ 3 0
+ 1 1
+ 1 0
+ 1 0
+ 3 2
+ 2 1
+ 3 3
+ 3 3
+ 3 3
+ 0 1
+ 3 0
+ 3 1
+ 0 1
+ 0 2
+ 1 2
+ 3 0
+ 3 1
+ 3 1
+ 3 3
+ 1 1
+ 1 0
+ 3 2
+ 2 1
+ 0 3
+ 1 2
+ 1 0
+ 0 0
+ 1 2
+ 1 0
+ 0 0
+ 0 2
+ 2 0
+ 1 1
+ 2 0
+ 0 3
+ 2 0
+ 1 1
+ 3 0
+ 0 1
+ 0 0
+ 0 0
+ 2 2
+ 2 1
+ 3 1
+ 0 3
+ 0 0
+ 0 0
+ 3 0
+ 0 1
+ 0 2
+ 2 2
+ 0 3
+ 0 0
+ 1 2
+ 0 2
+ 3 2
+ 3 1
+ 0 1
+ 0 0
+ 1 2
+ 2 0
+ 1 3
+ 2 0
+ 3 1
+ 1 1
+ 1 0
+ 0 0
+ 1 2
+ 2 0
+ 3 3
+ 2 3
+ 0 1
+ 1 0
+ 2 2
+ 1 1
+ 1 2
+ 1 0
+ 3 2
+ 3 3
+ 0 1
+ 3 2
+ 0 1
+ 2 0
+ 0 3
+ 2 2
+ 3 1
+ 3 1
+ 3 1
+ 1 3
+ 3 2
+ 1 3
+ 1 2
+ 2 2
+ 0 1
+ 1 0
+ 2 0
+ 1 3
+ 2 2
+ 0 3
+ 3 2
+ 3 3
+ 2 1
+ 3 1
+ 3 1
+ 0 3
+ 0 0
+ 3 2
+ 0 1
+ 2 2
+ 1 1
+ 3 0
+ 3 1
+ 3 1
+ 1 1
+ 0 0
+ 2 2
+ 0 1
+ 0 0
+ 0 0
+ 0 2
+ 1 0
+ 1 0
+ 3 2
+ 3 1
+ 3 3
+ 0 1
+ 1 2
+ 1 2
+ 3 2
+ 3 3
+ 0 3
+ 1 0
+ 1 0
+ 1 0
+ 3 2
+ 1 3
+ 0 2
+ 0 0
+ 2 0
+ 3 3
+ 1 1
+ 1 2
+ 1 2
+ 2 2
+ 0 3
+ 2 0
+ 3 3
+ 2 3
+ 3 1
+ 1 3
+ 3 2
+ 0 3
+ 1 0
+ 3 0
+ 0 3
+ 1 0
+ 3 2
+ 0 3
+ 1 2
+ 0 2
+ 3 0
+ 2 1
+ 0 1
+ 2 2
+ 0 1
+ 3 0
+ 0 3
+ 2 0
+ 1 1
+ 3 2
+ 0 3
+ 3 2
+ 0 1
+ 1 0
+ 2 2
+ 0 3
+ 3 2
+ 3 1
+ 0 1
+ 2 0
+ 2 3
+ 2 3
+ 0 3
+ 2 0
+ 2 3
+ 3 1
+ 2 1
+ 3 1
+ 1 3
+ 1 2
+ 0 0
+ 2 0
+ 1 1
+ 0 2
+ 3 0
+ 2 3
+ 1 1
+ 0 0
+ 2 2
+ 0 3
+ 0 2
+ 0 0
+ 3 0
+ 0 3
+ 2 0
+ 2 1
+ 1 1
+ 3 2
+ 1 3
+ 0 0
+ 1 2
+ 1 0
+ 2 2
+ 1 3
+ 3 2
+ 0 1
+ 3 0
+ 3 1
+ 2 3
+ 1 1
+ 0 2
+ 2 0
+ 0 3
+ 3 2
+ 3 1
+ 0 3
+ 2 2
+ 2 3
+ 0 3
+ 0 0
+ 3 2
+ 2 1
+ 0 3
+ 3 2
+ 0 1
+ 3 0
+ 3 3
+ 2 3
+ 3 3
+ 3 1
+ 2 1
+ 3 1
+ 2 1
+ 2 1
+ 2 1
+ 3 3
+ 2 3
+ 0 3
+ 1 2
+ 1 0
+ 2 2
+ 1 1
+ 2 2
+ 1 3
+ 0 0
+ 1 0
+ 1 0
+ 0 0
+ 2 0
+ 3 1
+ 0 3
+ 0 2
+ 1 2
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 1 1
+ 3 0
+ 2 1
+ 2 1
+ 0 1
+ 1 2
+ 1 2
+ 1 0
+ 0 0
+ 2 0
+ 0 1
+ 0 2
+ 2 0
+ 0 1
+ 1 0
+ 3 2
+ 0 1
+ 1 2
+ 0 0
+ 3 2
+ 2 1
+ 2 3
+ 2 3
+ 3 1
+ 2 1
+ 1 1
+ 3 2
+ 0 3
+ 2 2
+ 1 3
+ 1 2
+ 2 0
+ 3 1
+ 0 3
+ 3 0
+ 2 3
+ 0 1
+ 0 0
+ 3 0
+ 1 1
+ 1 2
+ 2 0
+ 1 1
+ 3 0
+ 0 3
+ 0 2
+ 2 2
+ 0 1
+ 1 0
+ 3 0
+ 0 1
+ 2 2
+ 1 3
+ 1 2
+ 1 2
+ 2 2
+ 0 1
+ 1 2
+ 2 2
+ 2 1
+ 0 3
+ 1 0
+ 2 2
+ 2 3
+ 1 1
+ 2 0
+ 3 1
+ 2 3
+ 2 1
+ 3 3
+ 1 1
+ 2 0
+ 0 1
+ 3 2
+ 0 3
+ 3 2
+ 2 1
+ 3 3
+ 2 3
+ 1 3
+ 2 2
+ 2 1
+ 3 1
+ 3 1
+ 1 1
+ 2 0
+ 1 3
+ 0 2
+ 2 0
+ 2 1
+ 0 3
+ 0 0
+ 2 0
+ 2 3
+ 0 3
+ 0 2
+ 2 0
+ 0 1
+ 0 0
+ 0 2
+ 0 2
+ 2 2
+ 0 3
+ 1 0
+ 1 0
+ 3 0
+ 1 1
+ 0 2
+ 3 2
+ 1 3
+ 1 2
+ 0 0
+ 1 0
+ 1 0
+ 3 2
+ 1 3
+ 3 2
+ 2 3
+ 1 1
+ 2 0
+ 3 1
+ 3 3
+ 3 1
+ 3 1
+ 3 1
+ 0 1
+ 1 0
+ 3 2
+ 1 3
+ 2 2
+ 2 3
+ 3 1
+ 0 3
+ 0 2
+ 1 2
+ 2 0
+ 1 1
+ 2 2
+ 0 3
+ 3 2
+ 2 1
+ 1 1
+ 1 0
+ 2 0
+ 3 3
+ 1 1
+ 3 2
+ 3 1
+ 2 1
+ 0 3
+ 3 0
+ 3 3
+ 1 3
+ 3 2
+ 3 1
+ 1 3
+ 1 2
+ 1 2
+ 0 0
+ 0 0
+ 2 0
+ 0 1
+ 0 0
+ 3 2
+ 0 1
+ 0 2
+ 3 0
+ 1 3
+ 1 2
+ 0 2
+ 1 0
+ 2 0
+ 1 3
+ 1 0
+ 3 2
+ 1 1
+ 3 0
+ 0 1
+ 3 2
+ 2 3
+ 0 3
+ 0 0
+ 2 2
+ 0 3
+ 3 0
+ 0 1
+ 1 0
+ 3 0
+ 2 3
+ 0 3
+ 2 0
+ 1 1
+ 1 2
+ 3 0
+ 1 1
+ 3 2
+ 0 1
+ 3 0
+ 2 3
+ 0 1
+ 3 0
+ 0 3
+ 1 2
+ 0 0
+ 1 2
+ 2 0
+ 2 1
+ 1 1
+ 1 0
+ 3 2
+ 1 1
+ 3 2
+ 2 3
+ 0 3
+ 3 2
+ 2 1
+ 1 3
+ 3 0
+ 0 1
+ 1 0
+ 1 0
+ 1 2
+ 1 2
+ 1 0
+ 2 2
+ 0 1
+ 3 0
+ 2 1
+ 0 3
+ 2 2
+ 3 3
+ 3 1
+ 3 3
+ 1 1
+ 0 2
+ 3 0
+ 2 1
+ 3 1
+ 2 3
+ 1 1
+ 0 2
+ 3 0
+ 0 1
+ 0 0
+ 1 2
+ 2 2
+ 2 3
+ 0 3
+ 3 2
+ 0 3
+ 3 0
+ 3 1
+ 1 3
+ 2 2
+ 1 1
+ 0 2
+ 1 0
+ 0 2
+ 0 0
+ 3 2
+ 3 3
+ 2 1
+ 3 3
+ 0 3
+ 0 0
+ 1 0
+ 2 0
+ 0 3
+ 2 0
+ 0 3
+ 0 0
+ 3 0
+ 3 1
+ 1 1
+ 1 0
+ 2 2
+ 1 3
+ 2 0
+ 0 3
+ 3 2
+ 0 3
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 3 3
+ 0 3
+ 0 2
+ 1 2
+ 0 0
+ 1 0
+ 0 0
+ 3 2
+ 1 1
+ 0 2
+ 1 2
+ 2 0
+ 2 1
+ 0 3
+ 1 2
+ 2 2
+ 0 1
+ 3 2
+ 1 3
+ 0 2
+ 1 2
+ 3 2
+ 0 1
+ 3 2
+ 2 3
+ 1 1
+ 3 2
+ 3 3
+ 1 3
+ 1 2
+ 2 2
+ 2 1
+ 0 3
+ 1 0
+ 1 2
+ 0 2
+ 3 0
+ 2 3
+ 1 3
+ 1 2
+ 3 0
+ 3 1
+ 0 1
+ 1 2
+ 3 2
+ 3 1
+ 0 1
+ 0 2
+ 2 2
+ 0 3
+ 1 0
+ 0 0
+ 1 2
+ 0 0
+ 0 0
+ 1 2
+ 2 2
+ 0 1
+ 2 0
+ 2 1
+ 2 3
+ 3 3
+ 3 3
+ 0 3
+ 0 0
+ 0 0
+ 0 2
+ 3 0
+ 0 3
+ 1 0
+ 0 2
+ 1 0
+ 2 2
+ 0 1
+ 1 0
+ 0 2
+ 1 0
+ 0 2
+ 1 2
+ 3 0
+ 1 1
+ 2 2
+ 2 1
+ 0 1
+ 2 2
+ 1 1
+ 1 2
+ 1 2
+ 2 2
+ 2 1
+ 1 3
+ 0 2
+ 2 2
+ 0 1
+ 1 2
+ 1 2
+ 1 2
+ 0 0
+ 1 2
+ 3 2
+ 1 3
+ 1 2
+ 1 0
+ 3 2
+ 1 3
+ 1 0
+ 2 0
+ 1 1
+ 0 2
+ 0 2
+ 2 2
+ 3 1
+ 3 3
+ 3 3
+ 0 3
+ 3 2
+ 1 3
+ 2 0
+ 3 3
+ 1 3
+ 2 0
+ 2 1
+ 2 1
+ 1 3
+ 0 2
+ 2 0
+ 1 3
+ 3 0
+ 1 3
+ 3 0
+ 0 1
+ 0 0
+ 1 2
+ 2 0
+ 2 1
+ 3 3
+ 3 1
+ 1 3
+ 1 0
+ 1 0
+ 0 0
+ 2 0
+ 0 1
+ 1 0
+ 3 2
+ 1 1
+ 1 0
+ 0 0
+ 0 2
+ 0 0
+ 0 0
+ 2 2
+ 3 3
+ 1 1
+ 2 0
+ 1 3
+ 2 2
+ 3 1
+ 2 1
+ 3 3
+ 0 3
+ 2 0
+ 1 1
+ 3 0
+ 3 1
+ 0 1
+ 2 2
+ 2 1
+ 1 3
+ 1 0
+ 3 2
+ 1 3
+ 0 2
+ 3 2
+ 0 1
+ 0 0
+ 3 2
+ 2 3
+ 2 1
+ 1 1
+ 1 2
+ 3 0
+ 0 1
+ 3 2
+ 1 3
+ 0 2
+ 1 0
+ 1 0
+ 2 2
+ 2 3
+ 1 3
+ 0 0
+ 1 2
+ 1 2
+ 3 0
+ 2 3
+ 3 1
+ 1 3
+ 2 2
+ 0 1
+ 1 2
+ 1 0
+ 2 2
+ 2 3
+ 0 3
+ 2 2
+ 1 1
+ 1 2
+ 3 2
+ 1 1
+ 3 2
+ 1 3
+ 2 0
+ 1 1
+ 1 0
+ 0 0
+ 3 0
+ 1 3
+ 0 0
+ 3 0
+ 0 3
+ 3 2
+ 0 3
+ 1 2
+ 1 0
+ 1 0
+ 3 2
+ 3 1
+ 1 3
+ 0 0
+ 0 0
+ 2 0
+ 0 1
+ 2 2
+ 0 1
+ 0 2
+ 2 0
+ 2 1
+ 0 1
+ 3 2
+ 2 3
+ 1 1
+ 3 2
+ 1 3
+ 3 2
+ 0 1
+ 3 2
+ 2 1
+ 3 1
+ 0 1
+ 3 0
+ 2 1
+ 0 3
+ 1 0
+ 2 0
+ 3 1
+ 0 3
+ 3 2
+ 3 3
+ 0 3
+ 3 0
+ 2 1
+ 0 1
+ 0 0
+ 3 2
+ 3 1
+ 3 3
+ 3 1
+ 0 1
+ 1 2
+ 1 0
+ 0 0
+ 1 2
+ 1 2
+ 2 2
+ 0 3
+ 3 0
+ 1 3
+ 3 0
+ 1 1
+ 2 2
+ 1 1
+ 1 0
+ 3 0
+ 3 3
+ 2 3
+ 0 3
+ 0 2
+ 3 0
+ 0 1
+ 0 2
+ 0 0
+ 1 2
+ 0 0
+ 0 0
+ 0 2
+ 1 2
+ 1 0
+ 0 0
+ 1 0
+ 0 0
+ 0 2
+ 1 0
+ 2 0
+ 1 3
+ 2 2
+ 3 3
+ 0 1
+ 3 2
+ 3 1
+ 1 3
+ 1 0
+ 1 0
+ 3 0
+ 0 3
+ 3 0
+ 0 1
+ 1 2
+ 2 0
+ 1 3
+ 0 0
+ 2 0
+ 0 1
+ 3 2
+ 1 1
+ 0 2
+ 2 2
+ 2 1
+ 1 3
+ 2 0
+ 0 3
+ 1 0
+ 1 0
+ 1 0
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 2 3
+ 0 1
+ 1 2
+ 1 0
+ 1 2
+ 2 2
+ 0 1
+ 2 2
+ 3 1
+ 0 3
+ 1 2
+ 2 2
+ 0 3
+ 3 2
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 2 2
+ 3 1
+ 2 3
+ 3 3
+ 3 1
+ 3 3
+ 3 1
+ 2 3
+ 2 3
+ 0 3
+ 0 0
+ 2 2
+ 0 1
+ 1 2
+ 2 2
+ 3 3
+ 1 3
+ 1 2
+ 0 0
+ 0 2
+ 3 2
+ 3 1
+ 0 1
+ 2 0
+ 0 3
+ 2 2
+ 2 1
+ 1 3
+ 0 2
+ 3 0
+ 3 3
+ 1 1
+ 3 0
+ 1 3
+ 3 2
+ 3 1
+ 3 1
+ 3 3
+ 2 1
+ 1 1
+ 2 2
+ 2 1
+ 0 3
+ 1 0
+ 1 2
+ 0 2
+ 0 2
+ 2 2
+ 1 1
+ 2 2
+ 1 1
+ 1 2
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 1 3
+ 2 0
+ 0 3
+ 3 0
+ 3 3
+ 2 1
+ 0 3
+ 0 2
+ 2 2
+ 0 1
+ 1 0
+ 2 0
+ 3 1
+ 1 1
+ 2 0
+ 0 3
+ 3 0
+ 2 1
+ 1 1
+ 2 0
+ 3 1
+ 2 3
+ 3 3
+ 3 1
+ 1 3
+ 3 0
+ 2 1
+ 1 3
+ 0 2
+ 2 0
+ 1 1
+ 3 0
+ 3 3
+ 3 1
+ 3 3
+ 3 1
+ 0 1
+ 2 2
+ 0 1
+ 3 2
+ 2 1
+ 3 3
+ 0 1
+ 0 2
+ 2 0
+ 0 3
+ 2 0
+ 1 1
+ 3 2
+ 1 3
+ 1 0
+ 3 0
+ 0 1
+ 2 2
+ 2 3
+ 1 1
+ 0 2
+ 1 0
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 0 1
+ 3 2
+ 0 1
+ 2 2
+ 2 1
+ 3 1
+ 0 1
+ 0 2
+ 2 2
+ 2 3
+ 3 1
+ 3 3
+ 2 3
+ 0 3
+ 2 0
+ 2 1
+ 2 3
+ 3 1
+ 3 1
+ 3 1
+ 2 1
+ 2 3
+ 2 3
+ 2 3
+ 1 3
+ 0 0
+ 0 0
+ 0 2
+ 1 0
+ 1 0
+ 0 0
+ 2 0
+ 3 1
+ 1 3
+ 1 2
+ 3 0
+ 1 1
+ 2 0
+ 3 1
+ 0 3
+ 3 2
+ 1 1
+ 2 0
+ 0 1
+ 0 2
+ 2 2
+ 2 3
+ 0 3
+ 3 2
+ 1 1
+ 2 0
+ 1 3
+ 3 0
+ 3 1
+ 2 1
+ 0 1
+ 1 0
+ 0 0
+ 0 0
+ 2 2
+ 2 3
+ 2 1
+ 0 1
+ 1 0
+ 3 0
+ 1 3
+ 1 2
+ 1 0
+ 3 2
+ 0 1
+ 0 0
+ 3 2
+ 3 3
+ 2 3
+ 0 3
+ 1 2
+ 3 0
+ 2 3
+ 0 1
+ 3 2
+ 0 3
+ 3 2
+ 1 1
+ 0 2
+ 2 2
+ 1 3
+ 2 0
+ 0 1
+ 3 0
+ 0 3
+ 1 0
+ 2 2
+ 1 1
+ 2 0
+ 2 3
+ 1 3
+ 1 2
+ 0 0
+ 1 0
+ 2 0
+ 2 1
+ 2 3
+ 1 3
+ 3 0
+ 0 1
+ 3 2
+ 1 1
+ 0 0
+ 2 2
+ 1 1
+ 2 0
+ 0 3
+ 2 2
+ 3 3
+ 1 3
+ 3 0
+ 1 1
+ 1 0
+ 2 0
+ 3 1
+ 3 1
+ 2 1
+ 3 1
+ 2 1
+ 3 1
+ 0 3
+ 1 0
+ 1 0
+ 0 0
+ 3 2
+ 2 3
+ 1 1
+ 0 0
+ 2 0
+ 1 3
+ 1 2
+ 1 2
+ 3 2
+ 0 3
+ 3 0
+ 1 1
+ 1 0
+ 3 2
+ 2 3
+ 3 1
+ 3 3
+ 0 3
+ 0 0
+ 0 0
+ 1 2
+ 0 0
+ 3 0
+ 0 3
+ 3 0
+ 3 1
+ 2 1
+ 0 1
+ 1 0
+ 0 2
+ 0 0
+ 0 2
+ 3 2
+ 1 1
+ 0 2
+ 0 0
+ 0 2
+ 0 0
+ 3 2
+ 0 3
+ 3 0
+ 1 3
+ 0 2
+ 1 0
+ 0 0
+ 3 2
+ 3 1
+ 1 1
+ 0 0
+ 1 0
+ 1 0
+ 2 0
+ 1 1
+ 3 2
+ 1 1
+ 0 0
+ 3 2
+ 0 1
+ 3 2
+ 1 1
+ 3 2
+ 0 1
+ 0 2
+ 1 2
+ 3 2
+ 0 1
+ 3 2
+ 2 3
+ 2 1
+ 3 3
+ 2 3
+ 2 1
+ 1 3
+ 1 2
+ 2 0
+ 0 3
+ 2 0
+ 1 1
+ 0 0
+ 2 2
+ 1 3
+ 0 0
+ 0 0
+ 2 0
+ 0 1
+ 2 0
+ 3 1
+ 3 3
+ 1 3
+ 2 2
+ 3 3
+ 2 3
+ 0 1
+ 1 0
+ 1 2
+ 1 0
+ 0 0
+ 3 0
+ 2 3
+ 1 3
+ 3 2
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 1 0
+ 2 2
+ 1 1
+ 2 0
+ 0 1
+ 3 2
+ 3 3
+ 3 1
+ 2 3
+ 2 1
+ 0 3
+ 1 2
+ 3 2
+ 3 1
+ 2 3
+ 0 1
+ 1 0
+ 2 2
+ 1 3
+ 0 0
+ 2 0
+ 0 3
+ 2 2
+ 2 1
+ 3 1
+ 1 3
+ 1 0
+ 1 2
+ 0 2
+ 3 0
+ 0 1
+ 0 0
+ 1 2
+ 2 2
+ 0 3
+ 3 2
+ 1 3
+ 3 0
+ 1 1
+ 2 0
+ 2 1
+ 2 1
+ 2 1
+ 0 3
+ 0 2
+ 1 0
+ 0 2
+ 1 0
+ 1 2
+ 3 0
+ 3 1
+ 3 3
+ 3 1
+ 0 3
+ 1 0
+ 3 0
+ 2 3
+ 2 3
+ 1 1
+ 1 0
+ 1 0
+ 1 0
+ 2 2
+ 3 3
+ 1 3
+ 0 2
+ 2 0
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 3 2
+ 2 1
+ 2 3
+ 2 3
+ 2 1
+ 3 1
+ 1 3
+ 0 0
+ 3 0
+ 3 3
+ 1 3
+ 2 0
+ 3 1
+ 1 3
+ 0 0
+ 1 2
+ 2 2
+ 2 1
+ 3 1
+ 1 1
+ 0 0
+ 1 0
+ 0 2
+ 1 0
+ 1 2
+ 2 0
+ 2 1
+ 2 1
+ 0 3
+ 3 0
+ 0 1
+ 0 2
+ 2 2
+ 0 3
+ 3 0
+ 1 1
+ 1 2
+ 0 2
+ 2 2
+ 0 1
+ 0 0
+ 0 2
+ 1 2
+ 1 0
+ 2 0
+ 0 1
+ 0 0
+ 1 2
+ 1 0
+ 2 0
+ 2 1
+ 3 3
+ 2 1
+ 1 3
+ 1 2
+ 2 0
+ 1 3
+ 2 2
+ 1 3
+ 0 2
+ 2 0
+ 0 1
+ 2 0
+ 0 1
+ 3 2
+ 3 3
+ 1 3
+ 2 0
+ 2 3
+ 3 1
+ 1 3
+ 3 2
+ 2 3
+ 0 3
+ 3 2
+ 0 3
+ 1 0
+ 0 2
+ 0 2
+ 2 2
+ 2 3
+ 1 3
+ 0 0
+ 3 0
+ 3 3
+ 0 3
+ 0 2
+ 3 0
+ 0 3
+ 0 0
+ 0 2
+ 2 0
+ 2 3
+ 1 3
+ 1 2
+ 2 2
+ 3 3
+ 2 1
+ 1 3
+ 2 0
+ 0 3
+ 0 2
+ 0 0
+ 0 0
+ 2 2
+ 1 1
+ 2 0
+ 3 1
+ 2 1
+ 1 1
+ 2 2
+ 0 3
+ 3 0
+ 0 1
+ 3 0
+ 1 1
+ 3 2
+ 0 3
+ 1 2
+ 2 2
+ 2 1
+ 2 3
+ 3 3
+ 1 1
+ 2 2
+ 1 3
+ 3 2
+ 2 1
+ 3 3
+ 0 3
+ 2 2
+ 1 3
+ 0 0
+ 0 0
+ 3 2
+ 0 1
+ 1 2
+ 0 2
+ 1 0
+ 2 0
+ 2 3
+ 1 3
+ 0 0
+ 1 0
+ 0 2
+ 2 0
+ 0 3
+ 1 2
+ 1 0
+ 0 2
+ 1 2
+ 3 2
+ 2 1
+ 3 1
+ 2 1
+ 1 3
+ 3 2
+ 3 1
+ 3 3
+ 1 1
+ 2 2
+ 0 1
+ 2 2
+ 0 1
+ 2 2
+ 0 1
+ 1 0
+ 2 2
+ 0 1
+ 0 0
+ 0 0
+ 3 2
+ 3 3
+ 1 3
+ 2 2
+ 1 1
+ 2 0
+ 2 1
+ 1 3
+ 2 0
+ 2 3
+ 0 3
+ 3 0
+ 1 1
+ 3 0
+ 2 1
+ 3 1
+ 1 1
+ 1 2
+ 1 2
+ 1 0
+ 0 0
+ 2 0
+ 0 1
+ 3 0
+ 0 1
+ 3 2
+ 0 3
+ 1 0
+ 3 2
+ 2 3
+ 0 1
+ 0 2
+ 3 0
+ 3 1
+ 1 1
+ 0 0
+ 2 2
+ 1 3
+ 2 0
+ 1 3
+ 0 2
+ 1 0
+ 2 0
+ 3 1
+ 2 1
+ 1 1
+ 0 0
+ 0 0
+ 3 0
+ 2 1
+ 3 1
+ 1 1
+ 2 0
+ 2 3
+ 2 3
+ 0 1
+ 2 0
+ 1 1
+ 1 2
+ 0 2
+ 0 2
+ 0 0
+ 3 2
+ 2 3
+ 1 3
+ 0 0
+ 3 2
+ 1 3
+ 3 0
+ 1 3
+ 3 0
+ 0 3
+ 1 2
+ 0 2
+ 3 0
+ 0 1
+ 1 2
+ 0 2
+ 3 2
+ 2 3
+ 0 3
+ 1 0
+ 2 0
+ 1 1
+ 3 2
+ 0 1
+ 0 2
+ 2 2
+ 1 1
+ 3 0
+ 2 3
+ 2 3
+ 1 3
+ 0 0
+ 3 2
+ 1 3
+ 1 2
+ 3 0
+ 3 3
+ 0 1
+ 1 2
+ 2 0
+ 3 3
+ 3 3
+ 2 3
+ 0 1
+ 2 2
+ 3 1
+ 2 1
+ 0 3
+ 3 0
+ 1 1
+ 3 2
+ 1 1
+ 1 0
+ 2 0
+ 0 3
+ 3 2
+ 1 1
+ 3 2
+ 1 1
+ 0 0
+ 3 0
+ 3 1
+ 0 3
+ 3 0
+ 0 1
+ 0 0
+ 0 2
+ 1 2
+ 2 0
+ 2 1
+ 3 1
+ 0 3
+ 3 0
+ 0 1
+ 2 0
+ 1 3
+ 0 0
+ 3 0
+ 0 1
+ 1 0
+ 0 0
+ 1 2
+ 2 2
+ 2 3
+ 2 3
+ 0 3
+ 3 2
+ 1 3
+ 3 0
+ 2 1
+ 2 3
+ 1 3
+ 0 2
+ 0 2
+ 1 0
+ 3 2
+ 1 1
+ 0 2
+ 0 0
+ 1 2
+ 0 0
+ 1 2
+ 2 2
+ 2 1
+ 3 1
+ 1 1
+ 1 0
+ 2 0
+ 2 3
+ 2 1
+ 1 1
+ 3 0
+ 1 1
+ 2 0
+ 2 3
+ 0 1
+ 1 2
+ 3 2
+ 1 3
+ 2 0
+ 0 1
+ 0 2
+ 3 2
+ 3 1
+ 0 1
+ 2 0
+ 3 3
+ 2 3
+ 2 3
+ 0 1
+ 3 2
+ 1 3
+ 1 2
+ 2 0
+ 2 1
+ 1 1
+ 2 2
+ 0 1
+ 2 0
+ 2 3
+ 2 3
+ 0 1
+ 2 0
+ 1 1
+ 0 0
+ 2 2
+ 3 3
+ 3 1
+ 1 3
+ 3 0
+ 0 1
+ 3 0
+ 2 3
+ 2 3
+ 1 1
+ 0 2
+ 3 0
+ 2 3
+ 2 3
+ 0 1
+ 3 0
+ 1 3
+ 3 0
+ 3 1
+ 3 1
+ 0 1
+ 3 2
+ 1 1
+ 1 2
+ 2 2
+ 2 3
+ 2 1
+ 2 3
+ 2 3
+ 0 3
+ 2 2
+ 0 1
+ 2 0
+ 0 1
+ 0 2
+ 3 2
+ 2 3
+ 3 3
+ 2 3
+ 3 3
+ 0 3
+ 3 0
+ 0 1
+ 1 0
+ 3 0
+ 2 1
+ 2 1
+ 2 1
+ 3 1
+ 0 3
+ 0 2
+ 3 0
+ 3 3
+ 0 1
+ 2 2
+ 2 3
+ 3 3
+ 0 3
+ 2 2
+ 3 1
+ 1 1
+ 2 0
+ 2 3
+ 3 1
+ 2 1
+ 1 3
+ 3 2
+ 1 1
+ 0 0
+ 3 2
+ 1 3
+ 3 2
+ 1 1
+ 2 2
+ 2 1
+ 2 1
+ 2 1
+ 3 1
+ 3 3
+ 2 1
+ 2 1
+ 2 1
+ 1 1
+ 0 2
+ 2 2
+ 0 1
+ 2 2
+ 3 3
+ 3 1
+ 1 3
+ 1 2
+ 2 2
+ 3 1
+ 2 3
+ 0 1
+ 1 2
+ 1 2
+ 3 2
+ 3 1
+ 0 1
+ 1 0
+ 1 2
+ 0 0
+ 2 2
+ 0 3
+ 1 0
+ 0 0
+ 3 0
+ 3 1
+ 3 1
+ 1 3
+ 1 0
+ 0 0
+ 2 0
+ 0 3
+ 3 0
+ 3 1
+ 0 3
+ 1 2
+ 1 2
+ 2 2
+ 3 1
+ 3 1
+ 0 1
+ 1 2
+ 3 0
+ 1 3
+ 3 2
+ 3 1
+ 1 1
+ 0 2
+ 2 0
+ 1 1
+ 1 2
+ 3 0
+ 3 1
+ 0 3
+ 3 0
+ 1 3
+ 0 2
+ 1 2
+ 1 2
+ 3 2
+ 3 3
+ 2 3
+ 2 3
+ 0 1
+ 0 0
+ 2 2
+ 2 1
+ 2 3
+ 2 1
+ 3 1
+ 0 3
+ 3 0
+ 2 1
+ 0 3
+ 2 0
+ 0 3
+ 3 2
+ 3 1
+ 0 3
+ 2 2
+ 2 1
+ 3 3
+ 2 1
+ 3 3
+ 3 1
+ 0 3
+ 3 0
+ 2 3
+ 0 3
+ 3 2
+ 2 3
+ 1 3
+ 3 0
+ 1 1
+ 1 0
+ 0 2
+ 3 2
+ 1 1
+ 2 0
+ 2 3
+ 3 1
+ 2 3
+ 2 3
+ 3 3
+ 3 1
+ 2 1
+ 2 1
+ 3 3
+ 2 3
+ 0 3
+ 1 0
+ 0 0
+ 2 2
+ 3 1
+ 3 1
+ 2 3
+ 3 1
+ 3 3
+ 0 1
+ 0 0
+ 1 0
+ 2 2
+ 1 1
+ 1 2
+ 2 2
+ 2 3
+ 1 1
+ 2 0
+ 0 3
+ 2 0
+ 0 1
+ 0 0
+ 1 0
+ 1 0
+ 3 2
+ 0 1
+ 2 2
+ 1 3
+ 2 2
+ 1 3
+ 1 0
+ 0 2
+ 0 2
+ 0 2
+ 0 0
+ 3 0
+ 0 3
+ 0 0
+ 1 2
+ 1 0
+ 1 2
+ 1 0
+ 1 0
+ 2 0
+ 1 1
+ 0 0
+ 3 0
+ 2 3
+ 3 1
+ 3 3
+ 1 3
+ 0 0
+ 2 2
+ 3 1
+ 1 3
+ 3 2
+ 3 3
+ 3 1
+ 1 3
+ 2 0
+ 3 1
+ 1 3
+ 3 0
+ 3 3
+ 1 3
+ 0 0
+ 0 2
+ 3 2
+ 3 3
+ 3 1
+ 0 3
+ 1 0
+ 0 2
+ 3 0
+ 3 1
+ 3 1
+ 3 3
+ 3 3
+ 3 1
+ 3 1
+ 0 3
+ 3 2
+ 0 3
+ 3 0
+ 1 3
+ 2 2
+ 2 1
+ 2 3
+ 2 3
+ 3 3
+ 3 1
+ 2 1
+ 1 3
+ 2 2
+ 0 3
+ 2 2
+ 3 1
+ 0 1
+ 0 0
+ 2 0
+ 0 1
+ 3 0
+ 3 1
+ 2 3
+ 1 1
+ 0 0
+ 2 0
+ 2 1
+ 3 3
+ 2 3
+ 0 1
+ 1 0
+ 2 0
+ 3 3
+ 3 3
+ 2 1
+ 3 3
+ 2 3
+ 3 3
+ 0 3
+ 3 2
+ 3 3
+ 0 1
+ 0 2
+ 2 0
+ 1 3
+ 2 2
+ 1 1
+ 3 2
+ 1 3
+ 2 0
+ 1 3
+ 3 0
+ 2 3
+ 3 1
+ 1 3
+ 0 0
+ 3 2
+ 0 1
+ 3 2
+ 0 1
+ 3 0
+ 0 1
+ 2 0
+ 0 3
+ 0 0
+ 0 0
+ 0 2
+ 0 2
+ 2 2
+ 2 3
+ 1 3
+ 3 2
+ 1 1
+ 3 2
+ 3 3
+ 2 1
+ 0 3
+ 3 0
+ 3 3
+ 1 1
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 1 3
+ 3 2
+ 3 1
+ 3 3
+ 1 3
+ 2 2
+ 0 1
+ 1 0
+ 3 2
+ 2 3
+ 0 1
+ 1 2
+ 3 0
+ 2 1
+ 0 1
+ 3 0
+ 3 1
+ 3 3
+ 3 1
+ 2 3
+ 3 1
+ 3 1
+ 3 1
+ 1 1
+ 2 2
+ 2 3
+ 0 1
+ 3 2
+ 2 1
+ 3 1
+ 2 1
+ 0 1
+ 3 0
+ 3 3
+ 0 3
+ 0 2
+ 2 2
+ 2 1
+ 2 1
+ 1 1
+ 3 0
+ 0 3
+ 2 0
+ 2 1
+ 3 3
+ 0 1
+ 1 0
+ 3 0
+ 3 3
+ 3 3
+ 1 3
+ 2 0
+ 1 1
+ 2 0
+ 2 1
+ 0 3
+ 3 2
+ 2 3
+ 0 3
+ 0 0
+ 2 2
+ 2 3
+ 1 3
+ 3 2
+ 1 3
+ 1 2
+ 1 0
+ 3 0
+ 2 3
+ 1 1
+ 3 0
+ 0 1
+ 3 2
+ 1 1
+ 3 0
+ 1 3
+ 0 2
+ 1 2
+ 3 0
+ 2 1
+ 2 1
+ 3 1
+ 1 1
+ 1 0
+ 0 2
+ 0 0
+ 1 2
+ 0 0
+ 1 0
+ 3 2
+ 2 3
+ 3 1
+ 1 1
+ 3 2
+ 1 3
+ 3 2
+ 0 3
+ 1 0
+ 2 0
+ 1 1
+ 1 0
+ 1 0
+ 0 0
+ 1 2
+ 3 0
+ 0 3
+ 2 2
+ 3 3
+ 0 1
+ 3 2
+ 3 3
+ 1 3
+ 0 2
+ 1 0
+ 1 2
+ 3 0
+ 3 3
+ 3 1
+ 1 3
+ 1 2
+ 2 0
+ 1 1
+ 3 2
+ 2 3
+ 3 1
+ 0 1
+ 3 0
+ 3 3
+ 2 3
+ 2 3
+ 1 3
+ 0 2
+ 2 0
+ 0 1
+ 1 2
+ 2 0
+ 1 1
+ 2 0
+ 1 3
+ 1 2
+ 3 0
+ 1 3
+ 1 0
+ 1 2
+ 0 0
+ 0 0
+ 3 0
+ 2 1
+ 1 3
+ 0 2
+ 2 2
+ 1 1
+ 2 2
+ 0 1
+ 2 0
+ 2 3
+ 2 3
+ 3 3
+ 0 1
+ 2 2
+ 2 1
+ 2 1
+ 0 1
+ 2 0
+ 1 1
+ 2 0
+ 0 3
+ 1 0
+ 3 0
+ 2 3
+ 0 1
+ 3 2
+ 2 3
+ 2 3
+ 2 1
+ 1 3
+ 3 2
+ 2 3
+ 0 3
+ 0 0
+ 0 2
+ 2 2
+ 2 1
+ 1 1
+ 1 0
+ 3 2
+ 0 3
+ 2 2
+ 1 3
+ 0 0
+ 1 2
+ 1 2
+ 0 2
+ 2 0
+ 0 3
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 0 1
+ 3 0
+ 2 3
+ 3 1
+ 1 3
+ 0 0
+ 0 2
+ 1 2
+ 1 0
+ 1 0
+ 3 0
+ 0 1
+ 1 0
+ 1 0
+ 3 0
+ 2 3
+ 2 3
+ 2 3
+ 0 3
+ 3 0
+ 1 3
+ 2 0
+ 3 1
+ 3 1
+ 2 3
+ 0 1
+ 2 2
+ 0 3
+ 2 2
+ 0 3
+ 0 2
+ 2 0
+ 3 1
+ 0 1
+ 3 2
+ 0 3
+ 0 2
+ 2 0
+ 3 1
+ 3 3
+ 3 1
+ 2 1
+ 3 1
+ 3 3
+ 2 3
+ 3 3
+ 0 3
+ 3 0
+ 2 1
+ 0 1
+ 0 0
+ 3 0
+ 1 3
+ 1 2
+ 1 0
+ 3 2
+ 1 3
+ 2 2
+ 1 1
+ 2 2
+ 1 3
+ 3 0
+ 2 3
+ 1 1
+ 3 2
+ 3 1
+ 3 1
+ 1 1
+ 1 2
+ 0 0
+ 2 0
+ 2 1
+ 3 3
+ 0 1
+ 0 0
+ 1 0
+ 2 0
+ 1 3
+ 3 0
+ 0 3
+ 3 0
+ 1 1
+ 1 0
+ 1 0
+ 3 0
+ 1 3
+ 0 2
+ 3 2
+ 2 1
+ 3 1
+ 1 1
+ 0 2
+ 3 0
+ 3 3
+ 0 1
+ 1 2
+ 1 2
+ 0 2
+ 0 0
+ 0 2
+ 3 0
+ 3 3
+ 1 3
+ 0 0
+ 1 2
+ 0 2
+ 3 0
+ 3 3
+ 2 1
+ 1 3
+ 0 2
+ 0 0
+ 2 0
+ 2 1
+ 0 1
+ 2 0
+ 3 3
+ 0 3
+ 0 2
+ 1 2
+ 1 0
+ 0 0
+ 0 0
+ 1 2
+ 2 0
+ 0 1
+ 0 2
+ 3 0
+ 0 1
+ 2 0
+ 1 1
+ 1 0
+ 0 0
+ 2 0
+ 1 1
+ 1 2
+ 2 0
+ 1 1
+ 1 0
+ 2 2
+ 1 1
+ 2 0
+ 0 3
+ 3 0
+ 0 3
+ 0 2
+ 1 2
+ 3 2
+ 0 1
+ 0 2
+ 1 2
+ 0 0
+ 1 2
+ 0 0
+ 1 0
+ 1 0
+ 2 2
+ 1 3
+ 3 2
+ 1 1
+ 1 2
+ 3 2
+ 1 3
+ 1 2
+ 1 2
+ 2 2
+ 1 3
+ 3 2
+ 3 3
+ 1 3
+ 1 2
+ 2 2
+ 2 1
+ 1 1
+ 1 0
+ 2 2
+ 3 1
+ 0 3
+ 2 2
+ 1 3
+ 0 2
+ 3 0
+ 1 3
+ 0 0
+ 0 2
+ 1 0
+ 1 0
+ 3 2
+ 2 3
+ 1 3
+ 0 0
+ 1 2
+ 2 0
+ 0 1
+ 2 0
+ 0 1
+ 2 2
+ 0 3
+ 1 2
+ 2 2
+ 2 3
+ 3 3
+ 2 1
+ 1 1
+ 1 2
+ 3 0
+ 2 3
+ 2 1
+ 1 1
+ 2 0
+ 3 3
+ 3 1
+ 1 1
+ 1 0
+ 0 2
+ 1 0
+ 3 0
+ 0 3
+ 3 2
+ 1 1
+ 0 2
+ 3 0
+ 2 3
+ 2 3
+ 1 1
+ 1 0
+ 3 0
+ 1 3
+ 0 2
+ 2 0
+ 2 3
+ 1 3
+ 0 0
+ 0 2
+ 2 0
+ 1 1
+ 2 2
+ 2 3
+ 1 1
+ 0 0
+ 0 2
+ 0 2
+ 1 0
+ 3 2
+ 2 3
+ 2 3
+ 3 3
+ 0 3
+ 1 0
+ 2 2
+ 0 3
+ 2 2
+ 2 3
+ 1 3
+ 0 2
+ 2 0
+ 2 3
+ 3 1
+ 0 1
+ 0 2
+ 2 0
+ 3 3
+ 2 1
+ 0 3
+ 2 0
+ 3 1
+ 2 3
+ 1 1
+ 1 2
+ 3 2
+ 0 3
+ 3 2
+ 3 1
+ 3 1
+ 3 3
+ 1 1
+ 3 2
+ 2 3
+ 1 3
+ 1 0
+ 3 2
+ 3 3
+ 1 1
+ 2 2
+ 0 3
+ 3 0
+ 1 3
+ 2 2
+ 1 1
+ 1 0
+ 1 0
+ 0 2
+ 1 2
+ 0 2
+ 2 0
+ 2 1
+ 2 3
+ 3 1
+ 1 1
+ 3 0
+ 3 3
+ 0 3
+ 2 0
+ 0 1
+ 0 2
+ 2 2
+ 2 3
+ 1 1
+ 1 0
+ 1 0
+ 0 0
+ 1 0
+ 2 0
+ 0 3
+ 1 2
+ 1 0
+ 1 0
+ 0 2
+ 2 2
+ 0 1
+ 3 2
+ 0 1
+ 2 0
+ 0 1
+ 3 2
+ 3 3
+ 3 1
+ 0 3
+ 2 2
+ 3 1
+ 3 1
+ 0 3
+ 3 2
+ 0 1
+ 2 2
+ 0 1
+ 2 0
+ 0 3
+ 3 0
+ 3 3
+ 0 1
+ 2 0
+ 3 3
+ 2 3
+ 0 3
+ 2 2
+ 1 3
+ 3 2
+ 2 1
+ 2 3
+ 3 3
+ 1 1
+ 2 0
+ 3 3
+ 0 1
+ 2 0
+ 1 3
+ 3 2
+ 0 3
+ 1 0
+ 0 2
+ 3 2
+ 3 3
+ 3 1
+ 3 1
+ 0 1
+ 2 0
+ 1 3
+ 3 2
+ 2 1
+ 3 1
+ 1 3
+ 3 2
+ 1 1
+ 0 2
+ 3 0
+ 1 3
+ 0 2
+ 2 2
+ 1 3
+ 2 2
+ 2 3
+ 3 3
+ 3 1
+ 3 1
+ 2 1
+ 3 1
+ 2 3
+ 2 3
+ 1 1
+ 0 0
+ 0 2
+ 2 0
+ 2 3
+ 3 3
+ 1 3
+ 3 0
+ 1 1
+ 2 0
+ 1 3
+ 0 2
+ 0 2
+ 3 2
+ 3 1
+ 1 3
+ 2 2
+ 2 3
+ 1 1
+ 2 0
+ 0 1
+ 3 0
+ 0 1
+ 3 0
+ 0 3
+ 3 0
+ 2 1
+ 3 3
+ 0 1
+ 1 2
+ 0 0
+ 2 2
+ 3 3
+ 0 3
+ 3 0
+ 1 1
+ 1 2
+ 2 0
+ 1 1
+ 3 2
+ 1 3
+ 1 2
+ 3 0
+ 2 3
+ 1 1
+ 2 2
+ 1 1
+ 1 0
+ 2 0
+ 2 3
+ 3 1
+ 3 1
+ 3 3
+ 0 1
+ 3 2
+ 2 3
+ 3 1
+ 3 1
+ 0 1
+ 0 2
+ 0 2
+ 0 2
+ 3 0
+ 1 3
+ 0 0
+ 1 2
+ 0 0
+ 0 0
+ 1 2
+ 3 0
+ 3 3
+ 2 3
+ 0 3
+ 0 2
+ 1 0
+ 3 2
+ 2 3
+ 0 1
+ 3 2
+ 0 3
+ 1 2
+ 1 0
+ 0 2
+ 1 2
+ 0 0
+ 0 2
+ 2 0
+ 1 3
+ 0 0
+ 3 2
+ 3 1
+ 0 3
+ 0 2
+ 1 0
+ 0 2
+ 0 2
+ 0 2
+ 1 0
+ 0 2
+ 0 0
+ 1 2
+ 2 0
+ 0 3
+ 3 2
+ 0 1
+ 2 2
+ 1 1
+ 3 2
+ 0 3
+ 1 0
+ 0 0
+ 3 0
+ 1 1
+ 2 0
+ 0 1
+ 2 2
+ 3 1
+ 0 1
+ 0 0
+ 1 0
+ 3 0
+ 3 3
+ 0 1
+ 2 0
+ 3 1
+ 2 1
+ 2 3
+ 1 3
+ 3 2
+ 0 3
+ 1 0
+ 3 2
+ 0 1
+ 0 0
+ 2 0
+ 1 1
+ 3 0
+ 1 1
+ 0 0
+ 0 0
+ 3 0
+ 0 1
+ 3 2
+ 0 1
+ 0 2
+ 2 2
+ 3 1
+ 3 1
+ 1 3
+ 1 2
+ 1 2
+ 3 2
+ 3 1
+ 1 3
+ 0 0
+ 0 2
+ 3 0
+ 3 1
+ 3 1
+ 2 1
+ 3 1
+ 3 1
+ 0 3
+ 3 2
+ 3 1
+ 1 1
+ 3 0
+ 2 1
+ 2 1
+ 1 1
+ 2 0
+ 2 3
+ 0 1
+ 1 2
+ 2 0
+ 1 1
+ 3 0
+ 0 3
+ 0 0
+ 3 2
+ 2 1
+ 1 1
+ 0 2
+ 0 2
+ 1 0
+ 0 0
+ 3 0
+ 0 3
+ 3 0
+ 1 3
+ 1 2
+ 0 2
+ 1 2
+ 1 2
+ 0 0
+ 0 0
+ 3 0
+ 2 1
+ 2 3
+ 1 3
+ 1 0
+ 1 0
+ 3 0
+ 0 1
+ 3 0
+ 0 3
+ 1 2
+ 1 2
+ 3 0
+ 1 3
+ 1 2
+ 1 0
+ 2 0
+ 3 3
+ 3 1
+ 0 1
+ 3 0
+ 0 3
+ 0 2
+ 1 2
+ 1 2
+ 3 0
+ 1 3
+ 2 2
+ 3 1
+ 3 3
+ 1 1
+ 1 0
+ 1 2
+ 3 0
+ 1 1
+ 1 0
+ 3 2
+ 0 3
+ 3 2
+ 2 1
+ 1 3
+ 1 0
+ 3 2
+ 1 3
+ 0 2
+ 2 2
+ 0 3
+ 3 2
+ 2 3
+ 0 1
+ 0 2
+ 1 2
+ 3 2
+ 0 1
+ 2 2
+ 0 1
+ 2 2
+ 3 3
+ 2 1
+ 0 3
+ 2 2
+ 2 3
+ 1 1
+ 3 0
+ 2 1
+ 1 1
+ 2 2
+ 3 3
+ 1 1
+ 2 2
+ 0 3
+ 2 2
+ 3 1
+ 1 3
+ 2 2
+ 1 1
+ 1 0
+ 0 0
+ 1 2
+ 0 0
+ 0 2
+ 1 0
+ 1 2
+ 3 0
+ 3 3
+ 2 3
+ 3 3
+ 3 1
+ 2 1
+ 2 3
+ 2 3
+ 1 3
+ 2 0
+ 0 1
+ 1 0
+ 3 0
+ 0 1
+ 3 0
+ 0 3
+ 1 0
+ 2 0
+ 3 1
+ 3 3
+ 0 3
+ 3 2
+ 0 1
+ 0 2
+ 2 0
+ 3 1
+ 1 3
+ 2 0
+ 0 1
+ 1 0
+ 2 0
+ 2 3
+ 2 3
+ 1 1
+ 3 0
+ 2 3
+ 1 3
+ 2 0
+ 3 1
+ 3 1
+ 3 3
+ 3 1
+ 2 1
+ 1 3
+ 1 2
+ 1 2
+ 3 2
+ 0 3
+ 1 2
+ 2 0
+ 2 1
+ 1 1
+ 1 2
+ 2 0
+ 0 1
+ 1 2
+ 1 2
+ 0 2
+ 2 0
+ 0 3
+ 0 0
+ 3 0
+ 1 1
+ 3 2
+ 2 1
+ 1 1
+ 1 0
+ 0 0
+ 3 0
+ 2 1
+ 2 1
+ 0 3
+ 2 0
+ 1 1
+ 2 0
+ 3 1
+ 0 3
+ 0 2
+ 1 2
+ 1 2
+ 2 0
+ 0 1
+ 2 2
+ 3 3
+ 1 1
+ 3 2
+ 0 3
+ 0 2
+ 2 0
+ 1 1
+ 1 0
+ 0 2
+ 2 2
+ 2 1
+ 1 3
+ 3 2
+ 0 1
+ 0 2
+ 1 2
+ 0 0
+ 1 2
+ 0 0
+ 1 0
+ 3 0
+ 0 1
+ 3 0
+ 3 3
+ 3 3
+ 2 3
+ 0 3
+ 1 0
+ 2 0
+ 1 1
+ 1 0
+ 2 0
+ 2 1
+ 3 3
+ 1 1
+ 3 0
+ 2 1
+ 2 1
+ 3 3
+ 1 3
+ 0 0
+ 1 0
+ 2 2
+ 1 1
+ 2 2
+ 0 1
+ 1 2
+ 1 2
+ 1 0
+ 0 0
+ 0 0
+ 2 2
+ 2 1
+ 0 3
+ 0 2
+ 3 2
+ 1 1
+ 3 0
+ 2 3
+ 1 1
+ 3 2
+ 0 3
+ 0 0
+ 2 2
+ 3 3
+ 1 3
+ 1 0
+ 2 0
+ 1 1
+ 3 0
+ 2 3
+ 1 3
+ 1 0
+ 1 0
+ 2 2
+ 0 3
+ 2 2
+ 1 1
+ 1 0
+ 0 0
+ 3 0
+ 0 3
+ 0 0
+ 1 0
+ 2 0
+ 3 1
+ 0 1
+ 0 0
+ 2 0
+ 2 1
+ 1 1
+ 3 0
+ 3 3
+ 1 1
+ 2 0
+ 2 1
+ 1 1
+ 0 0
+ 2 2
+ 1 1
+ 1 2
+ 2 0
+ 1 3
+ 3 2
+ 3 1
+ 0 1
+ 2 2
+ 1 1
+ 0 2
+ 0 0
+ 1 2
+ 1 2
+ 0 2
+ 1 2
+ 0 2
+ 2 2
+ 0 1
+ 3 2
+ 0 1
+ 2 2
+ 3 1
+ 0 3
+ 2 0
+ 3 3
+ 0 3
+ 2 2
+ 2 1
+ 2 3
+ 1 1
+ 2 0
+ 0 1
+ 2 0
+ 2 1
+ 3 3
+ 1 1
+ 3 0
+ 2 1
+ 2 3
+ 3 3
+ 0 1
+ 2 2
+ 3 3
+ 0 3
+ 0 0
+ 3 0
+ 1 3
+ 2 2
+ 1 3
+ 3 2
+ 3 3
+ 3 3
+ 0 3
+ 3 0
+ 1 1
+ 3 2
+ 3 3
+ 0 3
+ 1 2
+ 3 2
+ 2 1
+ 2 1
+ 0 1
+ 0 2
+ 2 2
+ 1 1
+ 1 2
+ 0 0
+ 1 0
+ 2 0
+ 0 1
+ 2 2
+ 3 3
+ 3 1
+ 0 3
+ 3 0
+ 1 1
+ 1 2
+ 3 0
+ 0 1
+ 3 2
+ 2 3
+ 1 1
+ 3 2
+ 2 1
+ 1 1
+ 0 2
+ 3 2
+ 0 1
+ 3 2
+ 0 1
+ 2 0
+ 3 1
+ 1 3
+ 0 0
+ 0 0
+ 2 2
+ 2 3
+ 3 1
+ 0 1
+ 3 2
+ 2 3
+ 3 3
+ 3 3
+ 1 3
+ 0 2
+ 3 2
+ 1 3
+ 2 2
+ 2 3
+ 1 3
+ 0 0
+ 3 2
+ 2 3
+ 1 1
+ 1 2
+ 1 0
+ 2 0
+

From 8d1d1ab0eb18d249a30f3deebddd25eee3992d6b Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 10:24:00 +0100
Subject: [PATCH 20/78] Moarr tests for KSG mixed.

---
 ...MultiVariateWithDiscreteKraskovTester.java | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index a2eb2bc..dead512 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -161,23 +161,25 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
     MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
         new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
 
-    double SEP = 4.0;
-    double true_val = 0.6327;
+    double[] separation = {2.0, 4.0, 8.0};
+    double[] true_val = {0.3368, 0.6327, 0.6931};
     int N = 10000;
 
     // Generate data with unit Gaussians separated by fixed distance
     RandomGenerator rg = new RandomGenerator();
-    double[][] contData = rg.generateNormalData(N, 1, 0, 1);
-    int[] discData = rg.generateRandomInts(N, 2);
-    for (int i = 0; i < N; i++) {
-      if (discData[i] > 0) {
-        contData[i][0] += SEP;
+    for (int j = 0; j < separation.length; j++) {
+      double[][] contData = rg.generateNormalData(N, 1, 0, 1);
+      int[] discData = rg.generateRandomInts(N, 2);
+      for (int i = 0; i < N; i++) {
+        if (discData[i] > 0) {
+          contData[i][0] += separation[j];
+        }
       }
-    }
 
-    miCalc.initialise(1, 2);
-    miCalc.setObservations(contData, discData);
-    double res = miCalc.computeAverageLocalOfObservations();
-    assertEquals(true_val, res, 0.05);
+      miCalc.initialise(1, 2);
+      miCalc.setObservations(contData, discData);
+      double res = miCalc.computeAverageLocalOfObservations();
+      assertEquals(true_val[j], res, 0.05);
+    }
   }
 }

From b5d2b7e367dddb6fdccda4a1734f31ad2222eb53 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 13:25:44 +0100
Subject: [PATCH 21/78] KSG mixed now throws error if discrete data is out of
 range.

---
 ...oCalculatorMultiVariateWithDiscreteKraskov.java | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 38c983c..772974d 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -286,9 +286,17 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 
 		// count the discrete states:
 		counts = new int[base];
-		for (int t = 0; t < discreteData.length; t++) {
-			counts[discreteData[t]]++;
-		}
+    try {
+      for (int t = 0; t < discreteData.length; t++) {
+        counts[discreteData[t]]++;
+      }
+    } catch (ArrayIndexOutOfBoundsException e) {
+      totalObservations = 0;
+      continuousData = null;
+      discreteData   = null;
+      throw new RuntimeException("Values of the discrete variable must range from 0 to base-1");
+    }
+
 		for (int b = 0; b < counts.length; b++) {
 			if (counts[b] < k) {
 				throw new RuntimeException("This implementation assumes there are at least k items in each discrete bin");

From bad50ce6631ec4073fe5bd33e40008a193e312b7 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 13:27:05 +0100
Subject: [PATCH 22/78] KSG mixed now ignores batches of data shorter than
 timeDiff.

---
 ...MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 772974d..c1d4968 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -224,8 +224,10 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		if (continuousObservations[0].length != dimensions) {
 			throw new Exception("The continuous observations do not have the expected number of variables (" + dimensions + ")");
 		}
-		vectorOfContinuousObservations.add(continuousObservations);
-		vectorOfDiscreteObservations.add(discreteObservations);
+    if (continuousObservations.length > timeDiff) {
+      vectorOfContinuousObservations.add(continuousObservations);
+      vectorOfDiscreteObservations.add(discreteObservations);
+    }
 	}
 
 	public void addObservations(double[][] source, double[][] destination, int startTime, int numTimeSteps) throws Exception {

From f55a7bd9613c7b353f127971e56e09c660e2482f Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 13:31:18 +0100
Subject: [PATCH 23/78] Add KdTrees to main compute() method in KSG mixed.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 91 +++++++++++++------
 1 file changed, 61 insertions(+), 30 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index c1d4968..7f6f481 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -25,9 +25,11 @@ import infodynamics.utils.MathsUtils;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
+import infodynamics.utils.KdTree;
 
 import java.util.Iterator;
 import java.util.Vector;
+import java.util.Arrays;
 
 /**
  * 
      Compute the Mutual Information between a vector of continuous variables and discrete
@@ -116,6 +118,18 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * type calls
 	 */
 	protected Vector vectorOfDiscreteObservations;
+  /**
+   * KdTree for range searches in full continuous dataset
+   */
+  protected KdTree kdTreeJoint;
+  /**
+   * KdTrees for neighbour searches in binned continuous datasets
+   */
+  protected KdTree[] kdTreeBins;
+  /**
+   * The norm type in use (see {@link #PROP_NORM_TYPE})
+   */
+  protected int normType = EuclideanUtils.NORM_MAX_NORM;
 	
 	protected EuclideanUtils normCalculator;
 	// Storage for the norms from each observation to each other one
@@ -173,6 +187,8 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		discreteData = null;
 		this.dimensions = dimensions;
 		this.base = base;
+    kdTreeJoint = null;
+    kdTreeBins = null;
 	}
 
 	/**
@@ -198,7 +214,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		if (propertyName.equalsIgnoreCase(PROP_K)) {
 			k = Integer.parseInt(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
-			normCalculator.setNormToUse(propertyValue);
+      normType = KdTree.validateNormType(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
 			normalise = Boolean.parseBoolean(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
@@ -307,6 +323,8 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 
     digammaN = MathsUtils.digamma(totalObservations);
     digammaK = MathsUtils.digamma(k);
+
+    ensureKdTreesConstructed();
 	}
 
 
@@ -338,6 +356,26 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			}
 		}
 	}
+
+
+  /**
+   * Internal method to ensure that the Kd-tree data structures to represent the
+   * observational data have been constructed (should be called prior to attempting
+   * to use these data structures)
+   */
+  public void ensureKdTreesConstructed() {
+    if (kdTreeJoint == null) {
+      kdTreeJoint = new KdTree(continuousData);
+      kdTreeJoint.setNormType(normType);
+      kdTreeBins = new KdTree[base];
+      for (int b = 0; b < base; b++) {
+        kdTreeBins[b] = new KdTree(
+            MatrixUtils.extractSelectedPointsMatchingCondition(
+              continuousData, discreteData, b));
+        kdTreeBins[b].setNormType(normType);
+      }
+    }
+  }
 	
 	/**
 	 * Compute what the average MI would look like were the second time series reordered
@@ -350,6 +388,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * @throws Exception
 	 */
 	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
+
+
+
 		int N = continuousData.length; // number of observations
 		if (!tryKeepAllPairsNorms || (N > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
 			// Generate a new re-ordered set of discrete data
@@ -416,17 +457,14 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	}
 
 	public double computeAverageLocalOfObservations() throws Exception {
-		if (!tryKeepAllPairsNorms || (continuousData.length > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
-			return computeAverageLocalOfObservationsWhileComputingDistances();
-		}
-		
-		// Postcondition: we'll compute the norms before the main loop,
-		//  unless they have already been computed:
-		
-		if (xNorms == null) {
-			computeNorms();
-		}
-		int N = continuousData.length; // number of observations
+
+    // FIXME
+    int dynCorrExclTime = 0;
+    boolean[] one_true = {true};
+
+    int[] cumcount = new int[base];
+    Arrays.fill(cumcount, 0);
+
 
 		// Count the average number of points within eps_x and eps_y
 		double averageDiGammas = 0;
@@ -434,25 +472,18 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		double avNy = 0;
 		
 		double testSum = 0.0; // Used for debugging prints
-		
-		for (int t = 0; t < N; t++) {
-			// Compute eps_x and eps_y for this time step:
-			//  using x norms to all neighbours
-			//  (note that norm of point t to itself will be set to infinity).
-			// Then find the k closest neighbours in the same discrete bin
-			double eps_x = MatrixUtils.kthMinSubjectTo(xNorms[t], k, discreteData, discreteData[t]);			
+    int N = totalObservations;
+
+		for (int t = 0; t < N; t++) {
+
+      // Compute eps_x for this time step:
+      // find neighbours in the same discrete bin, then
+      // count points within eps_x in the whole dataset.
+      int b = discreteData[t];
+      double eps_x = kdTreeBins[b].findKNearestNeighbours(k, cumcount[b]++).poll().distance;
+      int n_x = kdTreeJoint.countPointsWithinOrOnR(t, eps_x, dynCorrExclTime);
+      int n_y = counts[b] - 1;
 
-			// Count the number of points whose x distance is less
-			//  than or equal to eps_x (not including this point)
-			int n_x = 0;
-			for (int t2 = 0; t2 < N; t2++) {
-				if ((t2 != t) && (xNorms[t][t2] <= eps_x)) {
-					// xNorms has infinity for t == t2 anyway ...
-					n_x++;
-				}
-			}
-			// n_y is number of points in that bin, minus this point
-			int n_y = counts[discreteData[t]] - 1;
 			avNx += n_x;
 			avNy += n_y;
 			// And take the digamma before adding into the 

From 8251bdc1cb81ee1e8bef83dc6289b61f3da25c67 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 13:35:03 +0100
Subject: [PATCH 24/78] Removed unused variables from KSG mixed.

---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java     | 1 -
 1 file changed, 1 deletion(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 7f6f481..918e4ec 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -460,7 +460,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 
     // FIXME
     int dynCorrExclTime = 0;
-    boolean[] one_true = {true};
 
     int[] cumcount = new int[base];
     Arrays.fill(cumcount, 0);

From 23bb573bb13227bcbcdcae6c99d857aca53d8f56 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 14:33:47 +0100
Subject: [PATCH 25/78] Replace computeSignificance method in KSG mixed with
 new one based on clone().

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 85 +++++--------------
 1 file changed, 21 insertions(+), 64 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 918e4ec..970c497 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -63,7 +63,7 @@ import java.util.Arrays;
  * @author Joseph Lizier (email,
  * www)
  */
-public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements MutualInfoCalculatorMultiVariateWithDiscrete {
+public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements MutualInfoCalculatorMultiVariateWithDiscrete, Cloneable {
 
 	// Multiplier used in hueristic for determining whether to use a linear search
 	//  for min kth element or a binary search.
@@ -367,6 +367,8 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
     if (kdTreeJoint == null) {
       kdTreeJoint = new KdTree(continuousData);
       kdTreeJoint.setNormType(normType);
+    }
+    if (kdTreeBins == null) {
       kdTreeBins = new KdTree[base];
       for (int b = 0; b < base; b++) {
         kdTreeBins[b] = new KdTree(
@@ -387,75 +389,30 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * @return
 	 * @throws Exception
 	 */
-	public double computeAverageLocalOfObservations(int[] reordering) throws Exception {
-
-
-
-		int N = continuousData.length; // number of observations
-		if (!tryKeepAllPairsNorms || (N > MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM)) {
-			// Generate a new re-ordered set of discrete data
-			int[] originalDiscreteData = discreteData;
-			discreteData = MatrixUtils.extractSelectedTimePoints(discreteData, reordering);
-			// Compute the MI
-			double newMI = computeAverageLocalOfObservationsWhileComputingDistances();
-			// restore data2
-			discreteData = originalDiscreteData;
-			return newMI;
-		}
+	public double computeAverageLocalOfObservations(int[] newOrdering)
+			throws Exception {
 		
-		// Otherwise we will use the norms we've already computed, and use a "virtual"
-		//  reordered data2.
-		int[] reorderedDiscreteData = MatrixUtils.extractSelectedTimePoints(discreteData, reordering);
-
-		if (xNorms == null) {
-			computeNorms();
+		if (newOrdering == null) {
+			return computeAverageLocalOfObservations();
 		}
 
-		// Count the average number of points within eps_x and eps_y
-		double averageDiGammas = 0;
-		double avNx = 0;
-		double avNy = 0;
+		// Take a clone of the object to compute the MI of the surrogates:
+		// (this is a shallow copy, it doesn't make new copies of all
+		//  the arrays)
+		MutualInfoCalculatorMultiVariateWithDiscreteKraskov miSurrogateCalculator =
+				(MutualInfoCalculatorMultiVariateWithDiscreteKraskov) this.clone();
 		
-		for (int t = 0; t < N; t++) {
-			// Compute eps_x and eps_y for this time step:
-			//  using x norms to all neighbours
-			//  (note that norm of point t to itself will be set to infinity).
-			// Then find the k closest neighbours in the same discrete bin
-			double eps_x = MatrixUtils.kthMinSubjectTo(xNorms[t], k, reorderedDiscreteData, reorderedDiscreteData[t]);			
-			
-			// Count the number of points whose x distance is less
-			//  than or equal to eps_x
-			int n_x = 0;
-			for (int t2 = 0; t2 < N; t2++) {
-				if (xNorms[t][t2] <= eps_x) {
-					n_x++;
-				}
-			}
-			// n_y is number of points in that bin minus that point
-			int n_y = counts[reorderedDiscreteData[t]] - 1;
-			avNx += n_x;
-			avNy += n_y;
-			// And take the digamma before adding into the 
-			//  average:
-			averageDiGammas += MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
-		}
-		averageDiGammas /= (double) N;
-		if (debug) {
-			avNx /= (double)N;
-			avNy /= (double)N;
-			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
-		}
-		
-		// Don't need the 1/k correction here because the conditional entropy term
-		//  is taken over the continuous space only. The correction is (m-1)/k
-		//  for an entropy over m subspaces.
-		// mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
-		// Instead do:
-		mi = digammaK + digammaN - averageDiGammas;
-		miComputed = true;
-		return mi;
+		// Generate a new re-ordered data set
+		int[] shuffledDiscreteData = MatrixUtils.extractSelectedTimePoints(discreteData, newOrdering);
+		// Perform new initialisations
+		miSurrogateCalculator.initialise(dimensions, base);
+		// Set new observations
+		miSurrogateCalculator.setObservations(continuousData, shuffledDiscreteData);
+		// Compute the MI
+		return miSurrogateCalculator.computeAverageLocalOfObservations();
 	}
 
+
 	public double computeAverageLocalOfObservations() throws Exception {
 
     // FIXME

From 2d42e6ebad5d823c1785918ba8406e6f3aa0361d Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 14:39:53 +0100
Subject: [PATCH 26/78] Removed unused function from KSG mixed.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 86 -------------------
 1 file changed, 86 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 970c497..1ffffa4 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -485,92 +485,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		return mi;
 	}
 
-	/**
-	 * This method correctly computes the average local MI, but recomputes the x and y 
-	 *  distances between all tuples in time.
-	 * Kept here for cases where we have too many observations
-	 *  to keep the norm between all pairs, and for testing purposes.
-	 * 
-	 * @return
-	 * @throws Exception
-	 */
-	public double computeAverageLocalOfObservationsWhileComputingDistances() throws Exception {
-		int N = continuousData.length; // number of observations
-
-		// Count the average number of points within eps_x and eps_y
-		double averageDiGammas = 0;
-		double avNx = 0;
-		double avNy = 0;
-		
-		double testSum = 0.0; // Used for debugging prints
-		
-		for (int t = 0; t < N; t++) {
-			// Compute eps_x and eps_y for this time step:
-			//  First get x norms to all neighbours
-			//  (note that norm of point t to itself will be set to infinity).
-			double[] norms = new double[N]; 
-			for (int t2 = 0; t2 < N; t2++) {
-				if (t2 == t) {
-					norms[t2] = Double.POSITIVE_INFINITY;
-					continue;
-				}
-				// Compute norm in the continuous space
-				norms[t2] = normCalculator.norm(continuousData[t], continuousData[t2]);
-			}
-
-			// Then find the k closest neighbours in the same discrete bin
-			double eps_x = MatrixUtils.kthMinSubjectTo(norms, k, discreteData, discreteData[t]);			
-
-			// Count the number of points whose x distance is less
-			//  than or equal to eps_x
-			int n_x = 0;
-			for (int t2 = 0; t2 < N; t2++) {
-				if (norms[t2] <= eps_x) {
-					n_x++;
-				}
-			}
-			// n_y is number of points in that bin, minus that point
-			int n_y = counts[discreteData[t]] - 1;
-			avNx += n_x;
-			avNy += n_y;
-			// And take the digamma before adding into the 
-			//  average:
-			double localSum = MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
-			averageDiGammas += localSum;
-
-			if (debug) {
-				// Don't need the 1/k correction here because the conditional entropy term
-				//  is taken over the continuous space only. The correction is (m-1)/k
-				//  for an entropy over m subspaces.
-				// double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
-				// Instead do:
-				double localValue = digammaK + digammaN - localSum;
-				testSum += localValue;
-				if (dimensions == 1) {
-					System.out.printf("t=%d: x=%.3f, eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
-							t, continuousData[t][0], eps_x, n_x, n_y, localValue, testSum);
-				} else {
-					System.out.printf("t=%d: eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
-							t, eps_x, n_x, n_y, localValue, testSum);
-				}
-			}
-		}
-		averageDiGammas /= (double) N;
-		if (debug) {
-			avNx /= (double)N;
-			avNy /= (double)N;
-			System.out.println(String.format("Average n_x=%.3f, Average n_y=%.3f", avNx, avNy));
-		}
-		
-		// Don't need the 1/k correction here because the conditional entropy term
-		//  is taken over the continuous space only. The correction is (m-1)/k
-		//  for an entropy over m subspaces.
-		// mi = MathsUtils.digamma(k) - 1.0/(double)k - averageDiGammas + MathsUtils.digamma(N);
-		// Instead do:
-		mi = digammaK + digammaN - averageDiGammas;
-		miComputed = true;
-		return mi;
-	}
 
 	/**
 	 * Compute the significance of the mutual information of the previously supplied observations.

From 10656627395375f3f1c0b37ad75206812e649ba7 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 14:51:33 +0100
Subject: [PATCH 27/78] Added more tests to KSG mixed locals.

---
 ...ualInfoMultiVariateWithDiscreteKraskovTester.java | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index dead512..c6518ba 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -155,6 +155,18 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
     for (int i = 0; i < 5; i++) {
       assertEquals(locals1[i], locals2[i]);
     }
+
+    // Check that computeAverage and average of computeLocal are the same
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+    miCalc.setObservations(contData, discData);
+    double average = miCalc.computeAverageLocalOfObservations();
+
+    miCalc.initialise(TEST_DIMENSIONS, TEST_BASE);
+    miCalc.setObservations(contData, discData);
+    double averageOfLocals = MatrixUtils.mean(miCalc.computeLocalOfPreviousObservations());
+
+    assertEquals(average, averageOfLocals, 0.01);
+    
   }
 
   public void testCompareAnalyticalValue() throws Exception {

From 7635cf86893df9441864ee36c062d9a82169f49a Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 14:52:54 +0100
Subject: [PATCH 28/78] Added computeLocalOfPreviousObservations to KSG mixed.

It follows the same pattern as the continuous MI: there is a
computeFromObservations method that returns either locals or average
based on a flag.
---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 33 ++++++++++++++-----
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 1ffffa4..7e683ef 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -412,8 +412,15 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		return miSurrogateCalculator.computeAverageLocalOfObservations();
 	}
 
+  public double computeAverageLocalOfObservations() throws Exception {
+    return computeFromObservations(false)[0];
+  }
 
-	public double computeAverageLocalOfObservations() throws Exception {
+  public double[] computeLocalOfPreviousObservations() throws Exception {
+    return computeFromObservations(true);
+  }
+
+	public double[] computeFromObservations(boolean returnLocals) throws Exception {
 
     // FIXME
     int dynCorrExclTime = 0;
@@ -429,6 +436,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		
 		double testSum = 0.0; // Used for debugging prints
     int N = totalObservations;
+    double[] locals = new double[N];
 
 		for (int t = 0; t < N; t++) {
 
@@ -447,13 +455,15 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			double localSum = MathsUtils.digamma(n_x) + MathsUtils.digamma(n_y);
 			averageDiGammas += localSum;
 
+      // Don't need the 1/k correction here because the conditional entropy term
+      //  is taken over the continuous space only. The correction is (m-1)/k
+      //  for an entropy over m subspaces.
+      // double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
+      // Instead do:
+      double localValue = digammaK + digammaN - localSum;
+      locals[t] = localValue;
+
 			if (debug) {
-				// Don't need the 1/k correction here because the conditional entropy term
-				//  is taken over the continuous space only. The correction is (m-1)/k
-				//  for an entropy over m subspaces.
-				// double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
-				// Instead do:
-				double localValue = digammaK + digammaN - localSum;
 				testSum += localValue;
 				if (dimensions == 1) {
 					System.out.printf("t=%d: x=%.3f, eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
@@ -482,7 +492,14 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		// Instead, do:
 		mi = digammaK + digammaN - averageDiGammas;
 		miComputed = true;
-		return mi;
+
+    double[] returnValues;
+    if (returnLocals) {
+      returnValues = locals;
+    } else {
+      returnValues = new double[] {mi};
+    }
+		return returnValues;
 	}
 
 

From faa98a8d79d805137980325ff77ac2838a409983 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 18:21:24 +0100
Subject: [PATCH 29/78] Added more references and tests for KSG mixed calc.

---
 ...MultiVariateWithDiscreteKraskovTester.java | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index c6518ba..5729f8a 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -22,6 +22,7 @@ import junit.framework.TestCase;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
 import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MathsUtils;
 
 public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
 
@@ -172,8 +173,14 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
   public void testCompareAnalyticalValue() throws Exception {
     MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
         new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+    miCalc.setProperty("NORMALISE", "false");
 
     double[] separation = {2.0, 4.0, 8.0};
+
+    // These values were computed using the test script in the supplementary
+    // material of
+    // B. Ross, "Mutual Information Between Discrete and Continuous Data Sets",
+    // PLOS ONE (http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0087357)
     double[] true_val = {0.3368, 0.6327, 0.6931};
     int N = 10000;
 
@@ -193,5 +200,17 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
       double res = miCalc.computeAverageLocalOfObservations();
       assertEquals(true_val[j], res, 0.05);
     }
+
+    // Using the last dataset added, compare a few local
+    // values with the analytical calculation
+    int nb_locals_test = 10;
+    double delta = separation[separation.length-1];
+    for (int i = 0; i < nb_locals_test; i++) {
+      int y = rg.generateRandomInts(1, 2)[0];
+      double x = rg.generateNormalData(1, 0, 1)[0] + delta*y;
+      double analytical = Math.log(MathsUtils.normalPdf(x, delta*y, 1)) - Math.log(0.5*(MathsUtils.normalPdf(x, 0, 1) + MathsUtils.normalPdf(x, delta, 1)));
+      double estimated = miCalc.computeLocalUsingPreviousObservations(new double[][] {new double[] {x} }, new int[] {y})[0];
+      assertEquals(analytical, estimated, 0.05);
+    }
   }
 }

From 641d17ec249947728c976a35f14269e8212f347b Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 19:18:51 +0100
Subject: [PATCH 30/78] Fix bug with normalisation in KSG mixed and change
 unittest accordingly.

---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java    | 2 +-
 .../MutualInfoMultiVariateWithDiscreteKraskovTester.java        | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 7e683ef..5428c12 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -299,7 +299,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			//  that are supplied later:
 			means = MatrixUtils.means(continuousData);
 			stds = MatrixUtils.stdDevs(continuousData, means);
-			MatrixUtils.normaliseIntoNewArray(continuousData, means, stds);
+			MatrixUtils.normalise(continuousData, means, stds);
 		}
 
 		// count the discrete states:
diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index 5729f8a..49d7580 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -173,7 +173,6 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
   public void testCompareAnalyticalValue() throws Exception {
     MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
         new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
-    miCalc.setProperty("NORMALISE", "false");
 
     double[] separation = {2.0, 4.0, 8.0};
 

From bfe8156584f7b77f1eee1bdfc4a885042faa508d Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 16 Aug 2017 19:28:24 +0100
Subject: [PATCH 31/78] Add KDTrees to computeLocalUsingObservations in KSG
 mixed calc.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 69 +++++--------------
 1 file changed, 19 insertions(+), 50 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 5428c12..f311140 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -604,58 +604,27 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			continuousNewStates = MatrixUtils.normaliseIntoNewArray(
 					continuousNewStates, means, stds);
 		}
-		
-		// Thanks to Rosalind Wang for picking up the bug in not using two different N's here
-		int N_newObservations = continuousNewStates.length; // number of new observations we're evaluating
-		int N_samplesForPdfs = continuousData.length; // number of observations we're using to compute the PDFs
-		
-		double[] locals = new double[N_newObservations];
-		
-		// Don't need the 1/k correction here because the conditional entropy term
-		//  is taken over the continuous space only. The correction is (m-1)/k
-		//  for an entropy over m subspaces.
-		// double fixedPartOfLocals = MathsUtils.digamma(k) - 1.0/(double)k +
-		//							MathsUtils.digamma(N);
-		// Instead do:
-		double fixedPartOfLocals = digammaK + digammaN; // Use N_samplesForPdfs here because that's what would be in denominator of probability functions
-		double testSum = 0.0;
-		if (debug) {
-			System.out.printf("digamma(k)=%.3f + digamma(N)=%.3f\n",
-					digammaK, digammaN);
-		}
-		double avNx = 0;
-		double avNy = 0;
-		
-		for (int t = 0; t < N_newObservations; t++) {
-			// Compute eps_x and eps_y for this time step:
-			//  First get x norms to all points in the previously
-			//   given observations.
-			double[] norms = new double[continuousData.length];
-			for (int t2 = 0; t2 < continuousData.length; t2++) {
-				// Compute norm in the continuous space
-				norms[t2] = normCalculator.norm(continuousNewStates[t], continuousData[t2]);
-			}
 
-			// Then find the k closest neighbours in the same discrete bin
-			double eps_x = MatrixUtils.kthMinSubjectTo(norms, k, discreteData, discreteNewStates[t]);			
+		double fixedPartOfLocals = digammaK + MathsUtils.digamma(totalObservations); // Use N_samplesForPdfs here because that's what would be in denominator of probability functions
+		double testSum = 0.0, avNx = 0.0, avNy = 0.0;
+    double[] locals = new double[discreteNewStates.length];
+    for (int t = 0; t < discreteNewStates.length; t++) {
+
+      int b = discreteNewStates[t];
+
+      double[][] x = new double[][] {continuousNewStates[t]};
+      double eps_x = kdTreeBins[b].findKNearestNeighbours(k, x).poll().distance;
+      int n_x = kdTreeJoint.countPointsWithinR(x, eps_x, true);
+			int n_y = counts[b];
 
-			// Count the number of points whose x distance is less
-			//  than or equal to eps_x
-			int n_x = 0;
-			for (int t2 = 0; t2 < continuousData.length; t2++) {
-				if (norms[t2] <= eps_x) {
-					n_x++;
-				}
-			}
-			// n_y is number of points in that discrete bin
-			int n_y = counts[discreteData[t]];
-			avNx += n_x;
-			avNy += n_y;
 			// Now compute the local value:
 			locals[t] = fixedPartOfLocals -
 					MathsUtils.digamma(n_x) - MathsUtils.digamma(n_y);
-			if (debug) {
+
+      if (debug) {
 				testSum += locals[t];
+        avNx += n_x;
+        avNy += n_y;
 				if (dimensions == 1) {
 					System.out.printf("t=%d: x=%.3f, eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
 							t, continuousNewStates[t][0], eps_x, n_x, n_y, locals[t], testSum);
@@ -663,11 +632,11 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 					System.out.printf("t=%d: eps_x=%.3f, n_x=%d, n_y=%d, local=%.3f, running total = %.5f\n",
 							t, eps_x, n_x, n_y, locals[t], testSum);
 				}
-			}
-		}
+      }
+    }
 		if (debug) {
-			avNx /= (double)N_newObservations;
-			avNy /= (double)N_newObservations;
+			avNx /= (double) discreteNewStates.length;
+			avNy /= (double) discreteNewStates.length;
 			System.out.printf("Average n_x=%.3f, Average n_y=%.3f\n", avNx, avNy);
 		}
 		

From fa6307b4f2b94fec3a9503321c47394cfd3e3ddf Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 13:48:03 +1000
Subject: [PATCH 32/78] Fixing javadoc reference in
 AnalyticNullDistributionComputer to correct class

---
 .../infodynamics/utils/AnalyticNullDistributionComputer.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/source/infodynamics/utils/AnalyticNullDistributionComputer.java b/java/source/infodynamics/utils/AnalyticNullDistributionComputer.java
index c80d574..848b1d7 100755
--- a/java/source/infodynamics/utils/AnalyticNullDistributionComputer.java
+++ b/java/source/infodynamics/utils/AnalyticNullDistributionComputer.java
@@ -22,7 +22,7 @@ package infodynamics.utils;
  * Calculators implementing this interface must provide a
  *  {@link #computeSignificance()} method to compute
  *  the statistical significance of their measurement, returning an analytically
- *  determined distribution {@link AnalyticNullDistributionComputer}.
+ *  determined distribution {@link AnalyticMeasurementDistribution}.
  * 
  * @author Joseph Lizier (email,
  * www)

From 70740c97530117e98b102cbeba832656253560be Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 13:49:09 +1000
Subject: [PATCH 33/78] Added new interface EmpiricalNullDistributionComputer.
 Will separately alter discrete and continuous calculators to implement this
 class where they are implementing the required measures.

---
 .../EmpiricalNullDistributionComputer.java    | 102 ++++++++++++++++++
 1 file changed, 102 insertions(+)
 create mode 100644 java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java

diff --git a/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java
new file mode 100644
index 0000000..66e535d
--- /dev/null
+++ b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java
@@ -0,0 +1,102 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier, Ipek Oezdemir and Pedro Mediano
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.utils;
+
+/**
+ * Calculators implementing this interface must provide
+ *  {@link #computeSignificance(int)} and {@link #computeSignificance(int[][])}
+ *  methods to compute
+ *  the statistical significance of their measurement, returning an empirically
+ *  determined distribution {@link EmpiricalMeasurementDistribution}.
+ * 
+ * @author Joseph Lizier (email,
+ * www)
+ */
+public interface EmpiricalNullDistributionComputer {
+
+	/**
+	 * Generate an empirical (bootstrapped) distribution of what the given measure would look like,
+	 * under a null hypothesis that the source values of our
+	 * samples had no relation to the destination value (possibly
+	 * in the context of a conditional value).
+	 * 
+	 * 
      See Section II.E "Statistical significance testing" of 
+	 * the JIDT paper below for a description of how this is done for MI,
+	 * conditional MI and TE.
+	 * 
      
+	 * 
+	 * 
      Note that if several disjoint time-series have been added 
+	 * as observations using addObservations methods etc.,
+	 * then these separate "trials" will be mixed up in the generation
+	 * of surrogates here.
      
+	 * 
+	 * 
      References:
      
+	 *  
        
+	 *   
      • J.T. Lizier, "JIDT: An information-theoretic
+	 *    toolkit for studying the dynamics of complex systems", 2014.
        • 
+     * 
      
+	 * 
+	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
+	 *  to generate the distribution.
+	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
+	 *    toolkit for studying the dynamics of complex systems', 2014."
+	 * @return the empirical distribution of measure scores under this null hypothesis.
+	 */
+	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck);
+	
+	/**
+	 * Generate an empirical (bootstrapped) distribution of what the given measure would look like,
+	 * under a null hypothesis that the source values of our
+	 * samples had no relation to the destination value (possibly
+	 * in the context of a conditional value).
+	 * 
+	 * 
      See Section II.E "Statistical significance testing" of 
+	 * the JIDT paper below for a description of how this is done MI,
+	 * conditional MI and TE.
+	 * 
      
+	 * 
+	 * 
      Note that if several disjoint time-series have been added 
+	 * as observations using addObservations methods etc.,
+	 * then these separate "trials" will be mixed up in the generation
+	 * of surrogates here.
      
+	 * 
+	 * 
      This method (in contrast to {@link #computeSignificance(int)})
+	 * allows the user to specify how to construct the surrogates,
+	 * such that repeatable results may be obtained.
      
+	 * 
+	 * 
      References:
      
+	 *  
        
+	 *   
      • J.T. Lizier, "JIDT: An information-theoretic
+	 *    toolkit for studying the dynamics of complex systems", 2014.
        • 
+     * 
      
+	 * 
+	 * @param newOrderings a specification of how to shuffle the next values
+	 *  to create the surrogates to generate the distribution with. The first
+	 *  index is the permutation number (i.e. newOrderings.length is the number
+	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
+	 *  Each array newOrderings[i] should be an array of length N (where
+	 *  would be the value returned by a call to getNumObservations() for the given measure)
+	 *  containing a permutation of the values in 0..(N-1).
+	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
+	 *    toolkit for studying the dynamics of complex systems', 2014."
+	 * @return the empirical distribution of measure scores under this null hypothesis.
+	 */
+	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings);
+	
+}

From d77a9f274e07f1a51888e7c939f90ec1dff4703c Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 13:54:10 +1000
Subject: [PATCH 34/78] Added interface ChannelCalculator for
 MutualInfoCalculatorMultiVariate; this should have always been there but not
 all calculators implemented the univariate methods. It previously only
 implemented ChannelCalculatorMultiVariate. Added those methods into
 MutualInfoMultiVariateCommon, which provides the base implementation for all
 required calculators, so all are now taken care of. This means MI and TE can
 both be fully used in one interface with the set/addObservations methods.

---
 .../MutualInfoCalculatorMultiVariate.java     |  3 +-
 .../MutualInfoMultiVariateCommon.java         | 43 +++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariate.java
index bcef844..076283e 100755
--- a/java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/MutualInfoCalculatorMultiVariate.java
@@ -71,7 +71,8 @@ package infodynamics.measures.continuous;
  * @see "T. M. Cover and J. A. Thomas, 'Elements of Information
 Theory' (John Wiley & Sons, New York, 1991)."
  */
-public interface MutualInfoCalculatorMultiVariate extends ChannelCalculatorMultiVariate {
+public interface MutualInfoCalculatorMultiVariate
+	extends ChannelCalculator, ChannelCalculatorMultiVariate {
 
 	/**
 	 * Property name for time difference between source and destination (0 by default,
diff --git a/java/source/infodynamics/measures/continuous/MutualInfoMultiVariateCommon.java b/java/source/infodynamics/measures/continuous/MutualInfoMultiVariateCommon.java
index 61cfe8a..22aa4d3 100755
--- a/java/source/infodynamics/measures/continuous/MutualInfoMultiVariateCommon.java
+++ b/java/source/infodynamics/measures/continuous/MutualInfoMultiVariateCommon.java
@@ -259,6 +259,21 @@ public abstract class MutualInfoMultiVariateCommon implements
 		vectorOfDestinationObservations.add(destToAdd);
 	}
 
+	public void addObservations(double[] source, double[] destination,
+			int startTime, int numTimeSteps) throws Exception {
+		
+		if ((dimensionsDest != 1) || (dimensionsSource != 1)) {
+			throw new Exception("The number of source and dest dimensions (having been initialised to " +
+					dimensionsSource + " and " + dimensionsDest + ") can only be 1 when " +
+					"the univariate addObservations(double[],double[]) and " + 
+					"setObservations(double[],double[]) methods are called");
+		}
+		addObservations(MatrixUtils.reshape(source, source.length, 1),
+						MatrixUtils.reshape(destination, destination.length, 1),
+						startTime, numTimeSteps);
+	}
+
+
 	public void setObservations(double[][] source, double[][] destination,
 			boolean[] sourceValid, boolean[] destValid) throws Exception {
 		
@@ -274,6 +289,20 @@ public abstract class MutualInfoMultiVariateCommon implements
 		finaliseAddObservations();
 	}
 
+	public void setObservations(double[] source, double[] destination,
+			boolean[] sourceValid, boolean[] destValid) throws Exception {
+
+		if ((dimensionsDest != 1) || (dimensionsSource != 1)) {
+			throw new Exception("The number of source and dest dimensions (having been initialised to " +
+					dimensionsSource + " and " + dimensionsDest + ") can only be 1 when " +
+					"the univariate addObservations(double[],double[]) and " + 
+					"setObservations(double[],double[]) methods are called");
+		}
+		setObservations(MatrixUtils.reshape(source, source.length, 1),
+				MatrixUtils.reshape(destination, destination.length, 1),
+				sourceValid, destValid);
+	}
+	
 	public void setObservations(double[][] source, double[][] destination,
 			boolean[][] sourceValid, boolean[][] destValid) throws Exception {
 
@@ -485,6 +514,20 @@ public abstract class MutualInfoMultiVariateCommon implements
 		return miSurrogateCalculator.computeAverageLocalOfObservations();
 	}
 
+	public double[] computeLocalUsingPreviousObservations(
+			double[] newSourceObservations, double[] newDestObservations)
+					throws Exception {
+		if ((dimensionsDest != 1) || (dimensionsSource != 1)) {
+			throw new Exception("The number of source and dest dimensions (having been initialised to " +
+					dimensionsSource + " and " + dimensionsDest + ") can only be 1 when " +
+					"the univariate addObservations(double[],double[]) and " + 
+					"setObservations(double[],double[]) methods are called");
+		}
+		return computeLocalUsingPreviousObservations(
+				MatrixUtils.reshape(newSourceObservations, newSourceObservations.length, 1),
+				MatrixUtils.reshape(newDestObservations, newDestObservations.length, 1));
+	}
+
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 	}

From d9e43d966a6d2c1f78af577c3dc96a48fdc1e85c Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 13:58:38 +1000
Subject: [PATCH 35/78] Added implementation of
 EmpiricalNullDistributionComputer to various discrete calculators which
 implement computeSignificance() etc. Also involved removing the method from
 ChannelCalculatorDiscrete, as there's no way of having it (an interface)
 specify implementing an interface without extending it which I didn't think
 was completely logical. So ChannelCalculators can always be checked against
 implementing EmpiricalNullDistributionComputer if one wants to use
 computeSignificance with them. This also required adding the
 computeSignificance(int[][]) method to ConditionalTECalculatorDiscrete and
 TECalculatorDiscrete

---
 .../ActiveInformationCalculatorDiscrete.java  |  3 +-
 .../discrete/ChannelCalculatorDiscrete.java   | 28 +-------
 ...alMutualInformationCalculatorDiscrete.java | 67 ++++---------------
 ...onalTransferEntropyCalculatorDiscrete.java | 47 +++++--------
 .../MutualInformationCalculatorDiscrete.java  | 39 ++---------
 .../TransferEntropyCalculatorDiscrete.java    | 20 ++++--
 6 files changed, 49 insertions(+), 155 deletions(-)

diff --git a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
index 665226e..5659fbe 100755
--- a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
@@ -22,6 +22,7 @@ import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.RandomGenerator;
 
@@ -72,7 +73,7 @@ import infodynamics.utils.RandomGenerator;
  * www)
  */
 public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscreteInContextOfPastCalculator
-	implements AnalyticNullDistributionComputer {
+	implements EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
 
 	protected boolean aisComputed = false;
 	
diff --git a/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
index 4e79cf7..229ca8f 100755
--- a/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
@@ -18,8 +18,6 @@
 
 package infodynamics.measures.discrete;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
-
 /**
  * A basic interface for calculators computing measures on a univariate channel
  * for discrete (ie int[]) data from a
@@ -42,7 +40,8 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * 				
    10. the average channel measure: {@link #computeAverageLocalOfObservations()};
      11. 
  * 				
    11. the distribution of channel measure values under the null hypothesis
  * 					of no relationship between source and
- * 					destination values: {@link #computeSignificance(int)};
      12. 
+ * 					destination values (where the calculator also 
+ * 					implements EmpiricalNullDistributionComputer
  * 				
    12. or other quantities as defined by child classes.
      13. 
  * 			
  * 		
@@ -104,27 +103,4 @@ public interface ChannelCalculatorDiscrete {
 	 */
 	public double computeAverageLocalOfObservations();
 	
-	/**
-	 * Generate a bootstrapped distribution of what the channel measure would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination value.
-	 * (Precise null hypothesis varies between MI and TE).
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for MI,
-	 * conditional MI and TE.
-	 * 
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(int[], int[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of channel measure scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck);
 }
diff --git a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
index f11db0c..cbed332 100755
--- a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
@@ -21,6 +21,7 @@ package infodynamics.measures.discrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
@@ -64,7 +65,8 @@ Theory' (John Wiley & Sons, New York, 1991).
  * www)
  */
 public class ConditionalMutualInformationCalculatorDiscrete
-	extends InfoMeasureCalculatorDiscrete implements AnalyticNullDistributionComputer {
+	extends InfoMeasureCalculatorDiscrete
+	implements EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
 
 	/**
 	 * Store the number of symbols for each variable
@@ -403,29 +405,13 @@ public class ConditionalMutualInformationCalculatorDiscrete
 	}
 
 	/**
-	 * Generate a bootstrapped distribution of what the conditional MI would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination value (in the
-	 * context of the conditional).
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for MI,
-	 * conditional MI and TE.
-	 * Note that this method currently fixes the relationship
+	 * 
      Note -- in contrast to the generic computeSignificance() method 
+	 * described below, this method for a conditional MI calculator currently fixes the relationship
 	 * between variable 2 and the conditional, and shuffles
-	 * variable 1 with respect to these.
+	 * variable 1 with respect to these.
 	 * 
      
 	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(int[], int[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of conditional MI scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
+	 * @inheritDoc
 	 */
 	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) {
 		RandomGenerator rg = new RandomGenerator();
@@ -435,46 +421,17 @@ public class ConditionalMutualInformationCalculatorDiscrete
 	}
 	
 	/**
-	 * Generate a bootstrapped distribution of what the conditional MI would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination values.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for  
-	 * a conditional mutual information. Basically, the marginal PDFs
-	 * of each marginal
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of conditional MI under these conditions is generated.
-	 * Note that this method currently fixes the relationship
+	 * 
      Note -- in contrast to the generic computeSignificance() method 
+	 * described below, this method for a conditional MI calculator currently fixes the relationship
 	 * between variable 2 and the conditional, and shuffles
-	 * variable 1 with respect to these.
+	 * variable 1 with respect to these.
 	 * 
      
+	 * 
 	 * TODO Need to alter the method signature to allow callers to specify
 	 * which variable is shuffled. (Note to self: when doing this, will
 	 * need to update machine learning code to the new method signature)
 	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
-	 * 
-	 * @param newOrderings a specification of how to shuffle the values
-	 *  of variable 1
-	 *  to create the surrogates to generate the distribution with. The first
-	 *  index is the permutation number (i.e. newOrderings.length is the number
-	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
-	 *  Each array newOrderings[i] should be an array of length N (where
-	 *  would be the value returned by {@link #getNumObservations()}),
-	 *  containing a permutation of the values in 0..(N-1).
-	 * @return the distribution of conditional MI scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception where the length of each permutation in newOrderings
-	 *   is not equal to the number N samples that were previously supplied.
+	 * @inheritDoc
 	 */
 	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) {
 		
diff --git a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
index 0813bb1..d16c8d7 100644
--- a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
@@ -18,6 +18,7 @@
 
 package infodynamics.measures.discrete;
 
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MathsUtils;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
@@ -109,10 +110,12 @@ import infodynamics.utils.RandomGenerator;
  * www)
  *
  */
-public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCalculatorDiscrete {
+public class ConditionalTransferEntropyCalculatorDiscrete
+	extends InfoMeasureCalculatorDiscrete 
+	implements EmpiricalNullDistributionComputer {
 
 	protected int k = 0; // history length k.
-  protected int base_others = 0; // base of the conditional variables
+	protected int base_others = 0; // base of the conditional variables
 	protected int base_power_k = 0;
 	protected int base_power_num_others = 0;
 	protected int numOtherInfoContributors = 0;
@@ -624,32 +627,20 @@ public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCal
 		return te;
 	}
 	
-	/**
-	 * Generate a bootstrapped distribution of what the 
-	 * conditional TE would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination value
-	 * (in the context of the destination past and conditionals).
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for MI,
-	 * conditional MI and TE.
-	 * 
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(int[], int[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of conditional TE scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 */
+	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) {
+		RandomGenerator rg = new RandomGenerator();
+		// (Not necessary to check for distinct random perturbations)
+		int[][] newOrderings = rg.generateRandomPerturbations(observations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) {
 		double actualTE = computeAverageLocalOfObservations();
 		
+		int numPermutationsToCheck = newOrderings.length;
+		
 		// Reconstruct the source values (not necessarily in order)
 		int[] sourceValues = new int[observations];
 		int t_s = 0;
@@ -694,12 +685,6 @@ public class ConditionalTransferEntropyCalculatorDiscrete extends InfoMeasureCal
 			}
 		}
 		
-		// Construct new source orderings based on the source probabilities only
-		// Generate the re-ordered indices:
-		RandomGenerator rg = new RandomGenerator();
-		// (Not necessary to check for distinct random perturbations)
-		int[][] newOrderings = rg.generateRandomPerturbations(observations, numPermutationsToCheck);
-
 		// TODO stop using deprecated method
 		ConditionalTransferEntropyCalculatorDiscrete cte = newInstance(base, k, numOtherInfoContributors);
 		cte.initialise();
diff --git a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
index 1613125..7d658b6 100755
--- a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
@@ -21,6 +21,7 @@ package infodynamics.measures.discrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
@@ -64,7 +65,8 @@ Theory' (John Wiley & Sons, New York, 1991).
  * www)
  */
 public class MutualInformationCalculatorDiscrete extends InfoMeasureCalculatorDiscrete 
-	implements ChannelCalculatorDiscrete, AnalyticNullDistributionComputer {
+	implements ChannelCalculatorDiscrete, 
+	EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
 
 	private int timeDiff = 0;
 	private int[][]	jointCount = null; // Count for (i[t-timeDiff], j[t]) tuples
@@ -222,40 +224,7 @@ public class MutualInformationCalculatorDiscrete extends InfoMeasureCalculatorDi
 		return computeSignificance(newOrderings);
 	}
 	
-	/**
-	 * Generate a bootstrapped distribution of what the MI would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination values.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for  
-	 * a mutual information. Basically, the marginal PDFs
-	 * of each marginal
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of MI under these conditions is generated.
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
-	 * 
-	 * @param newOrderings a specification of how to shuffle the next values
-	 *  to create the surrogates to generate the distribution with. The first
-	 *  index is the permutation number (i.e. newOrderings.length is the number
-	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
-	 *  Each array newOrderings[i] should be an array of length N (where
-	 *  would be the value returned by {@link #getNumObservations()}),
-	 *  containing a permutation of the values in 0..(N-1).
-	 * @return the distribution of MI scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception where the length of each permutation in newOrderings
-	 *   is not equal to the number N samples that were previously supplied.
-	 */
+	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) {
 		double actualMI = computeAverageLocalOfObservations();
 		
diff --git a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
index 52c8382..853f0ee 100755
--- a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
@@ -21,6 +21,7 @@ package infodynamics.measures.discrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MathsUtils;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
@@ -87,7 +88,8 @@ import infodynamics.utils.RandomGenerator;
  * www
  */
 public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalculatorDiscrete 
-	implements ChannelCalculatorDiscrete, AnalyticNullDistributionComputer {
+	implements ChannelCalculatorDiscrete,
+	EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
 
 	/**
 	 * Counts of (source,dest_next,dest_embedded_past) tuples
@@ -1205,8 +1207,18 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 
 	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) {
+		RandomGenerator rg = new RandomGenerator();
+		// (Not necessary to check for distinct random perturbations)
+		int[][] newOrderings = rg.generateRandomPerturbations(observations, numPermutationsToCheck);
+		return computeSignificance(newOrderings);
+	}
+	
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) {
 		double actualTE = computeAverageLocalOfObservations();
 		
+		int numPermutationsToCheck = newOrderings.length;
+		
 		// Reconstruct the *joint* source values (not necessarily in order, but using joint values retains their l-tuples)
 		int[] sourceValues = new int[observations];
 		int t_s = 0;
@@ -1238,12 +1250,6 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 			}
 		}
 		
-		// Construct new source orderings based on the source probabilities only
-		// Generate the re-ordered indices:
-		RandomGenerator rg = new RandomGenerator();
-		// (Not necessary to check for distinct random perturbations)
-		int[][] newOrderings = rg.generateRandomPerturbations(observations, numPermutationsToCheck);
-
 		// If we want a calculator just like this one, we should provide all of 
 		//  the same parameters:
 		TransferEntropyCalculatorDiscrete ate2 =

From 60105e6d106b940d87f6a8c2ec8a9892e5a5a228 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 14:07:26 +1000
Subject: [PATCH 36/78] ActiveInformationCalculatorDiscrete was stuck in the
 past and returning results computing with the log using the base of the
 alphabet size (everything else uses bits). I'm not sure how I missed this
 one. Fixed now.

---
 .../ActiveInformationCalculatorDiscrete.java   | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
index 5659fbe..7511ea2 100755
--- a/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ActiveInformationCalculatorDiscrete.java
@@ -295,7 +295,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 				// Compute MI contribution:
 				if (p_joint > 0.0) {
 					double logTerm = p_joint / (p_next * p_prev);
-					double localValue = Math.log(logTerm) / log_base;
+					double localValue = Math.log(logTerm) / log_2;
 					miCont = p_joint * localValue;
 					if (localValue > max) {
 						max = localValue;
@@ -334,7 +334,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 				if (p_joint > 0.0) {
 					double logTerm = p_joint / p_prev;
 					// Entropy rate takes the negative log:
-					double localValue = - Math.log(logTerm) / log_base;
+					double localValue = - Math.log(logTerm) / log_2;
 					entRateCont = p_joint * localValue;
 				} else {
 					entRateCont = 0.0;
@@ -362,7 +362,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 		  ( (double) nextCount[next] *
 			(double) pastCount[past] );
 		logTerm *= (double) observations;
-		return Math.log(logTerm) / log_base;
+		return Math.log(logTerm) / log_2;
 	}
 
 	@Override
@@ -394,7 +394,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 			//  and we've got two counts on the bottom
 			//  but one count on the top:
 			logTerm *= (double) observations;
-			localActive[t] = Math.log(logTerm) / log_base;
+			localActive[t] = Math.log(logTerm) / log_2;
 			average += localActive[t];
 			if (localActive[t] > max) {
 				max = localActive[t];
@@ -445,7 +445,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 				//  and we've got two counts on the bottom
 				//  but one count on the top:
 				logTerm *= (double) observations;
-				localActive[r][c] = Math.log(logTerm) / log_base;
+				localActive[r][c] = Math.log(logTerm) / log_2;
 				average += localActive[r][c];
 				if (localActive[r][c] > max) {
 					max = localActive[r][c];
@@ -501,7 +501,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 					//  and we've got two counts on the bottom
 					//  but one count on the top:
 					logTerm *= (double) observations;
-					localActive[t][r][c] = Math.log(logTerm) / log_base;
+					localActive[t][r][c] = Math.log(logTerm) / log_2;
 					average += localActive[t][r][c];
 					if (localActive[t][r][c] > max) {
 						max = localActive[t][r][c];
@@ -550,7 +550,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 			//  and we've got two counts on the bottom
 			//  but one count on the top:
 			logTerm *= (double) observations;
-			localActive[r] = Math.log(logTerm) / log_base;
+			localActive[r] = Math.log(logTerm) / log_2;
 			average += localActive[r];
 			if (localActive[r] > max) {
 				max = localActive[r];
@@ -599,7 +599,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 			//  and we've got two counts on the bottom
 			//  but one count on the top:
 			logTerm *= (double) observations;
-			localActive[t] = Math.log(logTerm) / log_base;
+			localActive[t] = Math.log(logTerm) / log_2;
 			average += localActive[t];
 			if (localActive[t] > max) {
 				max = localActive[t];
@@ -767,7 +767,7 @@ public class ActiveInformationCalculatorDiscrete extends SingleAgentMeasureDiscr
 				// Compute MI contribution:
 				if (p_joint * p_next * p_prev > 0.0) {
 					double logTerm = p_joint / (p_next * p_prev);
-					double localValue = Math.log(logTerm) / log_base;
+					double localValue = Math.log(logTerm) / log_2;
 					miCont = p_joint * localValue;
 					System.out.println(String.format("%7d    %.2f %7d    %.2f     %.2f    %.2f     %.2f",
 							nextVal, p_next, prevVal, p_prev, p_joint, logTerm, localValue));

From 3e64fd28bd8b1dd5c0f44bc5e479402dd69e96ed Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 15:31:27 +1000
Subject: [PATCH 37/78] Altering various continuous calculators to implement
 the EmpiricalNullDistributionComputer interface (where they're already
 implementing computeSignificance() etc.). Also altered the
 EmpiricalNullDistributionComputer methods to throw Exceptions, since the
 continuous calculators generally do this (and doesn't harm the discrete
 ones). Also involved implementing the methods in
 ConditionalMIMultiVariateCommon by selecting to permute the first variable by
 default.

---
 .../ActiveInfoStorageCalculator.java          | 71 +---------------
 ...tiveInfoStorageCalculatorMultiVariate.java |  4 +-
 ...geCalculatorMultiVariateViaMutualInfo.java |  1 -
 .../continuous/ChannelCalculatorCommon.java   | 66 +--------------
 ...ionalMutualInfoCalculatorMultiVariate.java | 82 ++++++++-----------
 ...nditionalMutualInfoMultiVariateCommon.java | 12 +++
 .../continuous/PredictiveInfoCalculator.java  | 71 +---------------
 ...PredictiveInfoCalculatorViaMutualInfo.java |  4 +-
 ...alMutualInformationCalculatorDiscrete.java |  4 +-
 .../EmpiricalNullDistributionComputer.java    |  6 +-
 10 files changed, 63 insertions(+), 258 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
index b04b173..01467ac 100644
--- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
+++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
@@ -18,7 +18,7 @@
 
 package infodynamics.measures.continuous;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 
 /**
  * Interface for implementations of
@@ -84,7 +84,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface ActiveInfoStorageCalculator {
+public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionComputer {
 
 	/**
 	 * Property name for embedding length k of
@@ -294,73 +294,6 @@ public interface ActiveInfoStorageCalculator {
 	 */
 	public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception;
 
-	/**
-	 * Generate a bootstrapped distribution of what the AIS would look like,
-	 * under a null hypothesis that the previous k values of our
-	 * samples had no relation to the next value in the time-series.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for AIS 
-	 * as a mutual information. Basically, the marginal PDFs
-	 * of the past k values, and that of the next value, 
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of AIS under these conditions is generated.
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int[][])})
-	 * creates random shufflings of the next values for the surrogate AIS
-	 * calculations.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of AIS scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
-	
-	/**
-	 * Generate a bootstrapped distribution of what the AIS would look like,
-	 * under a null hypothesis that the previous k values of our
-	 * samples had no relation to the next value in the time-series.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for AIS 
-	 * as a mutual information. Basically, the marginal PDFs
-	 * of the past k values, and that of the next value, 
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of AIS under these conditions is generated.
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
-	 * 
-	 * @param newOrderings a specification of how to shuffle the next values
-	 *  to create the surrogates to generate the distribution with. The first
-	 *  index is the permutation number (i.e. newOrderings.length is the number
-	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
-	 *  Each array newOrderings[i] should be an array of length L being 
-	 *  the value returned by {@link #getNumObservations()},
-	 *  containing a permutation of the values in 0..(L-1).
-	 * @return the distribution of AIS scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception where the length of each permutation in newOrderings
-	 *   is not equal to the number L observations that were previously supplied.
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(
-			int[][] newOrderings) throws Exception;
-
 	/**
 	 * Set or clear debug mode for extra debug printing to stdout
 	 * 
diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java
index 7b64050..3da0fba 100755
--- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java
@@ -18,8 +18,6 @@
 
 package infodynamics.measures.continuous;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
-
 /**
  * Interface for implementations of Active Information Storage estimators on
  * multivariate continuous time-series data. That is, it is applied to
@@ -103,4 +101,6 @@ public interface ActiveInfoStorageCalculatorMultiVariate {
 	 */
 	public void initialise(int dimensions, int k, int tau) throws Exception;
 
+	// TODO We seem to be missing a lot of functionality which should be defined here,
+	//  as implemented in the univariate measure. will have to go back and do this.
 }
diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java
index a6b1ead..adf14b2 100755
--- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java
+++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java
@@ -21,7 +21,6 @@ package infodynamics.measures.continuous;
 import java.util.Iterator;
 import java.util.Vector;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.MatrixUtils;
 
 /**
diff --git a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
index ee2d7ab..afee934 100755
--- a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
+++ b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
@@ -18,7 +18,7 @@
 
 package infodynamics.measures.continuous;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 
 /**
  * A basic interface for calculators computing measures on a channel from a
@@ -76,7 +76,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * @see ChannelCalculatorMultiVariate
  *
  */
-public abstract interface ChannelCalculatorCommon {
+public abstract interface ChannelCalculatorCommon extends EmpiricalNullDistributionComputer {
 
 	/**
 	 * Initialise the calculator for (re-)use, with the existing
@@ -166,68 +166,6 @@ public abstract interface ChannelCalculatorCommon {
 	 */
 	public double[] computeLocalOfPreviousObservations() throws Exception;
 
-	/**
-	 * Generate a bootstrapped distribution of what the channel measure would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination value.
-	 * (Precise null hypothesis varies between MI and TE).
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for MI and TE.
-	 * 
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int[][])})
-	 * creates random shufflings of the next values for the surrogate AIS
-	 * calculations.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of channel measure scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
-	
-	/**
-	 * Generate a bootstrapped distribution of what the channel measure would look like,
-	 * under a null hypothesis that the source values of our
-	 * samples had no relation to the destination value.
-	 * (Precise null hypothesis varies between MI and TE).
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for MI and TE.
-	 * 
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
-	 * 
-	 * @param newOrderings a specification of how to shuffle the source values
-	 *  to create the surrogates to generate the distribution with. The first
-	 *  index is the permutation number (i.e. newOrderings.length is the number
-	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
-	 *  Each array newOrderings[i] should be an array of length N (where
-	 *  would be the value returned by {@link #getNumObservations()}),
-	 *  containing a permutation of the values in 0..(N-1).
-	 * @return the distribution of channel measure scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception where the length of each permutation in newOrderings
-	 *   is not equal to the number N samples that were previously supplied.
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(
-			int[][] newOrderings) throws Exception;
-
 	/**
 	 * Set or clear debug mode for extra debug printing to stdout
 	 * 
diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
index dfa1762..f66da14 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
@@ -19,6 +19,7 @@
 package infodynamics.measures.continuous;
 
 import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 
 /**
  * 
      Interface for implementations of the conditional mutual information,
@@ -75,7 +76,8 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * @see "T. M. Cover and J. A. Thomas, 'Elements of Information
 Theory' (John Wiley & Sons, New York, 1991)."
  */
-public interface ConditionalMutualInfoCalculatorMultiVariate {
+public interface ConditionalMutualInfoCalculatorMultiVariate
+	extends EmpiricalNullDistributionComputer {
 
 	/**
 	 * Initialise the calculator for (re-)use, clearing PDFs,
@@ -303,66 +305,53 @@ public interface ConditionalMutualInfoCalculatorMultiVariate {
 	public double[] computeLocalOfPreviousObservations() throws Exception;
 
 	/**
-	 * Generate a bootstrapped distribution of what the conditional MI would look like,
-	 * under a null hypothesis that the variable identified by
-	 * variableToReorder had no relation to the
-	 * other variable, given the conditional.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for
-	 * conditional MI. Basically, we shuffle the observations of 
-	 * variableToReorder against the other variable
-	 * and the conditional.
-	 * This keeps the marginal and joint PDFs of the unshuffled variable
-	 *  and conditional the same
-	 *  but destroys any correlation between the named variable and the others.
+	 * 
      Note -- in contrast to the generic computeSignificance() method 
+	 * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship
+	 * between variable 2 and the conditional, and shuffles
+	 * variable 1 with respect to these.
+	 * To shuffle variable 2 instead, call {@link #computeSignificance(int, int)}.
 	 * 
      
 	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[][], double[][], double[][])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int, int[][])})
-	 * creates random shufflings of the next values for the surrogate AIS
-	 * calculations.
      
+	 * @inheritDoc
+	 */
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception;
+	
+	/**
+	 * Defined as per {@link #computeSignificance(int)} except that this method allows
+	 * the user to specify which variable is shuffled (whilst the other has its
+	 * relationship with the conditional preserved).
 	 * 
 	 * @param variableToReorder which variable to shuffle:
 	 * 	1 for variable 1, 2 for variable 2.
 	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
 	 *  to generate the distribution.
 	 * @return the distribution of channel measure scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
+	 * @see {@link #computeSignificance(int)}
 	 * @throws Exception
 	 */
 	public EmpiricalMeasurementDistribution computeSignificance(int variableToReorder,
 			int numPermutationsToCheck) throws Exception;
 	
 	/**
-	 * Generate a bootstrapped distribution of what the conditional MI would look like,
-	 * under a null hypothesis that the variable identified by
-	 * variableToReorder had no relation to the
-	 * other variable, given the conditional.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for
-	 * conditional MI. Basically, we shuffle the observations of 
-	 * variableToReorder against the other variable
-	 * and the conditional.
-	 * This keeps the marginal and joint PDFs of the unshuffled variable
-	 *  and conditional the same
-	 *  but destroys any correlation between the named variable and the others.
+	 * 
      Note -- in contrast to the generic computeSignificance() method 
+	 * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship
+	 * between variable 2 and the conditional, and shuffles
+	 * variable 1 with respect to these.
+	 * To shuffle variable 2 instead, call {@link #computeSignificance(int, int[][])}.
 	 * 
      
 	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[][], double[][], double[][])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int, int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
+	 * @inheritDoc
+	 */
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception;
+	
+	/**
+	 * Defined as per {@link #computeSignificance(int[][])} except that this method allows
+	 * the user to specify which variable is shuffled (whilst the other has its
+	 * relationship with the conditional preserved).
 	 * 
 	 * @param variableToReorder which variable to shuffle:
 	 * 	1 for variable 1, 2 for variable 2.
@@ -375,8 +364,7 @@ public interface ConditionalMutualInfoCalculatorMultiVariate {
 	 *  would be the value returned by {@link #getNumObservations()}),
 	 *  containing a permutation of the values in 0..(N-1).
 	 * @return the distribution of channel measure scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
+	 * @see {@link #computeSignificance(int[][])} except that this method all 
 	 * @throws Exception where the length of each permutation in newOrderings
 	 *   is not equal to the number N samples that were previously supplied.
 	 */
diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
index 3c6fc0e..bbb0e46 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
@@ -390,6 +390,12 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements
 		vectorOfCondObservations = null;
 	}
 	
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(
+			int numPermutationsToCheck) throws Exception {
+		return computeSignificance(1, numPermutationsToCheck);
+	}
+	
 	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(
 			int variableToReorder, int numPermutationsToCheck) throws Exception {
@@ -403,6 +409,12 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements
 		return computeSignificance(variableToReorder, newOrderings);
 	}
 
+	@Override
+	public EmpiricalMeasurementDistribution computeSignificance(
+			int[][] newOrderings) throws Exception {
+		return computeSignificance(1, newOrderings);
+	}
+	
 	/**
 	 * 
      As described in
 	 * {@link ConditionalMutualInfoCalculatorMultiVariate#computeSignificance(int, int[][])}
diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
index 3f08e11..946d00e 100755
--- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
+++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
@@ -18,7 +18,7 @@
 
 package infodynamics.measures.continuous;
 
-import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 
 /**
  * Interface for implementations of
@@ -94,7 +94,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface PredictiveInfoCalculator {
+public interface PredictiveInfoCalculator extends EmpiricalNullDistributionComputer {
 
 	/**
 	 * Property name for embedding length k of
@@ -282,73 +282,6 @@ public interface PredictiveInfoCalculator {
 	 */
 	public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception;
 
-	/**
-	 * Generate a bootstrapped distribution of what the PI would look like,
-	 * under a null hypothesis that the previous k values of our
-	 * samples had no relation to the next k values in the time-series.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for PI 
-	 * as a mutual information. Basically, the marginal PDFs
-	 * of the past k values, and that of the next k values, 
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of PI under these conditions is generated.
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int[][])})
-	 * creates random shufflings of the next vectors for the surrogate PI
-	 * calculations.
      
-	 * 
-	 * @param numPermutationsToCheck number of surrogate samples to bootstrap
-	 *  to generate the distribution.
-	 * @return the distribution of PI scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
-	
-	/**
-	 * Generate a bootstrapped distribution of what the PI would look like,
-	 * under a null hypothesis that the previous k values of our
-	 * samples had no relation to the next k values in the time-series.
-	 * 
-	 * 
      See Section II.E "Statistical significance testing" of 
-	 * the JIDT paper below for a description of how this is done for PI 
-	 * as a mutual information. Basically, the marginal PDFs
-	 * of the past k values, and that of the next k values, 
-	 * are preserved, while their joint PDF is destroyed, and the 
-	 * distribution of PI under these conditions is generated.
      
-	 * 
-	 * 
      Note that if several disjoint time-series have been added 
-	 * as observations using {@link #addObservations(double[])} etc.,
-	 * then these separate "trials" will be mixed up in the generation
-	 * of surrogates here.
      
-	 * 
-	 * 
      This method (in contrast to {@link #computeSignificance(int)})
-	 * allows the user to specify how to construct the surrogates,
-	 * such that repeatable results may be obtained.
      
-	 * 
-	 * @param newOrderings a specification of how to shuffle the next k values vectors
-	 *  to create the surrogates to generate the distribution with. The first
-	 *  index is the permutation number (i.e. newOrderings.length is the number
-	 *  of surrogate samples we use to bootstrap to generate the distribution here.)
-	 *  Each array newOrderings[i] should be an array of length L being
-	 *  the value returned by {@link #getNumObservations()},
-	 *  containing a permutation of the values in 0..(L-1).
-	 * @return the distribution of PI scores under this null hypothesis.
-	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
-	 *    toolkit for studying the dynamics of complex systems', 2014."
-	 * @throws Exception where the length of each permutation in newOrderings
-	 *   is not equal to the number L observations that were previously supplied.
-	 */
-	public EmpiricalMeasurementDistribution computeSignificance(
-			int[][] newOrderings) throws Exception;
-
 	/**
 	 * Set or clear debug mode for extra debug printing to stdout
 	 * 
diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
index 835839b..108292a 100755
--- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
+++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
@@ -417,7 +417,7 @@ public class PredictiveInfoCalculatorViaMutualInfo implements
 	}
 	
 	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#computeSignificance(int)
+	 * @see infodynamics.utils.EmpiricalNullDistributionComputer#computeSignificance(int)
 	 */
 	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(
@@ -426,7 +426,7 @@ public class PredictiveInfoCalculatorViaMutualInfo implements
 	}
 
 	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#computeSignificance(int[][])
+	 * @see infodynamics.utils.EmpiricalNullDistributionComputer#computeSignificance(int[][])
 	 */
 	@Override
 	public EmpiricalMeasurementDistribution computeSignificance(
diff --git a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
index cbed332..55e6f34 100755
--- a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java
@@ -406,7 +406,7 @@ public class ConditionalMutualInformationCalculatorDiscrete
 
 	/**
 	 * 
      Note -- in contrast to the generic computeSignificance() method 
-	 * described below, this method for a conditional MI calculator currently fixes the relationship
+	 * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship
 	 * between variable 2 and the conditional, and shuffles
 	 * variable 1 with respect to these.
 	 * 
      
@@ -422,7 +422,7 @@ public class ConditionalMutualInformationCalculatorDiscrete
 	
 	/**
 	 * 
      Note -- in contrast to the generic computeSignificance() method 
-	 * described below, this method for a conditional MI calculator currently fixes the relationship
+	 * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship
 	 * between variable 2 and the conditional, and shuffles
 	 * variable 1 with respect to these.
 	 * 
      
diff --git a/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java
index 66e535d..9ef66bf 100644
--- a/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java
+++ b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java
@@ -58,7 +58,7 @@ public interface EmpiricalNullDistributionComputer {
 	 *    toolkit for studying the dynamics of complex systems', 2014."
 	 * @return the empirical distribution of measure scores under this null hypothesis.
 	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck);
+	public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception;
 	
 	/**
 	 * Generate an empirical (bootstrapped) distribution of what the given measure would look like,
@@ -96,7 +96,9 @@ public interface EmpiricalNullDistributionComputer {
 	 * @see "J.T. Lizier, 'JIDT: An information-theoretic
 	 *    toolkit for studying the dynamics of complex systems', 2014."
 	 * @return the empirical distribution of measure scores under this null hypothesis.
+	 * @throws Exception where e.g. the newOrderings don't supply arrays of the correct
+	 *   length matching the number of observations that we have.
 	 */
-	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings);
+	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception;
 	
 }

From 04351498aa7ef13235e935fc540dbd39c057a9f5 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 15:40:00 +1000
Subject: [PATCH 38/78] Changed my mind and pulled
 EmpiricalNullDistributionComputer in as being extended/implemented by
 ChannelCalculatorDiscrete. (This does make sense, since "extend" for an
 interface really means implements, it's not so much a child class). This
 necessitated removing the interface from explicitly being named in the
 definitions of the discrete MI and TE calculators since it's there
 implicitly.

---
 .../measures/discrete/ChannelCalculatorDiscrete.java       | 7 ++++---
 .../discrete/MutualInformationCalculatorDiscrete.java      | 4 +---
 .../discrete/TransferEntropyCalculatorDiscrete.java        | 4 +---
 3 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
index 229ca8f..d774124 100755
--- a/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ChannelCalculatorDiscrete.java
@@ -18,6 +18,8 @@
 
 package infodynamics.measures.discrete;
 
+import infodynamics.utils.EmpiricalNullDistributionComputer;
+
 /**
  * A basic interface for calculators computing measures on a univariate channel
  * for discrete (ie int[]) data from a
@@ -40,8 +42,7 @@ package infodynamics.measures.discrete;
  * 				
    13. the average channel measure: {@link #computeAverageLocalOfObservations()};
      14. 
  * 				
    14. the distribution of channel measure values under the null hypothesis
  * 					of no relationship between source and
- * 					destination values (where the calculator also 
- * 					implements EmpiricalNullDistributionComputer
      15. 
+ * 					destination values: {@link #computeSignificance(int)};
  * 				
    15. or other quantities as defined by child classes.
      16. 
  * 			
  * 		
@@ -53,7 +54,7 @@ package infodynamics.measures.discrete;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface ChannelCalculatorDiscrete {
+public interface ChannelCalculatorDiscrete extends EmpiricalNullDistributionComputer {
 
 	/**
 	 * Initialise the calculator for (re-)use, with the existing
diff --git a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
index 7d658b6..14b7513 100755
--- a/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/MutualInformationCalculatorDiscrete.java
@@ -21,7 +21,6 @@ package infodynamics.measures.discrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
-import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
 import infodynamics.utils.RandomGenerator;
@@ -65,8 +64,7 @@ Theory' (John Wiley & Sons, New York, 1991).
  * www)
  */
 public class MutualInformationCalculatorDiscrete extends InfoMeasureCalculatorDiscrete 
-	implements ChannelCalculatorDiscrete, 
-	EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
+	implements ChannelCalculatorDiscrete, AnalyticNullDistributionComputer {
 
 	private int timeDiff = 0;
 	private int[][]	jointCount = null; // Count for (i[t-timeDiff], j[t]) tuples
diff --git a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
index 853f0ee..e3d04d0 100755
--- a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
@@ -21,7 +21,6 @@ package infodynamics.measures.discrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ChiSquareMeasurementDistribution;
-import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MathsUtils;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
@@ -88,8 +87,7 @@ import infodynamics.utils.RandomGenerator;
  * www
  */
 public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalculatorDiscrete 
-	implements ChannelCalculatorDiscrete,
-	EmpiricalNullDistributionComputer, AnalyticNullDistributionComputer {
+	implements ChannelCalculatorDiscrete, AnalyticNullDistributionComputer {
 
 	/**
 	 * Counts of (source,dest_next,dest_embedded_past) tuples

From d6bd16546b6415d16bb6eb04e6d910b5dfa1866f Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 16:54:55 +1000
Subject: [PATCH 39/78] Added new interface InfoMeasureCalculatorContinuous to
 capture common methods for continuous calculators. Also adapted most
 continuous calculators to implement this, which involved in many cases
 removing duplicate definitions of these methods in the interfaces for these
 measures. For ConditionalMI calculator, this meant we needed to add an
 implementation of the zero-argument initialise() call also. Still need to do
 MultiInfo, PredictiveInfo and Entropy in future, as they're missing a few
 methods.

---
 .../ActiveInfoStorageCalculator.java          |  66 +--------
 .../continuous/ChannelCalculatorCommon.java   |  77 +----------
 ...ionalMutualInfoCalculatorMultiVariate.java |  63 +--------
 ...nditionalMutualInfoMultiVariateCommon.java |   5 +-
 .../InfoMeasureCalculatorContinuous.java      | 128 ++++++++++++++++++
 5 files changed, 139 insertions(+), 200 deletions(-)
 create mode 100644 java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java

diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
index 01467ac..2d79c11 100644
--- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
+++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java
@@ -84,7 +84,8 @@ import infodynamics.utils.EmpiricalNullDistributionComputer;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionComputer {
+public interface ActiveInfoStorageCalculator extends
+	InfoMeasureCalculatorContinuous, EmpiricalNullDistributionComputer {
 
 	/**
 	 * Property name for embedding length k of
@@ -100,13 +101,6 @@ public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionCo
 	 */
 	public static final String TAU_PROP_NAME = "TAU";
 
-	/**
-	 * Initialise the calculator for (re-)use, with the existing (or default) values of parameters
-	 * Clears any PDFs of previously supplied observations.
-	 * 
-	 */
-	public void initialise() throws Exception;
-
 	/**
 	 * Initialise the calculator for (re-)use, with some parameters
 	 * supplied here, and existing (or default) values of other parameters
@@ -148,23 +142,9 @@ public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionCo
 	 * @param propertyValue value of the property
 	 * @throws Exception for invalid property values
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue) throws Exception;
 
-	/**
-	 * Get current property values for the calculator.
-	 * 
-	 * 
      Valid property names, and what their
-	 * values should represent, are the same as those for
-	 * {@link #setProperty(String, String)}
      
-	 * 
-	 * 
      Unknown property values are responded to with a null return value.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @return current value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public String getProperty(String propertyName) throws Exception;
-	
 	/**
 	 * Sets a single time-series from which to compute the PDF for the AIS.
 	 * Cannot be called in conjunction with other methods for setting/adding
@@ -247,13 +227,6 @@ public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionCo
 	 */
 	public void addObservations(double[] observations, boolean[] valid) throws Exception;
 
-	/**
-	 * Compute the AIS from the previously-supplied samples.
-	 * 
-	 * @return the AIS estimate
-	 */
-	public double computeAverageLocalOfObservations() throws Exception;
-
 	/**
 	 * Compute the local AIS values for each of the
 	 * previously-supplied samples.
@@ -294,37 +267,4 @@ public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionCo
 	 */
 	public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception;
 
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Return the AIS last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise()} call.
-	 * 
-	 * @return the last computed AIS value
-	 */
-	public double getLastAverage();
-
-	/**
-	 * Get the number of samples to be used for the PDFs here 
-	 * which have been supplied by calls to
-	 * {@link #setObservations(double[])}, {@link #addObservations(double[])}
-	 * etc.
-	 * 
-	 * 
      Note that the number of samples is not equal to the length of time-series
-	 * supplied (since we need to accumulate the first
-	 * (k-1)*tau + 1
-	 * values of each time-series before taking a sample for AIS).
-	 * 
      
-	 * 
-	 * @return the number of samples to be used for the PDFs
-	 * @throws Exception
-	 */
-	public int getNumObservations() throws Exception;
-
 }
diff --git a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
index afee934..f1b6930 100755
--- a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
+++ b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java
@@ -76,45 +76,9 @@ import infodynamics.utils.EmpiricalNullDistributionComputer;
  * @see ChannelCalculatorMultiVariate
  *
  */
-public abstract interface ChannelCalculatorCommon extends EmpiricalNullDistributionComputer {
+public abstract interface ChannelCalculatorCommon
+	extends InfoMeasureCalculatorContinuous, EmpiricalNullDistributionComputer {
 
-	/**
-	 * Initialise the calculator for (re-)use, with the existing
-	 * (or default) values of parameters.
-	 * 
-	 * @throws Exception
-	 */
-	public void initialise() throws Exception;
-
-	/**
-	 * Set properties for the calculator.
-	 * New property values are not guaranteed to take effect until the next call
-	 *  to an initialise method. 
-	 * 
-	 * 
      No general properties are defined at the interface level here, i.e.
-	 * there are only properties defined by child interfaces and classes.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @param propertyValue value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public void setProperty(String propertyName, String propertyValue) throws Exception;
-
-	/**
-	 * Get current property values for the calculator.
-	 * 
-	 * 
      Valid property names, and what their
-	 * values should represent, are the same as those for
-	 * {@link #setProperty(String, String)}
      
-	 * 
-	 * 
      Unknown property values are responded to with a null return value.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @return current value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public String getProperty(String propertyName) throws Exception;
-	
 	/**
 	 * Signal that we will add in the samples for computing the PDF 
 	 * from several disjoint time-series or trials via calls to
@@ -140,13 +104,6 @@ public abstract interface ChannelCalculatorCommon extends EmpiricalNullDistribut
 	 */
 	public boolean getAddedMoreThanOneObservationSet();
 	
-	/**
-	 * Compute the channel measure from the previously-supplied samples.
-	 * 
-	 * @return the estimate of the channel measure
-	 */
-	public double computeAverageLocalOfObservations() throws Exception;
-
 	/**
 	 * 
      Computes the local values of the implemented channel measure,
 	 *  for each valid observation in the previously supplied observations
@@ -166,34 +123,4 @@ public abstract interface ChannelCalculatorCommon extends EmpiricalNullDistribut
 	 */
 	public double[] computeLocalOfPreviousObservations() throws Exception;
 
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Return the channel measure last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise()} call.
-	 * 
-	 * @return the last computed channel measure value
-	 */
-	public double getLastAverage();
-
-	/**
-	 * Get the number of samples to be used for the PDFs here 
-	 * which have been supplied by calls to
-	 * {@link #setObservations(double[])}, {@link #addObservations(double[])}
-	 * etc.
-	 * 	
-	 * 
      Note that the number of samples may not be equal to the length of time-series
-	 * supplied (e.g. for transfer entropy, where we need to accumulate
-	 * a number of samples for the past history of the destination).
-	 * 
      
-	 * 
-	 * @return the number of samples to be used for the PDFs
-	 */
-	public int getNumObservations() throws Exception;
 }
diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
index f66da14..a86ce64 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
@@ -77,7 +77,7 @@ import infodynamics.utils.EmpiricalNullDistributionComputer;
 Theory' (John Wiley & Sons, New York, 1991)."
  */
 public interface ConditionalMutualInfoCalculatorMultiVariate
-	extends EmpiricalNullDistributionComputer {
+	extends InfoMeasureCalculatorContinuous, EmpiricalNullDistributionComputer {
 
 	/**
 	 * Initialise the calculator for (re-)use, clearing PDFs,
@@ -91,35 +91,6 @@ public interface ConditionalMutualInfoCalculatorMultiVariate
 	 */
 	public void initialise(int var1Dimensions, int var2Dimensions, int condDimentions) throws Exception;
 
-	/**
-	 * Set properties for the calculator.
-	 * New property values are not guaranteed to take effect until the next call
-	 *  to an initialise method. 
-	 * 
-	 * 
      No general properties are defined at the interface level here, i.e.
-	 * there are only properties defined by child interfaces and classes.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @param propertyValue value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public void setProperty(String propertyName, String propertyValue) throws Exception;
-
-	/**
-	 * Get current property values for the calculator.
-	 * 
-	 * 
      Valid property names, and what their
-	 * values should represent, are the same as those for
-	 * {@link #setProperty(String, String)}
      
-	 * 
-	 * 
      Unknown property values are responded to with a null return value.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @return current value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public String getProperty(String propertyName) throws Exception;
-	
 	/**
 	 * Sets a single series from which to compute the PDF.
 	 * Cannot be called in conjunction with
@@ -274,15 +245,6 @@ public interface ConditionalMutualInfoCalculatorMultiVariate
 	 */
 	public void finaliseAddObservations() throws Exception;
 	
-	/**
-	 * Compute the average conditional MI from the previously-supplied samples.
-	 * 
-	 * @return the estimate of the conditional MI in either bits or nats
-	 *  depending on the estimator
-	 * @throws Exception
-	 */
-	public double computeAverageLocalOfObservations() throws Exception;
-
 	/**
 	 * 
      Computes the local values of the conditional mutual information,
 	 *  for each valid observation in the previously supplied observations
@@ -413,32 +375,11 @@ public interface ConditionalMutualInfoCalculatorMultiVariate
 		throws Exception;
 
 	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Return the conditional MI last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise(int, int, int)} call.
-	 * 
-	 * @return the last computed conditional MI value
-	 */
-	public double getLastAverage();
-
-	/**
-	 * Get the number of samples to be used for the PDFs here 
-	 * which have been supplied by calls to
-	 * {@link #setObservations(double[][], double[][], double[][])}, {@link #addObservations(double[][], double[][], double[][])}
-	 * etc.
-	 * 	
-	 * @return the number of samples to be used for the PDFs
 	 * @throws Exception if the implementing class computes MI without
 	 * explicit observations (e.g. see
 	 * {@link infodynamics.measures.continuous.gaussian.ConditionalMutualInfoCalculatorMultiVariateGaussian})
 	 */
+	@Override
 	public int getNumObservations() throws Exception;
 	
 	/**
diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
index bbb0e46..dd5460e 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
@@ -165,7 +165,10 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements
 	 */
 	protected double[] condStds = null;
 
-	
+	@Override
+	public void initialise() {
+		initialise(dimensionsVar1, dimensionsVar2, dimensionsCond);
+	}
 	
 	@Override
 	public void initialise(int var1Dimensions, int var2Dimensions, int condDimensions) {
diff --git a/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java b/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java
new file mode 100644
index 0000000..e2734a1
--- /dev/null
+++ b/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java
@@ -0,0 +1,128 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2017, Joseph T. Lizier, Ipek Oezdemir and Pedro Mediano
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.measures.continuous;
+
+/**
+ * 
      Base interface for our information-theoretic calculators
+ *  on continuous (double[]) data,
+ *  providing common functionality
+ *  for user-level measure classes.
      
+ * 
+ * 
      
+ * Usage of the child classes implementing this interface is intended to follow this paradigm:
+ * 
      
+ * 
        
+ * 		
      1. Construct the calculator;
        2. 
+ * 		
      2. set properties via {@link #setProperty(String, String)};
        3. 
+ *		
      3. Initialise the calculator using {@link #initialise()} or
+ *			other initialise methods defined by child classes;
        4. 
+ * 		
      4. Provide the observations/samples for the calculator
+ *      	to set up the PDFs, using one or more calls to
+ * 			sets of "addObservations" methods defined by child classes, then
        5. 
+ * 		
      5. Compute the required quantities, being one or more of:
+ * 			
          
+ * 				
        • the average measure: {@link #computeAverageLocalOfObservations()};
          • 
+ * 				
        • or other quantities as defined by child classes.
          • 
+ * 			
        
+ * 		
        6. 
+ * 		
      6. 
+ * 		Return to step 3 to re-use the calculator on a new data set.
+ * 		
        7. 
+ * 	
      
+ * 
+ * @author Joseph Lizier (email,
+ * www)
+ */
+public interface InfoMeasureCalculatorContinuous {
+
+	/**
+	 * Initialise the calculator for (re-)use, with the existing (or default) values of parameters.
+	 * Clears any PDFs of previously supplied observations.
+	 */
+	public void initialise() throws Exception;
+	
+	/**
+	 * Set properties for the underlying calculator implementation.
+	 * New property values are not guaranteed to take effect until the next call
+	 *  to an initialise method. 
+	 * 
+	 * 
      Property names are defined by the implementing class
+	 * (generally in their documentation).
+	 * Unknown property values are ignored.
      
+	 * 
+	 * @param propertyName name of the property
+	 * @param propertyValue value of the property
+	 * @throws Exception for invalid property values
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+
+	/**
+	 * Get current property values for the calculator.
+	 * 
+	 * 
      Valid property names, and what their
+	 * values should represent, are the same as those for
+	 * {@link #setProperty(String, String)}
      
+	 * 
+	 * 
      Unknown property values are responded to with a null return value.
      
+	 * 
+	 * @param propertyName name of the property
+	 * @return current value of the property
+	 * @throws Exception for invalid property values
+	 */
+	public String getProperty(String propertyName) throws Exception;
+
+	/**
+	 * Get the number of samples to be used for the PDFs here 
+	 * which have been supplied by calls to
+	 * "setObservations", "addObservations" etc.
+	 * 	
+	 * 
      Note that the number of samples may not be equal to the length of time-series
+	 * supplied (e.g. for transfer entropy, where we need to accumulate
+	 * a number of samples for the past history of the destination).
+	 * 
      
+	 * 
+	 * @return the number of samples to be used for the PDFs
+	 */
+	public int getNumObservations() throws Exception;
+
+	/**
+	 * Compute the average value of the measure
+	 * from the previously-supplied samples.
+	 * 
+	 * @return the estimate of the measure
+	 */
+	public abstract double computeAverageLocalOfObservations() throws Exception;
+	
+	/**
+	 * Return the measure last calculated in a call to
+	 * {@link #computeAverageLocalOfObservations()}
+	 * or related methods after the previous
+	 * {@link #initialise()} call.
+	 * 
+	 * @return the last computed measure value
+	 */
+	public double getLastAverage();
+
+	/**
+	 * Set or clear debug mode for extra debug printing to stdout
+	 * 
+	 * @param debug new setting for debug mode (on/off)
+	 */
+	public void setDebug(boolean debug);
+}

From 65c7cd99cc5579cfea28592ff571d1938f1bb201 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 20:18:29 +1000
Subject: [PATCH 40/78] Making continuous EntropyCalculator and
 EntropyCalculatorMultiVariate classes implement the
 InfoMeasureCalculatorContinuous interface. For the interfaces, this means
 removing methods where duplicated. Also added NUM_DIMENSIONS property to
 EntropyCalculatorMultiVariate interface, so that these calculators can have
 an initialise() method which takes no parameters. Fixed all implementing
 classes to have any methods that they were missing.

---
 .../continuous/EntropyCalculator.java         | 37 +--------
 .../EntropyCalculatorMultiVariate.java        | 76 +++++++------------
 .../InfoMeasureCalculatorContinuous.java      |  1 +
 .../gaussian/EntropyCalculatorGaussian.java   | 37 ++++++++-
 ...EntropyCalculatorMultiVariateGaussian.java | 44 +++++++++--
 .../kernel/EntropyCalculatorKernel.java       | 37 ++++++++-
 .../EntropyCalculatorMultiVariateKernel.java  | 58 +++++++++++++-
 ...tropyCalculatorMultiVariateKozachenko.java | 52 ++++++++++---
 8 files changed, 236 insertions(+), 106 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/EntropyCalculator.java b/java/source/infodynamics/measures/continuous/EntropyCalculator.java
index f268fcb..747a0c9 100755
--- a/java/source/infodynamics/measures/continuous/EntropyCalculator.java
+++ b/java/source/infodynamics/measures/continuous/EntropyCalculator.java
@@ -46,29 +46,7 @@ Theory' (John Wiley & Sons, New York, 1991).
  * @author Joseph Lizier (email,
  * www)
  */
-public interface EntropyCalculator {
-
-	/**
-	 * Initialise the calculator for (re-)use, with the existing (or default) values of calculator-specific parameters
- 	 * Clears any PDFs of previously supplied observations.
-	 */
-	public void initialise() throws Exception;
-	
-	/**
-	 * Set properties for the underlying calculator implementation.
-	 * New property values are not guaranteed to take effect until the next call
-	 *  to an initialise method. 
-	 * 
-	 * 
      No general properties are defined at the interface level, i.e.
-	 * there are only calculator-specific properties.
      
-	 * 
-	 * 
      Unknown property values are ignored.
      
-	 * 
-	 * @param propertyName name of the property
-	 * @param propertyValue value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public void setProperty(String propertyName, String propertyValue) throws Exception;
+public interface EntropyCalculator extends InfoMeasureCalculatorContinuous {
 
 	// TODO Add addObservations() methods for entropy calculator
 	
@@ -81,17 +59,4 @@ public interface EntropyCalculator {
 	 */
 	public void setObservations(double observations[]);
 	
-	/**
-	 * Compute the entropy from the previously-supplied samples.
-	 * 
-	 * @return the entropy estimate
-	 */
-	public double computeAverageLocalOfObservations();
-
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
 }
diff --git a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java
index 8bea318..eb4e5f3 100755
--- a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java
@@ -54,8 +54,36 @@ Theory' (John Wiley & Sons, New York, 1991).
  * @author Joseph Lizier (email,
  * www)
  */
-public interface EntropyCalculatorMultiVariate {
+public interface EntropyCalculatorMultiVariate 
+	extends InfoMeasureCalculatorContinuous {
 
+	/**
+	 * Property name for the number of dimensions
+	 */
+	public static final String NUM_DIMENSIONS_PROP_NAME = "NUM_DIMENSIONS";
+	
+	/**
+	 * Set properties for the underlying calculator implementation.
+	 * New property values are not guaranteed to take effect until the next call
+	 *  to an initialise method. 
+	 * 
+	 * 
      Property names defined at the interface level, and what their
+	 * values should represent, include:
      
+	 * 
        
+	 *  
      • {@link #NUM_DIMENSIONS_PROP_NAME} -- number of dimensions in the joint
+	 * 		variable that we are computing the entropy of.
        • 
+	 * 
      
+	 *  
+	 * 
      Unknown property values are ignored.
      
+	 * 
+	 * 
      Note that implementing classes may defined additional properties.
      
+	 * 
+	 * @param propertyName name of the property
+	 * @param propertyValue value of the property
+	 * @throws Exception for invalid property values
+	 */
+	public void setProperty(String propertyName, String propertyValue) throws Exception;
+	
 	/**
 	 * Initialise the calculator for (re-)use, with the existing (or default) values
 	 * of calculator-specific parameters.
@@ -65,20 +93,6 @@ public interface EntropyCalculatorMultiVariate {
 	 */
 	public void initialise(int dimensions);
 	
-	/**
-	 * Set properties for the underlying calculator implementation.
-	 * New property values are not guaranteed to take effect until the next call
-	 *  to an initialise method. 
-	 * 
-	 * 
      No general properties are defined at the interface level, i.e.
-	 * there are only calculator-specific properties.
      
-	 *  
-	 * @param propertyName name of the property
-	 * @param propertyValue value of the property
-	 * @throws Exception for invalid property values
-	 */
-	public void setProperty(String propertyName, String propertyValue) throws Exception;
-
 	/**
 	 * Set the observations for which to compute the PDFs for the entropy
 	 * Should only be called once, the last call contains the
@@ -93,13 +107,6 @@ public interface EntropyCalculatorMultiVariate {
 	 */
 	public void setObservations(double observations[][]) throws Exception;
 	
-	/**
-	 * Compute the entropy from the previously-supplied samples.
-	 * 
-	 * @return the entropy estimate, in bits or nats depending on the estimator.
-	 */
-	public double computeAverageLocalOfObservations();
-	
 	/**
 	 * Compute the local entropy values for each of the
 	 * supplied samples in newObservations.
@@ -127,29 +134,4 @@ public interface EntropyCalculatorMultiVariate {
 	 */
 	public double[] computeLocalOfPreviousObservations() throws Exception;
 	
-	/**
-	 * Return the entropy last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise(int)} call.
-	 * 
-	 * @return the last computed entropy value
-	 */
-	public double getLastAverage();
-	
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Get the number of samples to be used for the PDFs here 
-	 * which have been supplied by calls to
-	 * {@link #setObservations(double[][])}.
-	 * 
-	 * @return the number of samples to be used for the PDFs
-	 * @throws Exception
-	 */
-	public int getNumObservations() throws Exception;
 }
diff --git a/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java b/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java
index e2734a1..02831b4 100644
--- a/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java
+++ b/java/source/infodynamics/measures/continuous/InfoMeasureCalculatorContinuous.java
@@ -106,6 +106,7 @@ public interface InfoMeasureCalculatorContinuous {
 	 * from the previously-supplied samples.
 	 * 
 	 * @return the estimate of the measure
+	 *  (in bits or nats, depending on the estimator)
 	 */
 	public abstract double computeAverageLocalOfObservations() throws Exception;
 	
diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java
index c15ecad..4064af9 100755
--- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java
+++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java
@@ -59,6 +59,16 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
 	 */
 	protected boolean debug;
 	
+	/**
+	 * Total number of observations supplied.
+	 */
+	protected int totalObservations;
+	
+	/**
+	 * Store the last computed average Entropy
+	 */
+	protected double lastAverage;
+	
 	/**
 	 * Construct an instance
 	 */
@@ -66,6 +76,7 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
 		// Nothing to do
 	}
 	
+	@Override
 	public void initialise() {
 		// Nothing to do
 	}
@@ -73,6 +84,7 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
 	public void setObservations(double[] observations) {
 		variance = MatrixUtils.stdDev(observations);
 		variance *= variance;
+		totalObservations = observations.length;
 	}
 
 	/**
@@ -100,10 +112,13 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
 	 * @return the entropy of the previously provided observations or from the supplied
 	 *   covariance matrix. Entropy returned in nats, not bits!
 	 */
+	@Override
 	public double computeAverageLocalOfObservations() {
-		return 0.5 * Math.log(2.0*Math.PI*Math.E*variance);
+		lastAverage = 0.5 * Math.log(2.0*Math.PI*Math.E*variance);
+		return lastAverage;
 	}
 
+	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 	}
@@ -111,9 +126,29 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
 	/**
 	 * No properties are defined here, so this method will have no effect.
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue)
 			throws Exception {
 		// No properties to set here
 	}
 
+	/**
+	 * No properties are defined here, so this method will always return null.
+	 */
+	@Override
+	public String getProperty(String propertyName)
+			throws Exception {
+		// No properties to return here
+		return null;
+	}
+
+	@Override
+	public int getNumObservations() throws Exception {
+		return totalObservations;
+	}
+
+	@Override
+	public double getLastAverage() {
+		return lastAverage;
+	}
 }
diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java
index a7865c9..aaf0504 100755
--- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java
+++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java
@@ -72,22 +72,22 @@ public class EntropyCalculatorMultiVariateGaussian
 	/**
 	 * Number of dimensions for our multivariate data
 	 */
-	protected int dimensions;
+	protected int dimensions = 1;
 	
 	/**
 	 * Determinant of the covariance matrix; stored to save computation time
 	 */
-	protected double detCovariance;
+	protected double detCovariance = 0.0;
 
 	/**
 	 * Last average entropy we computed
 	 */
-	protected double lastAverage;
+	protected double lastAverage = 0;
 	
 	/**
 	 * Whether we are in debug mode
 	 */
-	protected boolean debug;
+	protected boolean debug = false;
 	
 	/**
 	 * Construct an instance
@@ -96,12 +96,18 @@ public class EntropyCalculatorMultiVariateGaussian
 		// Nothing to do
 	}
 	
+	@Override
+	public void initialise() throws Exception {
+		initialise(dimensions);
+	}
+
 	public void initialise(int dimensions) {
 		means = null;
 		L = null;
 		observations = null;
 		this.dimensions = dimensions;
 		detCovariance = 0;
+		lastAverage = 0.0;
 	}
 
 	/**
@@ -109,6 +115,7 @@ public class EntropyCalculatorMultiVariateGaussian
 	 *  dimensions, or covariance matrix is not positive definite (reflecting 
 	 *  redundant variables in the observations)
 	 */
+	@Override
 	public void setObservations(double[][] observations) throws Exception {
 		// Check that the observations was of the correct number of dimensions:
 		if (observations[0].length != dimensions) {
@@ -187,6 +194,7 @@ public class EntropyCalculatorMultiVariateGaussian
 	 * @return the joint entropy of the previously provided observations or from the
 	 *  supplied covariance matrix. Returned in nats (NOT bits).
 	 */
+	@Override
 	public double computeAverageLocalOfObservations() {
 		// Simple way:
 		// detCovariance = MatrixUtils.determinantSymmPosDefMatrix(covariance);
@@ -197,18 +205,38 @@ public class EntropyCalculatorMultiVariateGaussian
 		return lastAverage;
 	}
 
+	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 	}
 
-	/**
-	 * No properties are defined here, so this method will have no effect.
-	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue)
 			throws Exception {
-		// No properties to set here
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			dimensions = Integer.parseInt(propertyValue);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			return Integer.toString(dimensions);
+		} else {
+			// No property was set, and no superclass to call:
+			return null;
+		}
+	}
+	
+	@Override
 	public double getLastAverage() {
 		return lastAverage;
 	}
diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java
index de328a9..d1e8e10 100755
--- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java
+++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java
@@ -50,6 +50,10 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 	 * Number of observations supplied
 	 */
 	protected int totalObservations = 0;
+	/**
+	 * Last computed average
+	 */
+	private double lastAverage;
 	/**
 	 * Whether we're in debug mode
 	 */
@@ -93,8 +97,10 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 		svke = new KernelEstimatorUniVariate();
 		svke.setDebug(debug);
 		svke.setNormalise(normalise);
+		lastAverage = 0.0;
 	}
 
+	@Override
 	public void initialise() {
 		initialise(kernelWidth);
 	}
@@ -110,15 +116,18 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 	 */
 	public void initialise(double kernelWidth) {
 		this.kernelWidth = kernelWidth;
+		lastAverage = 0.0;
 		svke.initialise(kernelWidth);
 	}
 
+	@Override
 	public void setObservations(double observations[]) {
 		this.observations = observations;
 		svke.setObservations(observations);
 		totalObservations = observations.length;
 	}
 	
+	@Override
 	public double computeAverageLocalOfObservations() {
 		double entropy = 0.0;
 		for (int t = 0; t < observations.length; t++) {
@@ -131,9 +140,11 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 						(entropy/Math.log(2.0)));
 			}
 		}
-		return entropy / (double) totalObservations / Math.log(2.0);
+		lastAverage = entropy / (double) totalObservations / Math.log(2.0);
+		return lastAverage;
 	}
 	
+	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 		if (svke != null) {
@@ -163,6 +174,7 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 	 * @param propertyValue value of the property
 	 * @throws Exception for invalid property values
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue) throws Exception {
 		boolean propertySet = true;
 
@@ -186,4 +198,27 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(KERNEL_WIDTH_PROP_NAME) ||
+				propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			return Double.toString(kernelWidth);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			return Boolean.toString(normalise);
+		} else {
+			// no superclass to try:
+			return null;
+		}
+	}
+
+	@Override
+	public int getNumObservations() throws Exception {
+		return totalObservations;
+	}
+
+	@Override
+	public double getLastAverage() {
+		return lastAverage;
+	}
+
 }
diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java
index 6ee65a5..e039d4f 100755
--- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java
+++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java
@@ -74,6 +74,11 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 */
 	public static final String NORMALISE_PROP_NAME = "NORMALISE";
 	
+	/**
+	 * Number of joint variables/dimensions
+	 */
+	protected int dimensions = 1;
+
 	/**
 	 * Default value for kernel width
 	 */
@@ -101,6 +106,11 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		lastEntropy = 0.0;
 	}
 
+	@Override
+	public void initialise() throws Exception {
+		initialise(dimensions);
+	}
+
 	public void initialise(int dimensions) {
 		initialise(dimensions, kernelWidth);
 	}
@@ -118,17 +128,20 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 */
 	public void initialise(int dimensions, double kernelWidth) {
 		this.kernelWidth = kernelWidth;
+		this.dimensions = dimensions;
 		mvke.initialise(dimensions, kernelWidth);
 		// this.dimensions = dimensions;
 		lastEntropy = 0.0;
 	}
 
+	@Override
 	public void setObservations(double observations[][]) {
 		mvke.setObservations(observations);
 		totalObservations = observations.length;
 		this.observations = observations;
 	}
 
+	@Override
 	public double computeAverageLocalOfObservations() {
 		double entropy = 0.0;
 		for (int b = 0; b < totalObservations; b++) {
@@ -143,12 +156,28 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		return lastEntropy;
 	}
 
-	
+	@Override
 	public double[] computeLocalOfPreviousObservations() {
-		return computeLocalUsingPreviousObservations(observations);
+		return computeLocalUsingPreviousObservations(true, observations);
 	}
 
+	@Override
 	public double[] computeLocalUsingPreviousObservations(double states[][]) {
+		return computeLocalUsingPreviousObservations(false, states);
+	}
+	
+	/**
+	 * Internal method for computing locals of a set of observations
+	 *  based on existing PDFs.
+	 * 
+	 * @param isPreviousObservations whether we're using the points
+	 *   that the PDFs were computed from or not.
+	 * @param states samples to compute the local joint entropies on.
+	 * @return
+	 */
+	protected double[] computeLocalUsingPreviousObservations(
+			boolean isPreviousObservations, double states[][]) {
+		
 		double entropy = 0.0;
 		double[] localEntropy = new double[states.length];
 		for (int b = 0; b < states.length; b++) {
@@ -161,14 +190,19 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 			}
 		}
 		entropy /= (double) totalObservations / Math.log(2.0);
+		if (isPreviousObservations) {
+			lastEntropy = entropy; 
+		}
 		return localEntropy;
 	}
 
+	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 		mvke.setDebug(debug);
 	}
 
+	@Override
 	public double getLastAverage() {
 		return lastEntropy;
 	}
@@ -187,6 +221,7 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 * 			is an absolute value. Default is {@link #DEFAULT_KERNEL_WIDTH}.
 	 * 		
    16. {@link #NORMALISE_PROP_NAME} -- whether to normalise the incoming variable values
 	 * 			to mean 0, standard deviation 1, or not (default false). Sets {@link #normalise}.
      17. 
+	 *  	
    17. any valid properties for {@link EntropyCalculatorMultiVariate#setProperty(String, String)}.
      18. 
 	 *  
 	 * 
 	 * 
      Unknown property values are ignored.
      
@@ -195,6 +230,7 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 * @param propertyValue value of the property
 	 * @throws Exception for invalid property values
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue) throws Exception {
 		boolean propertySet = true;
 
@@ -208,6 +244,8 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
 			normalise = Boolean.parseBoolean(propertyValue);
 			mvke.setNormalise(normalise);
+		} else if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			dimensions = Integer.parseInt(propertyValue);
 		} else {
 			// No property was set
 			propertySet = false;
@@ -218,6 +256,22 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(KERNEL_WIDTH_PROP_NAME) ||
+				propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			return Double.toString(kernelWidth);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			return Boolean.toString(normalise);
+		} else if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			return Integer.toString(dimensions);
+		} else {
+			// No property was set, and no superclass to call:
+			return null;
+		}
+	}
+
+	@Override
 	public int getNumObservations() throws Exception {
 		return totalObservations;
 	}
diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
index 68480f3..edef0a1 100755
--- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
+++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
@@ -60,9 +60,9 @@ public class EntropyCalculatorMultiVariateKozachenko
 
 	protected boolean debug = false;
 	private int totalObservations;
-	private int dimensions;
+	private int dimensions = 1;
 	protected double[][] rawData;
-	private double lastEntropy;
+	private double lastAverage = 0.0;
 	private double[] lastLocalEntropy;
 	private boolean isComputed;
 	
@@ -73,11 +73,15 @@ public class EntropyCalculatorMultiVariateKozachenko
 	 */
 	public EntropyCalculatorMultiVariateKozachenko() {
 		totalObservations = 0;
-		dimensions = 0;
 		isComputed = false;
 		lastLocalEntropy = null;
 	}
 
+	@Override
+	public void initialise() throws Exception {
+		initialise(dimensions);
+	}
+
 	public void initialise(int dimensions) {
 		this.dimensions = dimensions;
 		rawData = null;
@@ -86,14 +90,33 @@ public class EntropyCalculatorMultiVariateKozachenko
 		lastLocalEntropy = null;
 	}
 	
-	/**
-	 * No properties are defined here, so this method will have no effect.
-	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue)
 			throws Exception {
-		// No properties here to set
+		boolean propertySet = true;
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			dimensions = Integer.parseInt(propertyValue);
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			return Integer.toString(dimensions);
+		} else {
+			// No property was set, and no superclass to call:
+			return null;
+		}
+	}
+
+	@Override
 	public void setObservations(double[][] observations) {
 		rawData = observations;
 		totalObservations = observations.length;
@@ -137,9 +160,10 @@ public class EntropyCalculatorMultiVariateKozachenko
 	/**
 	 * @return entropy in natural units
 	 */
+	@Override
 	public double computeAverageLocalOfObservations() {
 		if (isComputed) {
-			return lastEntropy;
+			return lastAverage;
 		}
 		double sdTermHere = sdTerm(totalObservations, dimensions);
 		double emConstHere = eulerMacheroniTerm(totalObservations);
@@ -167,7 +191,7 @@ public class EntropyCalculatorMultiVariateKozachenko
 		}
 		entropy += emConstHere;
 		entropy += sdTermHere;
-		lastEntropy = entropy;
+		lastAverage = entropy;
 		isComputed = true;
 		return entropy;
 	}
@@ -175,6 +199,7 @@ public class EntropyCalculatorMultiVariateKozachenko
 	/**
 	 * @return local entropies in natural units
 	 */
+	@Override
 	public double[] computeLocalOfPreviousObservations() {
 		if (lastLocalEntropy != null) {
 			return lastLocalEntropy;
@@ -205,7 +230,7 @@ public class EntropyCalculatorMultiVariateKozachenko
 			}
 		}
 		entropy /= (double) totalObservations;
-		lastEntropy = entropy;
+		lastAverage = entropy;
 		lastLocalEntropy = localEntropy;
 		return localEntropy;
 	}
@@ -213,6 +238,7 @@ public class EntropyCalculatorMultiVariateKozachenko
 	/**
 	 * Not implemented yet
 	 */
+	@Override
 	public double[] computeLocalUsingPreviousObservations(double[][] states) throws Exception {
 		throw new Exception("Local method for other data not implemented");
 	}
@@ -289,15 +315,19 @@ public class EntropyCalculatorMultiVariateKozachenko
 		return result;
 	}
 	
+	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 	}
 
+	@Override
 	public double getLastAverage() {
-		return lastEntropy;
+		return lastAverage;
 	}
 
+	@Override
 	public int getNumObservations() {
 		return totalObservations;
 	}
+
 }

From 057f17e338657da0adc8c45fb6bc3eb3fa20bf3f Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 23:11:37 +1000
Subject: [PATCH 41/78] Making continuous PredictiveInfoCalculator classes
 implement the InfoMeasureCalculatorContinuous interface. For the interfaces,
 this means removing methods where duplicated. And fixed the common
 implementing class to have the missing getProperty method.

---
 .../continuous/PredictiveInfoCalculator.java  | 50 ++-----------------
 ...PredictiveInfoCalculatorViaMutualInfo.java | 28 ++++++-----
 2 files changed, 19 insertions(+), 59 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
index 946d00e..7a0aca3 100755
--- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
+++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java
@@ -94,7 +94,8 @@ import infodynamics.utils.EmpiricalNullDistributionComputer;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface PredictiveInfoCalculator extends EmpiricalNullDistributionComputer {
+public interface PredictiveInfoCalculator
+	extends InfoMeasureCalculatorContinuous, EmpiricalNullDistributionComputer {
 
 	/**
 	 * Property name for embedding length k of
@@ -117,13 +118,6 @@ public interface PredictiveInfoCalculator extends EmpiricalNullDistributionCompu
 	 */
 	public static final String TAU_PROP_NAME = "TAU";
 
-	/**
-	 * Initialise the calculator for (re-)use, with the existing (or default) values of parameters
-	 * Clears any PDFs of previously supplied observations.
-	 * 
-	 */
-	public void initialise() throws Exception;
-
 	/**
 	 * Initialise the calculator for (re-)use, with some parameters
 	 * supplied here, and existing (or default) values of other parameters
@@ -166,6 +160,7 @@ public interface PredictiveInfoCalculator extends EmpiricalNullDistributionCompu
 	 * @param propertyValue value of the property
 	 * @throws Exception for invalid property values
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue) throws Exception;
 
 	/**
@@ -234,13 +229,6 @@ public interface PredictiveInfoCalculator extends EmpiricalNullDistributionCompu
 	public void setObservations(double[] observations,
 			boolean[] valid) throws Exception;
 
-	/**
-	 * Compute the PI from the previously-supplied samples.
-	 * 
-	 * @return the PI estimate
-	 */
-	public double computeAverageLocalOfObservations() throws Exception;
-
 	/**
 	 * Compute the local PI values for each of the
 	 * previously-supplied samples.
@@ -282,36 +270,4 @@ public interface PredictiveInfoCalculator extends EmpiricalNullDistributionCompu
 	 */
 	public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception;
 
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Return the PI last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise()} call.
-	 * 
-	 * @return the last computed PI value
-	 */
-	public double getLastAverage();
-
-	/**
-	 * Get the number of samples to be used for the PDFs here 
-	 * which have been supplied by calls to
-	 * {@link #setObservations(double[])}, {@link #addObservations(double[])}
-	 * etc.
-	 * 
-	 * 
      Note that the number of samples is not equal to the length of time-series
-	 * supplied (since we need to accumulate the first and last
-	 * (k-1)*tau + 1
-	 * values of each time-series).
-	 * 
      
-	 * 
-	 * @return the number of samples to be used for the PDFs
-	 * @throws Exception
-	 */
-	public int getNumObservations() throws Exception;
 }
diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
index 108292a..8cfb53c 100755
--- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
+++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java
@@ -217,6 +217,22 @@ public class PredictiveInfoCalculatorViaMutualInfo implements
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(K_PROP_NAME) ||
+			propertyName.equalsIgnoreCase(K_EMBEDDING_PROP_NAME)) {
+			return Integer.toString(k);
+		} else if (propertyName.equalsIgnoreCase(TAU_PROP_NAME)) {
+			return Integer.toString(tau);
+		} else {
+			// No property was set on this class, assume it is for the underlying
+			//  MI calculator, even if it is for 
+			//  MutualInfoCalculatorMultiVariate.PROP_TIME_DIFF which
+			//  is not a valid property for the PI calculator:
+			return miCalc.getProperty(propertyName);
+		}
+	}
+	
 	/* (non-Javadoc)
 	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#setObservations(double[])
 	 */
@@ -372,9 +388,6 @@ public class PredictiveInfoCalculatorViaMutualInfo implements
 		return startAndEndTimePairs;
 	}
 	
-	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#computeAverageLocalOfObservations()
-	 */
 	@Override
 	public double computeAverageLocalOfObservations() throws Exception {
 		return miCalc.computeAverageLocalOfObservations();
@@ -434,26 +447,17 @@ public class PredictiveInfoCalculatorViaMutualInfo implements
 		return miCalc.computeSignificance(newOrderings);
 	}
 
-	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#setDebug(boolean)
-	 */
 	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
 		miCalc.setDebug(debug);
 	}
 
-	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#getLastAverage()
-	 */
 	@Override
 	public double getLastAverage() {
 		return miCalc.getLastAverage();
 	}
 
-	/* (non-Javadoc)
-	 * @see infodynamics.measures.continuous.PredictiveInfoCalculator#getNumObservations()
-	 */
 	@Override
 	public int getNumObservations() throws Exception {
 		return miCalc.getNumObservations();

From 086bee594509fa8dfab88a17f4ee3f5f51290698 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Fri, 18 Aug 2017 23:42:43 +1000
Subject: [PATCH 42/78] Making continuous MultiInfoCalculator classes implement
 the InfoMeasureCalculatorContinuous interface. For the interfaces, this means
 removing methods where duplicated. For implementing classes, this means
 adding the missing methods.

---
 .../continuous/MultiInfoCalculator.java       | 28 ++-----------------
 .../continuous/MultiInfoCalculatorCommon.java | 23 ++++++++++++++-
 .../gaussian/MultiInfoCalculatorGaussian.java |  1 -
 .../kernel/MultiInfoCalculatorKernel.java     | 13 +++++++++
 .../kraskov/MultiInfoCalculatorKraskov.java   | 24 ++++++++++++++--
 5 files changed, 60 insertions(+), 29 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/MultiInfoCalculator.java b/java/source/infodynamics/measures/continuous/MultiInfoCalculator.java
index 28ea57e..7236d80 100755
--- a/java/source/infodynamics/measures/continuous/MultiInfoCalculator.java
+++ b/java/source/infodynamics/measures/continuous/MultiInfoCalculator.java
@@ -76,7 +76,8 @@ import infodynamics.utils.EmpiricalMeasurementDistribution;
  * @author Joseph Lizier (email,
  * www)
  */
-public interface MultiInfoCalculator {
+public interface MultiInfoCalculator
+	extends InfoMeasureCalculatorContinuous {
 
 	/**
 	 * Property name for whether to normalise incoming values to mean 0,
@@ -120,6 +121,7 @@ public interface MultiInfoCalculator {
 	 * @param propertyValue value of the property
 	 * @throws Exception for invalid property values
 	 */
+	@Override
 	public void setProperty(String propertyName, String propertyValue) throws Exception;
 	
 	/**
@@ -189,14 +191,6 @@ public interface MultiInfoCalculator {
 	 */
 	public void finaliseAddObservations() throws Exception;
 
-	/**
-	 * Compute the multi-information from the previously-supplied samples.
-	 * 
-	 * @return the estimate of the multi-information
-	 * @throws Exception
-	 */
-	public double computeAverageLocalOfObservations() throws Exception;
-
 	/**
 	 * 
      Computes the local values of the multi-information,
 	 *  for each valid observation in the previously supplied observations
@@ -337,20 +331,4 @@ public interface MultiInfoCalculator {
 	 *   is not equal to the number N samples that were previously supplied.
 	 */
 	public EmpiricalMeasurementDistribution computeSignificance(int[][][] newOrderings) throws Exception;
-	
-	/**
-	 * Set or clear debug mode for extra debug printing to stdout
-	 * 
-	 * @param debug new setting for debug mode (on/off)
-	 */
-	public void setDebug(boolean debug);
-	
-	/**
-	 * Return the multi-information last calculated in a call to {@link #computeAverageLocalOfObservations()}
-	 * or {@link #computeLocalOfPreviousObservations()} after the previous
-	 * {@link #initialise(int)} call.
-	 * 
-	 * @return the last computed multi-information value
-	 */
-	public double getLastAverage();
 }
diff --git a/java/source/infodynamics/measures/continuous/MultiInfoCalculatorCommon.java b/java/source/infodynamics/measures/continuous/MultiInfoCalculatorCommon.java
index f6ad8e7..256e555 100755
--- a/java/source/infodynamics/measures/continuous/MultiInfoCalculatorCommon.java
+++ b/java/source/infodynamics/measures/continuous/MultiInfoCalculatorCommon.java
@@ -45,7 +45,7 @@ public abstract class MultiInfoCalculatorCommon implements MultiInfoCalculator {
 	/**
 	 * Number of joint variables to consider
 	 */
-	protected int dimensions = 0;
+	protected int dimensions = 1;
 	/**
 	 * Number of samples supplied
 	 */
@@ -81,6 +81,11 @@ public abstract class MultiInfoCalculatorCommon implements MultiInfoCalculator {
 	private double samplingFactor = 0.1;
 	private Random rand;
 
+	@Override
+	public void initialise() {
+		initialise(dimensions);
+	}
+	
 	@Override
 	public void initialise(int dimensions) {
 		this.dimensions = dimensions;
@@ -127,6 +132,17 @@ public abstract class MultiInfoCalculatorCommon implements MultiInfoCalculator {
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
+			return Boolean.toString(normalise);
+		} else if (propertyName.equalsIgnoreCase(SAMPLING_FACTOR_PROP_NAME)) {
+			return Double.toString(samplingFactor);
+		} else {
+			return null;
+		}
+	}
+
 	@Override
 	public void setObservations(double[][] observations) throws Exception {
 		startAddObservations();
@@ -258,6 +274,11 @@ public abstract class MultiInfoCalculatorCommon implements MultiInfoCalculator {
 		return miSurrogateCalculator.computeAverageLocalOfObservations();
 	}
 
+	@Override
+	public int getNumObservations() throws Exception {
+		return totalObservations;
+	}
+
 	@Override
 	public void setDebug(boolean debug) {
 		this.debug = debug;
diff --git a/java/source/infodynamics/measures/continuous/gaussian/MultiInfoCalculatorGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/MultiInfoCalculatorGaussian.java
index 54b8dcc..dd4b4af 100644
--- a/java/source/infodynamics/measures/continuous/gaussian/MultiInfoCalculatorGaussian.java
+++ b/java/source/infodynamics/measures/continuous/gaussian/MultiInfoCalculatorGaussian.java
@@ -2,7 +2,6 @@ package infodynamics.measures.continuous.gaussian;
 
 import infodynamics.measures.continuous.MultiInfoCalculatorCommon;
 import infodynamics.utils.MatrixUtils;
-import infodynamics.utils.MathsUtils;
 
 /**
  * 
      Computes the differential multi-information of a given multivariate
diff --git a/java/source/infodynamics/measures/continuous/kernel/MultiInfoCalculatorKernel.java b/java/source/infodynamics/measures/continuous/kernel/MultiInfoCalculatorKernel.java
index 950ace4..d78dddb 100755
--- a/java/source/infodynamics/measures/continuous/kernel/MultiInfoCalculatorKernel.java
+++ b/java/source/infodynamics/measures/continuous/kernel/MultiInfoCalculatorKernel.java
@@ -209,6 +209,19 @@ public class MultiInfoCalculatorKernel
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(KERNEL_WIDTH_PROP_NAME) ||
+				propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
+			return Double.toString(kernelWidth);
+		} else if (propertyName.equalsIgnoreCase(DYN_CORR_EXCL_TIME_NAME)) {
+			return Integer.toString(dynCorrExclTime);
+		} else {
+			// Try the superclass, including for PROP_NORMALISE
+			return super.getProperty(propertyName);
+		}
+	}
+
 	@Override
 	public void startAddObservations() {
 		if (dynCorrExcl) {
diff --git a/java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov.java b/java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov.java
index bb8518a..9853774 100755
--- a/java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov.java
+++ b/java/source/infodynamics/measures/continuous/kraskov/MultiInfoCalculatorKraskov.java
@@ -183,7 +183,7 @@ public abstract class MultiInfoCalculatorKraskov
 	 * 
        
 	 *  
      • {@link #PROP_K} -- number of k nearest neighbours to use in joint kernel space
 	 *      in the KSG algorithm (default is 4).
        • 
-	 *  
      • {@link #PROP_NORM_TYPE} -- normalization type to apply to 
+	 *  
      • {@link #PROP_NORM_TYPE} -- norm type to apply to 
 	 *      working out the norms between the points in each marginal space.
 	 * 	    Options are defined by {@link KdTree#setNormType(String)} -
 	 * 	    default is {@link EuclideanUtils#NORM_MAX_NORM}.
        • 
@@ -198,7 +198,9 @@ public abstract class MultiInfoCalculatorKraskov
 	 *      so can be considered as a number of standard deviations of the data.
 	 *      (Recommended by Kraskov. MILCA uses 1e-8; but adds in
 	 *      a random amount of noise in [0,noiseLevel) ).
-	 *      Default 1e-8 to match the noise order in MILCA toolkit..
+	 *      Default 1e-8 to match the noise order in MILCA toolkit.
+	 *   
      • Any property accepted by the superclass
+	 *      {@link MultiInfoCalculatorCommon#setProperty(String, String)}
        • 
 	 * 
      
 	 * 
 	 * 
      Unknown property values are ignored.
      
@@ -244,6 +246,24 @@ public abstract class MultiInfoCalculatorKraskov
 		}
 	}
 
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(PROP_K)) {
+			return Integer.toString(k);
+		} else if (propertyName.equalsIgnoreCase(PROP_NORM_TYPE)) {
+			return KdTree.convertNormTypeToString(normType);
+		} else if (propertyName.equalsIgnoreCase(PROP_DYN_CORR_EXCL_TIME)) {
+			return Integer.toString(dynCorrExclTime);
+		} else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
+			return Double.toString(noiseLevel);
+		} else if (propertyName.equalsIgnoreCase(PROP_NUM_THREADS)) {
+			return Integer.toString(numThreads);
+		} else {
+			// try the superclass, including for PROP_NORMALISE:
+			return super.getProperty(propertyName);
+		}
+	}
+
 	/**
 	 * Set observations from two separate time series: join the rows at each time step
 	 *  together to make a joint vector, then effectively call {@link #setObservations(double[][])}

From e75eb761ca1e07be9fc3a25708d460460a7b345d Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 21 Aug 2017 14:25:01 +1000
Subject: [PATCH 43/78] Split AutoAnalyser into 2 classes, itself and the new
 AutoAnalyserChannelCalculator, keeping all the common
 functionality/generality for other calculator types in AutoAnalyser. This
 will allow us to have the AutoAnalyser for other types of calculators as well
 later.

---
 .../demos/autoanalysis/AutoAnalyser.java      | 653 ++++++++++--------
 .../AutoAnalyserChannelCalculator.java        | 207 ++++++
 .../demos/autoanalysis/AutoAnalyserMI.java    |  15 +-
 .../demos/autoanalysis/AutoAnalyserTE.java    |  19 +-
 4 files changed, 594 insertions(+), 300 deletions(-)
 create mode 100644 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index c7dfbeb..54edbcf 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -18,12 +18,13 @@
 
 package infodynamics.demos.autoanalysis;
 
-import infodynamics.measures.continuous.ChannelCalculatorCommon;
-import infodynamics.measures.discrete.ChannelCalculatorDiscrete;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
 import infodynamics.utils.AnalyticMeasurementDistribution;
 import infodynamics.utils.AnalyticNullDistributionComputer;
 import infodynamics.utils.ArrayFileReader;
 import infodynamics.utils.EmpiricalMeasurementDistribution;
+import infodynamics.utils.EmpiricalNullDistributionComputer;
 import infodynamics.utils.MatrixUtils;
 
 import javax.swing.BorderFactory;
@@ -89,13 +90,16 @@ public abstract class AutoAnalyser extends JFrame
 	
 	// Set options for the Calculator type:
 	public final static String CALC_TYPE_DISCRETE = "Discrete";
+	// Child classes must define options for other calculator types
+	//  and initialise the calcTypes array and unitsForEachCalc array
+	protected String[] calcTypes;
+	protected String[] unitsForEachCalc;
+	
+	// Set additional general options for the Calculator type:
 	public final static String CALC_TYPE_GAUSSIAN = "Gaussian";
 	public final static String CALC_TYPE_KRASKOV  = "Kraskov (KSG)";
 	public final static String CALC_TYPE_KERNEL   = "Kernel";
-	protected String[] calcTypes = {
-			CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
-			CALC_TYPE_KRASKOV, CALC_TYPE_KERNEL};
-	protected String[] unitsForEachCalc = {"bits", "nats", "nats", "bits"};
+	public final static String CALC_TYPE_KOZ_LEO = "Kozachenko-Leonenko";
 	
 	// Properties for each calculator
 	protected static final String DISCRETE_PROPNAME_BASE = "base";
@@ -107,16 +111,15 @@ public abstract class AutoAnalyser extends JFrame
 	protected String[] commonContPropertyNames;
 	protected String[] commonContPropertiesFieldNames;
 	protected String[] commonContPropertyDescriptions;
-	protected String[] gaussianProperties;
-	protected String[] gaussianPropertiesFieldNames;
-	protected String[] gaussianPropertyDescriptions;
-	protected String[] kernelProperties;
-	protected String[] kernelPropertiesFieldNames;
-	protected String[] kernelPropertyDescriptions;
-	protected String[] kraskovProperties;
-	protected String[] kraskovPropertiesFieldNames;
-	protected String[] kraskovPropertyDescriptions;
+	// Children can define properties for specific continuous
+	//  calculators
 
+	// Number of variables that this measure operates on (1 to 3)
+	protected int numVariables = 1;
+	// Labels for each of the fields to enter the column number for the variables.
+	//  Should be overridden by the child classes.
+	protected String[] variableColNumLabels = null;
+	
 	// Store which calculator type we're using:
 	@SuppressWarnings("rawtypes")
 	protected Class calcClass = null;
@@ -128,6 +131,11 @@ public abstract class AutoAnalyser extends JFrame
 
 	protected String measureAcronym = null;
 	
+	// String to describe the relationship between the variables
+	//  being measured here, with format tokens %%d in order for
+	//  each variable. To be assigned by the child class.
+	protected String variableRelationshipFormatString = null;
+	
 	protected JButton computeButton;
 	protected JButton openDataButton;
 	// Stores the current data file
@@ -145,12 +153,19 @@ public abstract class AutoAnalyser extends JFrame
 	// Number of rows and columns of the data:
 	protected int dataRows = 0;
 	protected int dataColumns = 0;
-	// Source column field
-	protected JTextField sourceColTextField;
-	// Destination column field
-	protected JTextField destColTextField;
-	// CheckBox for "all pairs?"
-	protected JCheckBox allPairsCheckBox;
+	// Boolean for whether we include the all combinations checkbox and label on the GUI
+	//  (default false). Child classes can alter this in makeSpecificInitialisations()
+	protected boolean useAllCombosCheckBox = false;
+	// Which word should be used to describe combinations for this measure? "pairs", "variables", etc?
+	// Child classes should override this.
+	protected String wordForCombinations = null;
+	// CheckBox for "all variables?" OR "all pairs?"
+	protected JCheckBox allCombosCheckBox;
+	// Variable text fields
+	protected JTextField[] variableColTextFields;
+	// Boolean for whether we include the statistical significance checkbox and label on the GUI
+	//  (default false). Child classes can alter this in makeSpecificInitialisations()
+	protected boolean useStatSigCheckBox = false;
 	// CheckBox for statistical significance
 	protected JCheckBox statSigCheckBox;
 	// CheckBox for statistical significance analytically
@@ -252,7 +267,7 @@ public abstract class AutoAnalyser extends JFrame
 		Image watermarkImage = (new ImageIcon("JIDT-logo-watermark.png")).getImage();
 		
 		setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
-		setSize(1100,640);
+		setSize(1100,670);
 		setTitle(appletTitle);
 		// Centre in the middle of the screen
 		setLocationRelativeTo(null);
@@ -262,7 +277,7 @@ public abstract class AutoAnalyser extends JFrame
 		calcTypeLabel.setToolTipText("Select estimator type. \"Discrete\" is for discrete or pre-binned data; all others for continuous data.");
 		calcTypeLabel.setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
 		calcTypeComboBox = (JComboBox) new JComboBox(calcTypes);
-		calcTypeComboBox.setSelectedIndex(2); // Select Kraskov by default
+		calcTypeComboBox.setSelectedIndex(2); // Select 2nd continuous estimator by default
 		calcTypeComboBox.addActionListener(this);
 		// Don't set an empty border as it becomes clickable as well,
 		//  we'll use an empty label instead
@@ -270,7 +285,7 @@ public abstract class AutoAnalyser extends JFrame
 
 		// Create the fields for the data file
 		JLabel fileLabel = new JLabel("Data file:");
-		fileLabel.setToolTipText("Must be a text file of time-series values with time increasing in rows, and variables along columns");
+		fileLabel.setToolTipText("Must be a text file of samples or time-series values with time/sample increasing in rows, and variables along columns");
 		dataFileTextField = new JTextField(30);
 		dataFileTextField.setEnabled(false);
 		dataFileTextField.addMouseListener(this);
@@ -283,29 +298,29 @@ public abstract class AutoAnalyser extends JFrame
 		dataFileDescriptorLabel.setHorizontalAlignment(JLabel.RIGHT);
 		dataFileDescriptorLabel.setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
 		
-		// From column:
-		JLabel sourceLabel = new JLabel("Source column:");
-		sourceLabel.setToolTipText("First column is 0.");
-		sourceColTextField = new JTextField(5);
-		sourceColTextField.setEnabled(true);
-		sourceColTextField.setText("0");
-		// Must add document listener, not add action listener
-		sourceColTextField.getDocument().addDocumentListener(this);
-		// To column:
-		JLabel destLabel = new JLabel("Destination column:");
-		destLabel.setToolTipText("First column is 0.");
-		destLabel.setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
-		destColTextField = new JTextField(5);
-		destColTextField.setEnabled(true);
-		destColTextField.setText("1");
-		destColTextField.getDocument().addDocumentListener(this);
+		// We define the all combos checkbox, but we won't add it later
+		//  if it's not used for this measure type.
+		allCombosCheckBox = new JCheckBox("All " + wordForCombinations + "?");
+		allCombosCheckBox.setToolTipText("Compute for all column " + wordForCombinations + "?");
+		allCombosCheckBox.addChangeListener(this);
+       
+		variableColTextFields = new JTextField[numVariables];
+		JLabel[] labels = new JLabel[numVariables];
+		for (int i = 0; i < numVariables; i++) {
+			labels[i] = new JLabel(variableColNumLabels[i] + " column:");
+			labels[i].setToolTipText("First column is 0.");
+			// This border was only done on the 2nd variable before, so may need
+			//  to restrict its use:
+			labels[i].setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
+			variableColTextFields[i] = new JTextField(5);
+			variableColTextFields[i].setEnabled(true);
+			variableColTextFields[i].setText(Integer.toString(i));
+			// Must add document listener, not add action listener
+			variableColTextFields[i].getDocument().addDocumentListener(this);
+		}
 
-		// Checkbox for all pairs
-		allPairsCheckBox = new JCheckBox("All pairs?");
-		allPairsCheckBox.setToolTipText("Compute for all pairs of columns?");
-		allPairsCheckBox.addChangeListener(this);
-		
-		// CheckBox for statistical signficance
+		// We define the CheckBox for statistical signficance,
+		//  but we won't add it later if it's not used for this measure type.
 		statSigCheckBox = new JCheckBox("Add stat. signif.?");
 		statSigCheckBox.setToolTipText("Compute the null surrogate distribution under the null hypothesis that source and target are not related?");
 		statSigCheckBox.addChangeListener(this);
@@ -465,42 +480,40 @@ public abstract class AutoAnalyser extends JFrame
         
         paramsPanel.add(dataFileDescriptorLabel, c);
 
-        // Add the all pairs label and checkbox
-        c.gridwidth = GridBagConstraints.REMAINDER;     //end row
-        c.gridx = 1;
-        c.fill = GridBagConstraints.HORIZONTAL;
-        c.weightx = 1.0;
-        paramsPanel.add(allPairsCheckBox, c);
-        c.gridx = -1; // reset to no indication
-        
-        // Add the source selector
-        c.gridwidth = GridBagConstraints.RELATIVE; //next-to-last
-        c.fill = GridBagConstraints.NONE;      //reset to default
-        c.weightx = 0.0;                       //reset to default
-        paramsPanel.add(sourceLabel, c);
-        c.gridwidth = GridBagConstraints.REMAINDER;     //end row
-        c.fill = GridBagConstraints.HORIZONTAL;
-        c.weightx = 1.0;
-        paramsPanel.add(sourceColTextField, c);
-        // Add the destination selector
-        c.gridwidth = GridBagConstraints.RELATIVE; //next-to-last
-        c.fill = GridBagConstraints.NONE;      //reset to default
-        c.weightx = 0.0;                       //reset to default
-        paramsPanel.add(destLabel, c);
-        c.gridwidth = GridBagConstraints.REMAINDER;     //end row
-        c.fill = GridBagConstraints.HORIZONTAL;
-        c.weightx = 1.0;
-        paramsPanel.add(destColTextField, c);
+        if (useAllCombosCheckBox) {
+            // Add the all combos label and checkbox
+            c.gridwidth = GridBagConstraints.REMAINDER;     //end row
+            c.gridx = 1;
+            c.fill = GridBagConstraints.HORIZONTAL;
+            c.weightx = 1.0;
+            paramsPanel.add(allCombosCheckBox, c);
+            c.gridx = -1; // reset to no indication
+        }
 
-        // Add the CheckBoxes for statistical significance checks:
-        c.gridwidth = GridBagConstraints.RELATIVE; //next-to-last
-        c.fill = GridBagConstraints.NONE;      //reset to default
-        c.weightx = 0.0;                       //reset to default
-        paramsPanel.add(statSigCheckBox, c);
-        c.gridwidth = GridBagConstraints.REMAINDER;     //end row
-        c.fill = GridBagConstraints.HORIZONTAL;
-        c.weightx = 1.0;
-        paramsPanel.add(statSigAnalyticCheckBox, c);
+        // Add the column number selectors
+        for (int i = 0; i < numVariables; i++) {
+            // Add the column selector for variable i
+            c.gridwidth = GridBagConstraints.RELATIVE; //next-to-last
+            c.fill = GridBagConstraints.NONE;      //reset to default
+            c.weightx = 0.0;                       //reset to default
+            paramsPanel.add(labels[i], c);
+            c.gridwidth = GridBagConstraints.REMAINDER;     //end row
+            c.fill = GridBagConstraints.HORIZONTAL;
+            c.weightx = 1.0;
+            paramsPanel.add(variableColTextFields[i], c);		
+        }
+
+        if (useStatSigCheckBox) {
+	        // Add the CheckBoxes for statistical significance checks:
+	        c.gridwidth = GridBagConstraints.RELATIVE; //next-to-last
+	        c.fill = GridBagConstraints.NONE;      //reset to default
+	        c.weightx = 0.0;                       //reset to default
+	        paramsPanel.add(statSigCheckBox, c);
+	        c.gridwidth = GridBagConstraints.REMAINDER;     //end row
+	        c.fill = GridBagConstraints.HORIZONTAL;
+	        c.weightx = 1.0;
+	        paramsPanel.add(statSigAnalyticCheckBox, c);
+        }
 
         // Add dummy label for spacing
         paramsPanel.add(dummyLabel1, c);
@@ -579,7 +592,6 @@ public abstract class AutoAnalyser extends JFrame
 				statSigAnalyticCheckBox.setSelected(false);
 				statSigAnalyticCheckBox.setEnabled(false);
 			}
-
 		}
 		// Else nothing extra to do
 	}
@@ -673,34 +685,24 @@ public abstract class AutoAnalyser extends JFrame
 			return;
 		}
 		
-		int singleSourceColumn = -1, singleDestColumn = -1;
-		Vector sourceDestPairs = new Vector();
-		if (allPairsCheckBox.isSelected()) {
-			// We're doing all pairs
-			for (int s = 0; s < dataColumns; s++) {
-				for (int d = 0; d < dataColumns; d++) {
-					sourceDestPairs.add(new int[] {s, d});
+		int[] singleCalcColumns = new int[numVariables];
+		Vector variableCombinations = new Vector();
+		if (allCombosCheckBox.isSelected()) {
+			// We're doing all combinations
+			fillOutAllCombinations(variableCombinations);
+		} else {
+			// we're doing a single combination
+			for (int i = 0; i < numVariables; i++) {
+				singleCalcColumns[i] = Integer.parseInt(variableColTextFields[i].getText());
+				if ((singleCalcColumns[i] < 0) || (singleCalcColumns[i] >= dataColumns)) {
+					JOptionPane.showMessageDialog(this,
+							String.format("%s column must be between 0 and %d for this data set",
+									variableColNumLabels[i], dataColumns-1));
+					resultsLabel.setText(" ");				
+					return;
 				}
 			}
-		} else {
-			// we're doing a single source-destination pair
-			singleSourceColumn = Integer.parseInt(sourceColTextField.getText());
-			singleDestColumn = Integer.parseInt(destColTextField.getText());
-			sourceDestPairs.add(new int[] {singleSourceColumn, singleDestColumn});
-			if ((singleSourceColumn < 0) || (singleSourceColumn >= dataColumns)) {
-				JOptionPane.showMessageDialog(this,
-						String.format("Source column must be between 0 and %d for this data set",
-								dataColumns-1));
-				resultsLabel.setText(" ");				
-				return;
-			}
-			if ((singleDestColumn < 0) || (singleDestColumn >= dataColumns)) {
-				JOptionPane.showMessageDialog(this,
-						String.format("Destination column must be between 0 and %d for this data set",
-								dataColumns-1));
-				resultsLabel.setText(" ");
-				return;
-			}
+			variableCombinations.add(singleCalcColumns);
 		}
 
 		// Generate headers:
@@ -745,8 +747,8 @@ public abstract class AutoAnalyser extends JFrame
 		try{
 			// Create both discrete and continuous calculators to make
 			//  following code simpler:
-			ChannelCalculatorCommon calcContinuous = null;
-			ChannelCalculatorDiscrete calcDiscrete = null;
+			InfoMeasureCalculatorContinuous calcContinuous = null;
+			InfoMeasureCalculatorDiscrete calcDiscrete = null;
 			
 			// Construct an instance of the selected calculator:
 			String javaConstructorLine = null;
@@ -781,6 +783,14 @@ public abstract class AutoAnalyser extends JFrame
 							calcContinuous.getClass().getSimpleName() + "\n";
 					matlabConstructorLine = "calc = javaObject('infodynamics.measures.continuous.kernel." +
 							calcContinuous.getClass().getSimpleName() + "');\n";
+				} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KOZ_LEO)) {
+					// Cover the calculator and any references to other classes
+					javaCode.append("import infodynamics.measures.continuous.kozachenko.*;\n");
+					javaConstructorLine = "    calc = new " + calcContinuous.getClass().getSimpleName() + "();\n";
+					pythonPreConstructorLine = "calcClass = JPackage(\"infodynamics.measures.continuous.kozachenko\")." +
+							calcContinuous.getClass().getSimpleName() + "\n";
+					matlabConstructorLine = "calc = javaObject('infodynamics.measures.continuous.kozachenko." +
+							calcContinuous.getClass().getSimpleName() + "');\n";
 				} else {
 					throw new Exception("No recognised calculator selected: " +
 							selectedCalcType);
@@ -798,15 +808,21 @@ public abstract class AutoAnalyser extends JFrame
 			javaCode.append("    ArrayFileReader afr = new ArrayFileReader(dataFile);\n");
 			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
 				javaCode.append("    int[][] data = afr.getInt2DMatrix();\n");
-				if (! allPairsCheckBox.isSelected()) {
-					javaCode.append("    int[] source = MatrixUtils.selectColumn(data, " + singleSourceColumn + ");\n");
-					javaCode.append("    int[] dest = MatrixUtils.selectColumn(data, " + singleDestColumn + ");\n\n");
+				if (! allCombosCheckBox.isSelected()) {
+					for (int i=0; i < numVariables; i++) {
+						javaCode.append("    int[] " +  variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.selectColumn(data, " + singleCalcColumns[i] + ");\n");
+					}
+					javaCode.append("\n");
 				}
 			} else {
 				javaCode.append("    double[][] data = afr.getDouble2DMatrix();\n");
-				if (! allPairsCheckBox.isSelected()) {
-					javaCode.append("    double[] source = MatrixUtils.selectColumn(data, " + singleSourceColumn + ");\n");
-					javaCode.append("    double[] dest = MatrixUtils.selectColumn(data, " + singleDestColumn + ");\n\n");
+				if (! allCombosCheckBox.isSelected()) {
+					for (int i=0; i < numVariables; i++) {
+						javaCode.append("    double[] " +  variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.selectColumn(data, " + singleCalcColumns[i] + ");\n");
+					}
+					javaCode.append("\n");
 				}
 			}
 			// 2. Python
@@ -818,21 +834,38 @@ public abstract class AutoAnalyser extends JFrame
 			}
 			pythonCode.append("# As numpy array:\n");
 			pythonCode.append("data = numpy.array(dataRaw)\n");
-			if (! allPairsCheckBox.isSelected()) {
-				pythonCode.append("source = JArray(JInt, 1)(data[:," + singleSourceColumn + "].tolist())\n");
-				pythonCode.append("dest = JArray(JInt, 1)(data[:," + singleDestColumn + "].tolist())\n\n");
+			if (! allCombosCheckBox.isSelected()) {
+				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+					for (int i=0; i < numVariables; i++) {
+						pythonCode.append(variableColNumLabels[i].toLowerCase() + " = JArray(JInt, 1)(data[:," +
+								singleCalcColumns[i] + "].tolist())\n");
+					}
+					pythonCode.append("\n");
+				} else {
+					for (int i=0; i < numVariables; i++) {
+						pythonCode.append(variableColNumLabels[i].toLowerCase() + " = data[:," +
+								singleCalcColumns[i] + "]\n");
+					}
+					pythonCode.append("\n");
+				}
 			}
 			// 3. Matlab
 			matlabCode.append("% " + loadDataComment);
 			matlabCode.append("data = load('" + dataFile.getAbsolutePath() + "');\n");
-			if (! allPairsCheckBox.isSelected()) {
+			if (! allCombosCheckBox.isSelected()) {
 				matlabCode.append("% Column indices start from 1 in Matlab:\n");
 				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-					matlabCode.append("source = octaveToJavaIntArray(data(:," + (singleSourceColumn+1) + "));\n");
-					matlabCode.append("dest = octaveToJavaIntArray(data(:," + (singleDestColumn+1) + "));\n\n");
+					for (int i=0; i < numVariables; i++) {
+						matlabCode.append(variableColNumLabels[i].toLowerCase() + " = octaveToJavaIntArray(data(:," +
+								(singleCalcColumns[i]+1) + "));\n");
+					}
+					matlabCode.append("\n");
 				} else {
-					matlabCode.append("source = octaveToJavaDoubleArray(data(:," + (singleSourceColumn+1) + "));\n");
-					matlabCode.append("dest = octaveToJavaDoubleArray(data(:," + (singleDestColumn+1) + "));\n\n");
+					for (int i=0; i < numVariables; i++) {
+						matlabCode.append(variableColNumLabels[i].toLowerCase() + " = octaveToJavaDoubleArray(data(:," +
+								(singleCalcColumns[i]+1) + "));\n");
+					}
+					matlabCode.append("\n");
 				}
 			}
 
@@ -936,59 +969,46 @@ public abstract class AutoAnalyser extends JFrame
 			String javaPrefix = "    ";
 			String pythonPrefix = "";
 			String matlabPrefix = "";
-			if (sourceDestPairs.size() > 1) {
-				// We're doing all pairs
+			StringBuffer extraFormatTerms = new StringBuffer();
+			if (variableCombinations.size() > 1) {
+				// We're doing all combinations.
+				// Set up the loops for these:
+				String[] columnVariables = setUpLoopsForAllCombos(javaCode, pythonCode, matlabCode);
+				
+				// Get the indents right from here on, 
+				//  and prepare a string of the column labels for
+				//  each variable, to be used in later formatting.
+				for (int i = 0; i < numVariables; i++) {
+					javaPrefix += "    ";
+					pythonPrefix += "\t";
+					matlabPrefix += "\t";
+					extraFormatTerms.append(columnVariables[i] + ", ");
+				}
 				
-				// Set up loops in the code:
-				// 1. Java code
-				javaCode.append("    \n");
-				javaCode.append("    // Compute for all pairs:\n");
-				javaCode.append("    for (int s = 0; s < " + dataColumns +
-							"; s++) {\n");
-				javaCode.append("        for (int d = 0; d < " + dataColumns +
-						"; d++) {\n");
-				javaPrefix = "            ";
-				javaCode.append(javaPrefix + "// For each source-dest pair:\n");
-				javaCode.append(javaPrefix + "if (s == d) {\n");
-				javaCode.append(javaPrefix + "    continue;\n");
-				javaCode.append(javaPrefix + "}\n");
-				// 2. Python code
-				pythonCode.append("\n");
-				pythonCode.append("# Compute for all pairs:\n");
-				pythonCode.append("for s in range(" + dataColumns + "):\n");
-				pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
-				pythonPrefix = "\t\t";
-				pythonCode.append(pythonPrefix+ "# For each source-dest pair:\n");
-				pythonCode.append(pythonPrefix + "if (s == d):\n");
-				pythonCode.append(pythonPrefix + "\tcontinue\n");
-				// 3. Matlab code
-				matlabCode.append("\n");
-				matlabCode.append("% Compute for all pairs:\n");
-				matlabCode.append("for s = 1:" + dataColumns + "\n");
-				matlabCode.append("\tfor d = 1:" + dataColumns + "\n");
-				matlabPrefix = "\t\t";
-				matlabCode.append(matlabPrefix + "% For each source-dest pair:\n");
-				matlabCode.append(matlabPrefix + "if (s == d)\n");
-				matlabCode.append(matlabPrefix + "\tcontinue;\n");
-				matlabCode.append(matlabPrefix + "end\n");
-
 				// Read in the data for these columns for all pairs
 				matlabCode.append(matlabPrefix + "% Column indices start from 1 in Matlab:\n");
 				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-					javaCode.append(javaPrefix + "int[] source = MatrixUtils.selectColumn(data, s);\n");
-					javaCode.append(javaPrefix + "int[] dest = MatrixUtils.selectColumn(data, d);\n\n");
-					matlabCode.append(matlabPrefix + "source = octaveToJavaIntArray(data(:, s));\n");
-					matlabCode.append(matlabPrefix + "dest = octaveToJavaIntArray(data(:, d));\n\n");
-					pythonCode.append(pythonPrefix + "source = JArray(JInt, 1)(data[:, s].tolist())\n");
-					pythonCode.append(pythonPrefix + "dest = JArray(JInt, 1)(data[:, d].tolist())\n\n");
+					for (int i = 0; i < numVariables; i++) {
+						javaCode.append(javaPrefix + "int[] " + variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.selectColumn(data, " + columnVariables[i] + ");\n");
+						matlabCode.append(matlabPrefix + variableColNumLabels[i].toLowerCase() +
+								" = octaveToJavaIntArray(data(:, " + columnVariables[i] + "));\n");
+						pythonCode.append(pythonPrefix + variableColNumLabels[i].toLowerCase() +
+								" = JArray(JInt, 1)(data[:, " + columnVariables[i] + "].tolist())\n");						
+					}
 				} else {
-					javaCode.append(javaPrefix + "double[] source = MatrixUtils.selectColumn(data, s);\n");
-					javaCode.append(javaPrefix + "double[] dest = MatrixUtils.selectColumn(data, d);\n\n");
-					matlabCode.append(matlabPrefix + "source = octaveToJavaDoubleArray(data(:, s));\n");
-					matlabCode.append(matlabPrefix + "dest = octaveToJavaDoubleArray(data(:, d));\n\n");
-					pythonCode.append(pythonPrefix + "source = data[:, s]\n");
-					pythonCode.append(pythonPrefix + "dest = data[:, d]\n\n");
+					for (int i = 0; i < numVariables; i++) {
+						javaCode.append(javaPrefix + "double[] " + variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.selectColumn(data, " + columnVariables[i] + ");\n");
+						matlabCode.append(matlabPrefix + variableColNumLabels[i].toLowerCase() +
+								" = octaveToJavaDoubleArray(data(:, " + columnVariables[i] + "));\n");
+						pythonCode.append(pythonPrefix + variableColNumLabels[i].toLowerCase() +
+								" = data[:, " + columnVariables[i] + "]\n");						
+					}
 				}
+				javaCode.append("\n");
+				matlabCode.append("\n");
+				pythonCode.append("\n");
 			} else {
 				// For a single pair, we don't need to set up loops, and
 				// we've already read in the data to source and dest variables above
@@ -1011,15 +1031,23 @@ public abstract class AutoAnalyser extends JFrame
 			} else {
 				setObservationsMethod = "setObservations";
 			}
+			StringBuffer methodArguments = new StringBuffer();
+			for (int i = 0; i < numVariables; i++) {
+				methodArguments.append(variableColNumLabels[i].toLowerCase());
+				if (i < numVariables - 1) {
+					// There are more variables to come:
+					methodArguments.append(", ");
+				}
+			}
 			// 1. Java
 			javaCode.append(javaPrefix + "// " + supplyDataComment);
-			javaCode.append(javaPrefix + "calc." + setObservationsMethod + "(source, dest);\n");
+			javaCode.append(javaPrefix + "calc." + setObservationsMethod + "(" + methodArguments + ");\n");
 			// 2. Python
 			pythonCode.append(pythonPrefix + "# " + supplyDataComment);
-			pythonCode.append(pythonPrefix + "calc." + setObservationsMethod + "(source, dest)\n");
+			pythonCode.append(pythonPrefix + "calc." + setObservationsMethod + "(" + methodArguments + ")\n");
 			// 3. Matlab
 			matlabCode.append(matlabPrefix + "% " + supplyDataComment);
-			matlabCode.append(matlabPrefix + "calc." + setObservationsMethod + "(source, dest);\n");
+			matlabCode.append(matlabPrefix + "calc." + setObservationsMethod + "(" + methodArguments + ");\n");
 			
 			// Compute the result
 			String computeComment = "5. Compute the estimate:\n";
@@ -1030,15 +1058,16 @@ public abstract class AutoAnalyser extends JFrame
 			matlabCode.append(matlabPrefix + "% " + computeComment);
 			matlabCode.append(matlabPrefix + "result = calc.computeAverageLocalOfObservations();\n");
 			
-			String resultsPrefixString, extraFormatTerms;
-			if (sourceDestPairs.size() > 1) {
-				resultsPrefixString = String.format(measureAcronym + "_%s(col_%%d -> col_%%d) = ",
-						selectedCalcType);
-				extraFormatTerms = "s, d, ";
+			String resultsPrefixString;
+			if (variableCombinations.size() > 1) {
+				resultsPrefixString = String.format(measureAcronym + "_%s(%s) = ",
+						selectedCalcType, variableRelationshipFormatString);
 			} else {
-				resultsPrefixString = String.format(measureAcronym + "_%s(col_%d -> col_%d) = ",
-					selectedCalcType, singleSourceColumn, singleDestColumn);
-				extraFormatTerms = "";
+				resultsPrefixString = String.format(measureAcronym + "_%s(%s) = ",
+					selectedCalcType,
+					formatStringWithColumnNumbers(
+							variableRelationshipFormatString,
+							singleCalcColumns));
 			}
 			
 			String resultsSuffixString = "";
@@ -1076,18 +1105,11 @@ public abstract class AutoAnalyser extends JFrame
 					"%.4f " + units + resultsSuffixString + "\\n', ...\n\t" + matlabPrefix +
 					extraFormatTerms + "result" + statSigFormatTerms + ");\n");
 			
-			if (sourceDestPairs.size() > 1) {
-				// We're doing all pairs
+			if (variableCombinations.size() > 1) {
+				// We're doing all combos
 				
 				// Finalise the loops in the code:
-				// 1. Java code
-				javaCode.append("        }\n");
-				javaCode.append("    }\n");
-				// 2. Python code
-				// Nothing to do
-				// 3. Matlab code
-				matlabCode.append("\tend\n");
-				matlabCode.append("end\n");
+				finaliseLoopsForAllCombos(javaCode, pythonCode, matlabCode);
 				
 				// And tell the user to see results in the console:
 				resultsLabel.setText("See console for all pairs calculation results");
@@ -1126,11 +1148,11 @@ public abstract class AutoAnalyser extends JFrame
 				// Now make our own calculations here:
 				// TODO make the calculation in a separate thread, and have a 
 				//  button to cancel the calculation whilst it's running
-				for (int[] pair : sourceDestPairs) {
-					int sourceColumn = pair[0];
-					int destColumn = pair[1];
+				for (int[] columnCombo : variableCombinations) {
 					
-					if (sourceColumn == destColumn) {
+					if ((variableCombinations.size() > 1) && skipColumnCombo(columnCombo)) {
+						System.out.print("Skipping column combo ");
+						MatrixUtils.printArray(System.out, columnCombo);
 						continue;
 					}
 					
@@ -1141,25 +1163,8 @@ public abstract class AutoAnalyser extends JFrame
 						calcContinuous.initialise();
 					}
 					
-					// Set observations
-					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-						calcDiscrete.addObservations(
-								MatrixUtils.selectColumn(dataDiscrete, sourceColumn),
-								MatrixUtils.selectColumn(dataDiscrete, destColumn));
-					} else {
-						// TODO We should be able to directly call setObservations(double[], double[])
-						//  here but we can't right now because ChannelCalculator does not
-						//  define this method. It would be complicated to fix this
-						//  because it involves Conditional MI calculator it seems.
-						// Revisit this later -- for now fix by deferring to child classes
-						// calcContinuous.setObservations(
-						//		MatrixUtils.selectColumn(data, sourceColumn),
-						//		MatrixUtils.selectColumn(data, destColumn));
-						setObservations(calcContinuous,
-								MatrixUtils.selectColumn(data, sourceColumn),
-								MatrixUtils.selectColumn(data, destColumn));
-					}
-					
+					setObservations(calcDiscrete, calcContinuous, columnCombo);
+										
 					// Compute the result:
 					double result;
 					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
@@ -1189,11 +1194,15 @@ public abstract class AutoAnalyser extends JFrame
 						} else {
 							// permutation test of statistical significance:
 							EmpiricalMeasurementDistribution measDist;
+							EmpiricalNullDistributionComputer empiricalNullDistCalc;
 							if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-								measDist = calcDiscrete.computeSignificance(numPermutationsToCheck);
+								empiricalNullDistCalc =
+										(EmpiricalNullDistributionComputer) calcDiscrete;
 							} else {
-								measDist = calcContinuous.computeSignificance(numPermutationsToCheck);
+								empiricalNullDistCalc =
+										(EmpiricalNullDistributionComputer) calcContinuous;
 							}
+							measDist = empiricalNullDistCalc.computeSignificance(numPermutationsToCheck);
 							resultsSuffixStringFormatted = String.format(
 									resultsSuffixString,
 									measDist.getMeanOfDistribution(),
@@ -1204,11 +1213,16 @@ public abstract class AutoAnalyser extends JFrame
 					}
 					
 					String resultsText;
-					if (sourceDestPairs.size() > 1) {
+					if (variableCombinations.size() > 1) {
 						// We're doing all pairs
-						resultsText = String.format(
-								resultsPrefixString + "%.4f %s" + resultsSuffixStringFormatted,
-								sourceColumn, destColumn, result, units);
+						// OLD:
+						// resultsText = String.format(
+						//		resultsPrefixString + "%.4f %s" + resultsSuffixStringFormatted,
+						//		sourceColumn, destColumn, result, units);
+						// NEW:
+						resultsText = String.format(formatStringWithColumnNumbers(
+								resultsPrefixString + "%%.4f %%s" + resultsSuffixStringFormatted,
+								columnCombo), result, units);
 					} else {
 						resultsText = String.format(
 								resultsPrefixString + "%.4f %s" + resultsSuffixStringFormatted,
@@ -1219,7 +1233,7 @@ public abstract class AutoAnalyser extends JFrame
 					System.out.println(resultsText);
 				}
 	
-				if ((sourceDestPairs.size() == 1) && 
+				if ((variableCombinations.size() == 1) && 
 						!selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
 					// Read the current property values back out (in case of 
 					//  automated parameter assignment)
@@ -1249,11 +1263,64 @@ public abstract class AutoAnalyser extends JFrame
 
 	}
 	
+	/**
+	 * Method to allow child to fill out what all of the variable combinations would be that 
+	 *  would be evaluated by this measure
+	 *  
+	 * @param variableCombinations
+	 */
+	protected abstract void fillOutAllCombinations(Vector variableCombinations);
+	
+	/**
+	 * Method to allow child classes to set up the loops over all combinations of
+	 *  columns
+	 * 
+	 * @param javaCode
+	 * @param pythonCode
+	 * @param matlabCode
+	 * @return A string array with the names used for the variables iterating over 
+	 *   each column
+	 */
+	protected abstract String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode);
+
+	/**
+	 * Method to allow child classes to finalise the loops over all combinations of
+	 *  columns
+	 * 
+	 * @param javaCode
+	 * @param pythonCode
+	 * @param matlabCode
+	 */
+	protected abstract void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode);
+	
+	/**
+	 * Method to allow child classes to format a string with 
+	 *  an array of ints for the column numbers (since the number
+	 *  of these is variable, it's easier if the child classes do this) 
+	 * 
+	 * @param columnNumbers array of column numbers used to identify the variables.
+	 * @return
+	 */
+	protected abstract String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers);
+
+	/**
+	 * Check whether this measure type should skip this particular
+	 *  combination of columns or not
+	 *  (e.g. for a pairwise measurement we do want to skip
+	 *  where the two columns are equal). 
+	 * 
+	 * @param columnCombo
+	 * @return
+	 */
+	protected abstract boolean skipColumnCombo(int[] columnCombo);
+
 	/**
 	 * Method to assign and initialise our continuous calculator class
 	 * @throws Exception 
 	 */
-	protected abstract ChannelCalculatorCommon assignCalcObjectContinuous(String selectedCalcType) throws Exception;
+	protected abstract InfoMeasureCalculatorContinuous assignCalcObjectContinuous(String selectedCalcType) throws Exception;
 	
 	/**
 	 * Method to assign and initialise our discrete calculator class
@@ -1269,11 +1336,11 @@ public abstract class AutoAnalyser extends JFrame
 	 *
 	 */
 	protected class DiscreteCalcAndArguments {
-		ChannelCalculatorDiscrete calc;
+		InfoMeasureCalculatorDiscrete calc;
 		int base;
 		String arguments;
 		
-		DiscreteCalcAndArguments(ChannelCalculatorDiscrete calc, int base,
+		DiscreteCalcAndArguments(InfoMeasureCalculatorDiscrete calc, int base,
 				String arguments) {
 			this.calc = calc;
 			this.base = base;
@@ -1281,16 +1348,17 @@ public abstract class AutoAnalyser extends JFrame
 		}
 	}
 	
-	/** 
+	/**
 	 * Method to set the observations on the underlying calculator
 	 * 
-	 * @param calc
-	 * @param source
-	 * @param dest
+	 * @param calcDiscrete
+	 * @param calcContinuous
+	 * @param columnCombo int array for the columns of data being computed with.
 	 * @throws Exception
 	 */
-	protected abstract void setObservations(ChannelCalculatorCommon calc,
-			double[] source, double[] dest) throws Exception;
+	protected abstract void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception;
 
 
 	/**
@@ -1417,45 +1485,23 @@ public abstract class AutoAnalyser extends JFrame
 		System.out.println("Getting calc properties");
 		String selectedCalcType = (String)
 				calcTypeComboBox.getSelectedItem();
-		String[] classSpecificPropertyNames = null;
-		String[] classSpecificPropertiesFieldNames = null;
-		String[] classSpecificPropertyDescriptions = null;
-		ChannelCalculatorCommon calc = null;
+		CalcProperties calcProperties = null;
 		try {
-			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-				calcClass = discreteClass;
-				calc = null; // Not used
-				classSpecificPropertyNames = discreteProperties;
-				classSpecificPropertiesFieldNames = null; // Not used
-				classSpecificPropertyDescriptions = discretePropertyDescriptions;
-			} else {
-				calc = assignCalcObjectContinuous(selectedCalcType);
-				calcClass = calc.getClass();
-				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
-					classSpecificPropertyNames = gaussianProperties;
-					classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
-					classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
-				} else if (selectedCalcType.startsWith(CALC_TYPE_KRASKOV)) {
-					// The if statement will work for both MI Kraskov calculators
-					classSpecificPropertyNames = kraskovProperties;
-					classSpecificPropertiesFieldNames = kraskovPropertiesFieldNames;
-					classSpecificPropertyDescriptions = kraskovPropertyDescriptions;
-				} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
-					classSpecificPropertyNames = kernelProperties;
-					classSpecificPropertiesFieldNames = kernelPropertiesFieldNames;
-					classSpecificPropertyDescriptions = kernelPropertyDescriptions;
-				} else {
-					calcClass = null;
-					throw new Exception("No recognised calculator selected: " +
-							selectedCalcType);
-				}
-			}
+			calcProperties = assignCalcProperties(selectedCalcType);
 		} catch (Exception ex) {
 			ex.printStackTrace(System.err);
 			JOptionPane.showMessageDialog(this,
 					ex.getMessage());
 			resultsLabel.setText("Cannot load requested calculator");
 		}
+		String[] classSpecificPropertyNames =
+					calcProperties.classSpecificPropertyNames;
+		String[] classSpecificPropertiesFieldNames =
+					calcProperties.classSpecificPropertiesFieldNames;
+		String[] classSpecificPropertyDescriptions =
+					calcProperties.classSpecificPropertyDescriptions;
+		calcClass = calcProperties.calcClass;
+		Object calc = calcProperties.calc;
 		
 		// Now get all of the possible properties for this class:
 		propertyNames = new Vector();
@@ -1513,20 +1559,65 @@ public abstract class AutoAnalyser extends JFrame
 			// Now extract the default values for all of these properties:
 			propertyValues = new HashMap();
 			for (String propName : propertyNames) {
-				String defaultPropValue = null;
 				try {
-					defaultPropValue = calc.getProperty(propName);
+					InfoMeasureCalculatorContinuous calcCont =
+							(InfoMeasureCalculatorContinuous) calc;
+					String propValue = calcCont.getProperty(propName);
+					propertyValues.put(propName, propValue);
 				} catch (Exception ex) {
 					ex.printStackTrace(System.err);
 					JOptionPane.showMessageDialog(this,
 							ex.getMessage());
 					propertyValues.put(propName, "Cannot find a value");
 				}
-				propertyValues.put(propName, defaultPropValue);
 			}
 		}
 	}
 
+	/**
+	 * Structure to return calculator properties
+	 * 
+	 * @author Joseph Lizier
+	 *
+	 */
+	protected class CalcProperties {
+		// An default instantiation of the required calculator object
+		//  for continuous data (this is not used for discrete)
+		InfoMeasureCalculatorContinuous calc;
+		// Class of the required calculator
+		@SuppressWarnings("rawtypes")
+		Class calcClass;
+		String[] classSpecificPropertyNames;
+		String[] classSpecificPropertiesFieldNames;
+		String[] classSpecificPropertyDescriptions;
+	}
+	
+	/**
+	 * Assign the property details for the calculator in use.
+	 * Should be overridden by child classes, though they
+	 *  can refer here to handle discrete calculators. 
+	 * 
+	 * @param selectedCalcType String name of the calculator
+	 * @return
+	 * @throws Exception
+	 */
+	protected CalcProperties assignCalcProperties(String selectedCalcType) 
+		throws Exception {
+		
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			CalcProperties calcProperties = new CalcProperties();
+			calcProperties.calcClass = discreteClass;
+			calcProperties.calc = null; // Not used
+			calcProperties.classSpecificPropertyNames = discreteProperties;
+			calcProperties.classSpecificPropertiesFieldNames = null; // Not used
+			calcProperties.classSpecificPropertyDescriptions = discretePropertyDescriptions;
+			return calcProperties;
+		} else {
+			// We don't handle any other calculators here
+			return null;
+		}
+	}
+
 	@Override
 	public void changedUpdate(DocumentEvent e) {
 		// Source or dest col number updated
@@ -1594,21 +1685,16 @@ public abstract class AutoAnalyser extends JFrame
 		//  so we'll just act on potential changes from both.
 		// This won't cause a problem here
 
-		if (e.getSource() == allPairsCheckBox) {
+		if (e.getSource() == allCombosCheckBox) {
 			// "All pairs" checkbox -- update in case changed:
-			if (allPairsCheckBox.isSelected()) {
-				sourceColTextField.setEnabled(false);
-				destColTextField.setEnabled(false);
+			if (allCombosCheckBox.isSelected()) {
+				for (int i = 0; i < numVariables; i++) {
+					variableColTextFields[i].setEnabled(false);
+				}
 			} else {
-				sourceColTextField.setEnabled(true);
-				destColTextField.setEnabled(true);
-			}
-		} else if (e.getSource() == computeResultCheckBox) {
-			// Compute checkbox -- update in case changed:
-			if (computeResultCheckBox.isSelected()) {
-				computeButton.setText(buttonTextCodeAndCompute);
-			} else {
-				computeButton.setText(buttonTextCodeOnly);
+				for (int i = 0; i < numVariables; i++) {
+					variableColTextFields[i].setEnabled(true);
+				}
 			}
 		} else if (e.getSource() == statSigCheckBox) {
 			System.out.println("StatSigCheckBox state changed to " + statSigCheckBox.isSelected());
@@ -1626,9 +1712,16 @@ public abstract class AutoAnalyser extends JFrame
 				statSigAnalyticCheckBox.setSelected(false);
 				statSigAnalyticCheckBox.setEnabled(false);
 			}
+		} else if (e.getSource() == computeResultCheckBox) {
+			// Compute checkbox -- update in case changed:
+			if (computeResultCheckBox.isSelected()) {
+				computeButton.setText(buttonTextCodeAndCompute);
+			} else {
+				computeButton.setText(buttonTextCodeOnly);
+			}
 		}
 	}
-	
+
 	/**
 	 * Checks whether the current calculator class implements the 
 	 * AnalyticNullDistributionComputer interface
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
new file mode 100644
index 0000000..17632da
--- /dev/null
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
@@ -0,0 +1,207 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2015, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.demos.autoanalysis;
+
+import infodynamics.measures.continuous.ChannelCalculator;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.discrete.ChannelCalculatorDiscrete;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+
+/**
+ * This abstract class provides a GUI to build a simple calculation
+ *  over a source-target channel,
+ *  and supply the code to execute it.
+ * Child classes fix this to a TE or MI calculation.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public abstract class AutoAnalyserChannelCalculator extends AutoAnalyser {
+
+	/**
+	 * Need serialVersionUID to be serializable
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	// Property names for specific continuous calculators:
+	protected String[] gaussianProperties;
+	protected String[] gaussianPropertiesFieldNames;
+	protected String[] gaussianPropertyDescriptions;
+	protected String[] kernelProperties;
+	protected String[] kernelPropertiesFieldNames;
+	protected String[] kernelPropertyDescriptions;
+	protected String[] kraskovProperties;
+	protected String[] kraskovPropertiesFieldNames;
+	protected String[] kraskovPropertyDescriptions;
+
+	/**
+	 * Constructor to initialise the GUI for a channel calculator
+	 */
+	protected void makeSpecificInitialisations() {
+		numVariables = 2;
+		variableColNumLabels = new String[] {"Source", "Destination"};
+		useAllCombosCheckBox = true;
+		useStatSigCheckBox = true;
+		wordForCombinations = "pairs";
+		variableRelationshipFormatString = "col_%d -> col_%d";
+	}
+
+	@Override
+	protected void fillOutAllCombinations(Vector variableCombinations) {
+		// All combinations here means all pairs
+		for (int s = 0; s < dataColumns; s++) {
+			for (int d = 0; d < dataColumns; d++) {
+				variableCombinations.add(new int[] {s, d});
+			}
+		}
+	}
+
+	@Override
+	protected String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		// Set up loops in the code:
+		// 1. Java code
+		javaCode.append("    \n");
+		javaCode.append("    // Compute for all pairs:\n");
+		javaCode.append("    for (int s = 0; s < " + dataColumns +
+					"; s++) {\n");
+		javaCode.append("        for (int d = 0; d < " + dataColumns +
+				"; d++) {\n");
+		String javaPrefix = "            ";
+		javaCode.append(javaPrefix + "// For each source-dest pair:\n");
+		javaCode.append(javaPrefix + "if (s == d) {\n");
+		javaCode.append(javaPrefix + "    continue;\n");
+		javaCode.append(javaPrefix + "}\n");
+		// 2. Python code
+		pythonCode.append("\n");
+		pythonCode.append("# Compute for all pairs:\n");
+		pythonCode.append("for s in range(" + dataColumns + "):\n");
+		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "\t\t";
+		pythonCode.append(pythonPrefix+ "# For each source-dest pair:\n");
+		pythonCode.append(pythonPrefix + "if (s == d):\n");
+		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		// 3. Matlab code
+		matlabCode.append("\n");
+		matlabCode.append("% Compute for all pairs:\n");
+		matlabCode.append("for s = 1:" + dataColumns + "\n");
+		matlabCode.append("\tfor d = 1:" + dataColumns + "\n");
+		String matlabPrefix = "\t\t";
+		matlabCode.append(matlabPrefix + "% For each source-dest pair:\n");
+		matlabCode.append(matlabPrefix + "if (s == d)\n");
+		matlabCode.append(matlabPrefix + "\tcontinue;\n");
+		matlabCode.append(matlabPrefix + "end\n");
+		
+		// Return the variables to index each column:
+		return new String[] {"s", "d"}; 
+	}
+
+	@Override
+	protected void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		
+		// 1. Java code
+		javaCode.append("        }\n");
+		javaCode.append("    }\n");
+		// 2. Python code
+		// Nothing to do
+		// 3. Matlab code
+		matlabCode.append("\tend\n");
+		matlabCode.append("end\n");
+	}
+	
+	@Override
+	protected String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers) {
+		// We format the source and target variable numbers into the
+		//  return string here:
+		return String.format(formatStr,
+				columnNumbers[0], columnNumbers[1]);
+	}
+	
+	@Override
+	protected boolean skipColumnCombo(int[] columnCombo) {
+		if (columnCombo[0] == columnCombo[1]) {
+			// source and destination columns are the same here,
+			//  so don't compute the channel measure
+			return true;
+		}
+		return false;
+	}
+	
+	@Override
+	protected void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception {
+		
+		String selectedCalcType = (String)
+			calcTypeComboBox.getSelectedItem();
+		
+		int sourceColumn = columnCombo[0];
+		int destColumn = columnCombo[1];
+		
+		// Set observations
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			ChannelCalculatorDiscrete cc = (ChannelCalculatorDiscrete) calcDiscrete; 
+			cc.addObservations(
+					MatrixUtils.selectColumn(dataDiscrete, sourceColumn),
+					MatrixUtils.selectColumn(dataDiscrete, destColumn));
+		} else {
+			ChannelCalculator cc = (ChannelCalculator) calcContinuous;
+			cc.setObservations(
+					MatrixUtils.selectColumn(data, sourceColumn),
+					MatrixUtils.selectColumn(data, destColumn));
+		}
+	}
+	
+	protected CalcProperties assignCalcProperties(String selectedCalcType)
+			throws Exception {
+		// Let the super class handle discrete calculators
+		CalcProperties calcProperties = super.assignCalcProperties(selectedCalcType);
+		if (calcProperties == null) {
+			// We need to assign properties for a continuous calculator
+			calcProperties = new CalcProperties();
+			calcProperties.calc = assignCalcObjectContinuous(selectedCalcType);
+			calcProperties.calcClass = calcProperties.calc.getClass();
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+				calcProperties.classSpecificPropertyNames = gaussianProperties;
+				calcProperties.classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
+			} else if (selectedCalcType.startsWith(CALC_TYPE_KRASKOV)) {
+				// The if statement will work for both MI Kraskov calculators
+				calcProperties.classSpecificPropertyNames = kraskovProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kraskovPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kraskovPropertyDescriptions;
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
+				calcProperties.classSpecificPropertyNames = kernelProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kernelPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kernelPropertyDescriptions;
+			} else {
+				calcProperties = null;
+				throw new Exception("No recognised calculator selected: " +
+						selectedCalcType);
+			}
+		}
+		return calcProperties;
+	}
+
+}
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
index 5a2b39e..13afaeb 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
@@ -18,7 +18,6 @@
 
 package infodynamics.demos.autoanalysis;
 
-import infodynamics.measures.continuous.ChannelCalculatorCommon;
 import infodynamics.measures.continuous.MutualInfoCalculatorMultiVariate;
 import infodynamics.measures.continuous.gaussian.MutualInfoCalculatorMultiVariateGaussian;
 import infodynamics.measures.continuous.kernel.MutualInfoCalculatorMultiVariateKernel;
@@ -41,7 +40,7 @@ import java.awt.event.MouseListener;
  * @author Joseph Lizier
  *
  */
-public class AutoAnalyserMI extends AutoAnalyser
+public class AutoAnalyserMI extends AutoAnalyserChannelCalculator
 	implements ActionListener, DocumentListener, MouseListener {
 
 	/**
@@ -59,6 +58,8 @@ public class AutoAnalyserMI extends AutoAnalyser
 	 */
 	protected void makeSpecificInitialisations() {
 		
+		super.makeSpecificInitialisations();
+		
 		// Set up the properties for MI:
 		measureAcronym = "MI";
 		appletTitle = "JIDT Mutual Information Auto-Analyser"; 
@@ -159,7 +160,8 @@ public class AutoAnalyserMI extends AutoAnalyser
 	/**
 	 * Method to assign and initialise our continuous calculator class
 	 */
-	protected ChannelCalculatorCommon assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+	@Override
+	protected MutualInfoCalculatorMultiVariate assignCalcObjectContinuous(String selectedCalcType) throws Exception {
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
 			return new MutualInfoCalculatorMultiVariateGaussian();
 		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV_ALG1)) {
@@ -205,13 +207,6 @@ public class AutoAnalyserMI extends AutoAnalyser
 				base + ", " + timeDiff);
 	}
 
-	protected void setObservations(ChannelCalculatorCommon calc,
-			double[] source, double[] dest) throws Exception {
-		// We know this is a MutualInfoCalculatorMultiVariate
-		MutualInfoCalculatorMultiVariate miCalc = (MutualInfoCalculatorMultiVariate) calc;
-		miCalc.setObservations(source, dest);
-	}
-
 	/**
 	 * @param args
 	 */
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
index 75a6ace..29ee653 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
@@ -18,7 +18,6 @@
 
 package infodynamics.demos.autoanalysis;
 
-import infodynamics.measures.continuous.ChannelCalculatorCommon;
 import infodynamics.measures.continuous.ConditionalMutualInfoMultiVariateCommon;
 import infodynamics.measures.continuous.TransferEntropyCalculator;
 import infodynamics.measures.continuous.gaussian.TransferEntropyCalculatorGaussian;
@@ -41,7 +40,7 @@ import java.awt.event.MouseListener;
  * @author Joseph Lizier
  *
  */
-public class AutoAnalyserTE extends AutoAnalyser
+public class AutoAnalyserTE extends AutoAnalyserChannelCalculator
 	implements ActionListener, DocumentListener, MouseListener {
 
 	/**
@@ -60,10 +59,17 @@ public class AutoAnalyserTE extends AutoAnalyser
 	 */
 	protected void makeSpecificInitialisations() {
 		
+		super.makeSpecificInitialisations();
+		
 		// Set up the properties for TE:
 		measureAcronym = "TE";
 		appletTitle = "JIDT Transfer Entropy Auto-Analyser"; 
 		
+		calcTypes = new String[] {
+				CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_KRASKOV, CALC_TYPE_KERNEL};
+		unitsForEachCalc = new String[] {"bits", "nats", "nats", "bits"};
+		
 		// Discrete:
 		discreteClass = TransferEntropyCalculatorDiscrete.class;
 		discreteProperties = new String[] {
@@ -214,7 +220,7 @@ public class AutoAnalyserTE extends AutoAnalyser
 	/**
 	 * Method to assign and initialise our continuous calculator class
 	 */
-	protected ChannelCalculatorCommon assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+	protected TransferEntropyCalculator assignCalcObjectContinuous(String selectedCalcType) throws Exception {
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
 			return new TransferEntropyCalculatorGaussian();
 		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV)) {
@@ -294,13 +300,6 @@ public class AutoAnalyserTE extends AutoAnalyser
 				base + ", " + k + ", " + k_tau + ", " + l + ", " + l_tau + ", " + delay);
 	}
 
-	protected void setObservations(ChannelCalculatorCommon calc,
-			double[] source, double[] dest) throws Exception {
-		// We know this is a TransferEntropyCalculator
-		TransferEntropyCalculator teCalc = (TransferEntropyCalculator) calc;
-		teCalc.setObservations(source, dest);
-	}
-
 	/**
 	 * @param args
 	 */

From 4d6fc1a57226d4370666d298ac1ae78a158799b6 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 21 Aug 2017 14:56:20 +1000
Subject: [PATCH 44/78] Updated Kozachenko-Leonenko multivariate Entropy
 calculator to implement EntropyCalculator interface, in preparation for
 Entropy AutoAnalyser

---
 ...ntropyCalculatorMultiVariateKozachenko.java | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
index edef0a1..9007c8b 100755
--- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
+++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
@@ -18,9 +18,11 @@
 
 package infodynamics.measures.continuous.kozachenko;
 
+import infodynamics.measures.continuous.EntropyCalculator;
 import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
 import infodynamics.utils.EuclideanUtils;
 import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
 
 /**
  * 
      Computes the differential entropy of a given set of observations
@@ -56,7 +58,7 @@ import infodynamics.utils.MathsUtils;
  * www)
  */
 public class EntropyCalculatorMultiVariateKozachenko  
-	implements EntropyCalculatorMultiVariate {
+	implements EntropyCalculator, EntropyCalculatorMultiVariate {
 
 	protected boolean debug = false;
 	private int totalObservations;
@@ -124,6 +126,19 @@ public class EntropyCalculatorMultiVariateKozachenko
 		lastLocalEntropy = null;
 	}
 
+	/*
+	 * (non-Javadoc)
+	 * @see infodynamics.measures.continuous.EntropyCalculator#setObservations(double[])
+	 * 
+	 * This method here to ensure we make compatibility with the 
+	 *  EntropyCalculator interface.
+	 */
+	@Override
+	public void setObservations(double[] observations) {
+		rawData = MatrixUtils.reshape(observations, observations.length, 1);
+		setObservations(rawData);
+	}
+	
 	/**
 	 * Each row of the data is an observation; each column of
 	 *  the row is a new variable in the multivariate observation.
@@ -329,5 +344,4 @@ public class EntropyCalculatorMultiVariateKozachenko
 	public int getNumObservations() {
 		return totalObservations;
 	}
-
 }

From e71df111cf39ffe2cc4ef2576a0c80edec406991 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 21 Aug 2017 21:08:31 +1000
Subject: [PATCH 45/78] Adding AutoAnalyserEntropy

---
 .../autoanalysis/AutoAnalyserEntropy.java     | 299 ++++++++++++++++++
 1 file changed, 299 insertions(+)
 create mode 100644 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
new file mode 100644
index 0000000..3756082
--- /dev/null
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
@@ -0,0 +1,299 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2015, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.demos.autoanalysis;
+
+import infodynamics.measures.continuous.EntropyCalculator;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.continuous.gaussian.EntropyCalculatorGaussian;
+import infodynamics.measures.continuous.kernel.EntropyCalculatorKernel;
+import infodynamics.measures.continuous.kozachenko.EntropyCalculatorMultiVariateKozachenko;
+import infodynamics.measures.discrete.EntropyCalculatorDiscrete;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
+import infodynamics.measures.discrete.MutualInformationCalculatorDiscrete;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+
+import javax.swing.JOptionPane;
+
+/**
+ * This class provides a GUI to build a simple calculation
+ *  of entropy,
+ *  and supply the code to execute it.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class AutoAnalyserEntropy extends AutoAnalyser {
+
+	/**
+	 * Need serialVersionUID to be serializable
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	// Property names for specific continuous calculators:
+	protected String[] gaussianProperties;
+	protected String[] gaussianPropertiesFieldNames;
+	protected String[] gaussianPropertyDescriptions;
+	protected String[] kernelProperties;
+	protected String[] kernelPropertiesFieldNames;
+	protected String[] kernelPropertyDescriptions;
+	protected String[] klProperties;
+	protected String[] klPropertiesFieldNames;
+	protected String[] klPropertyDescriptions;
+
+	/**
+	 * Constructor to initialise the GUI for a channel calculator
+	 */
+	protected void makeSpecificInitialisations() {
+		numVariables = 1;
+		variableColNumLabels = new String[] {"Variable"};
+		useAllCombosCheckBox = true;
+		useStatSigCheckBox = false;
+		wordForCombinations = "variables";
+		variableRelationshipFormatString = "col_%d";
+
+		// Set up the properties for Entropy:
+		measureAcronym = "H";
+		appletTitle = "JIDT Entropy Auto-Analyser"; 
+		
+		calcTypes = new String[] {
+				CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_KOZ_LEO, CALC_TYPE_KERNEL};
+		unitsForEachCalc = new String[] {"bits", "nats", "nats", "bits"};
+		
+		// Discrete:
+		discreteClass = MutualInformationCalculatorDiscrete.class;
+		discreteProperties = new String[] {
+				DISCRETE_PROPNAME_BASE
+		};
+		discretePropertyDefaultValues = new String[] {
+				"2"
+		};
+		discretePropertyDescriptions = new String[] {
+				"Number of discrete states available for each variable (i.e. 2 for binary)"
+		};
+		
+		// Continuous:
+		abstractContinuousClass = EntropyCalculator.class;
+		// Common properties for all continuous calcs:
+		commonContPropertyNames = new String[] {
+				// None
+		};
+		commonContPropertiesFieldNames = new String[] {
+				// None
+		};
+		commonContPropertyDescriptions = new String[] {
+				// None
+		};
+		// Gaussian properties:
+		gaussianProperties = new String[] {
+		};
+		gaussianPropertiesFieldNames = new String[] {
+		};
+		gaussianPropertyDescriptions = new String[] {
+		};
+		// Kernel:
+		kernelProperties = new String[] {
+				EntropyCalculatorKernel.KERNEL_WIDTH_PROP_NAME,
+				EntropyCalculatorKernel.NORMALISE_PROP_NAME,			
+		};
+		kernelPropertiesFieldNames = new String[] {
+				"KERNEL_WIDTH_PROP_NAME",
+				"NORMALISE_PROP_NAME"			
+		};
+		kernelPropertyDescriptions = new String[] {
+				"Kernel width to be used in the calculation. 
      If the property " +
+						EntropyCalculatorKernel.NORMALISE_PROP_NAME +
+						" is set, then this is a number of standard deviations; " +
+						"otherwise it is an absolute value.",
+				"(boolean) whether to normalise 
      the incoming time-series to mean 0, standard deviation 1, or not  (recommended)",
+		};
+		// KSG (Kraskov):
+		klProperties = new String[] {
+				// There is a NUM_DIMENSIONS_PROP_NAME property, but this will
+				//  only be relevant if we go to multivariate later.
+		};
+		klPropertiesFieldNames = new String[] {
+		};
+		klPropertyDescriptions = new String[] {
+		};
+	}
+
+	@Override
+	protected void fillOutAllCombinations(Vector variableCombinations) {
+		// All combinations here means all pairs
+		for (int s = 0; s < dataColumns; s++) {
+			variableCombinations.add(new int[] {s});
+		}
+	}
+
+	@Override
+	protected String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		// Set up loops in the code:
+		// 1. Java code
+		javaCode.append("    \n");
+		javaCode.append("    // Compute for all variables:\n");
+		javaCode.append("    for (int v = 0; v < " + dataColumns +
+					"; v++) {\n");
+		String javaPrefix = "        ";
+		javaCode.append(javaPrefix + "// For each variable:\n");
+		// 2. Python code
+		pythonCode.append("\n");
+		pythonCode.append("# Compute for all variables:\n");
+		pythonCode.append("for v in range(" + dataColumns + "):\n");
+		String pythonPrefix = "\t";
+		pythonCode.append(pythonPrefix+ "# For each variable:\n");
+		// 3. Matlab code
+		matlabCode.append("\n");
+		matlabCode.append("% Compute for all variables:\n");
+		matlabCode.append("for v = 1:" + dataColumns + "\n");
+		String matlabPrefix = "\t";
+		matlabCode.append(matlabPrefix + "% For each variable:\n");
+		
+		// Return the variables to index each column:
+		return new String[] {"v"}; 
+	}
+
+	@Override
+	protected void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		
+		// 1. Java code
+		javaCode.append("    }\n");
+		// 2. Python code
+		// Nothing to do
+		// 3. Matlab code
+		matlabCode.append("end\n");
+	}
+	
+	@Override
+	protected String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers) {
+		// We format the variable number into the
+		//  return string here:
+		return String.format(formatStr,
+				columnNumbers[0]);
+	}
+	
+	@Override
+	protected boolean skipColumnCombo(int[] columnCombo) {
+		// No reason to skip any columns here
+		return false;
+	}
+	
+	@Override
+	protected void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception {
+		
+		String selectedCalcType = (String)
+			calcTypeComboBox.getSelectedItem();
+		
+		int variableColumn = columnCombo[0];
+		
+		// Set observations
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			EntropyCalculatorDiscrete calc = (EntropyCalculatorDiscrete) calcDiscrete; 
+			calc.addObservations(
+					MatrixUtils.selectColumn(dataDiscrete, variableColumn));
+		} else {
+			EntropyCalculator calc = (EntropyCalculator) calcContinuous;
+			calc.setObservations(
+					MatrixUtils.selectColumn(data, variableColumn));
+		}
+	}
+	
+	protected CalcProperties assignCalcProperties(String selectedCalcType)
+			throws Exception {
+		// Let the super class handle discrete calculators
+		CalcProperties calcProperties = super.assignCalcProperties(selectedCalcType);
+		if (calcProperties == null) {
+			// We need to assign properties for a continuous calculator
+			calcProperties = new CalcProperties();
+			calcProperties.calc = assignCalcObjectContinuous(selectedCalcType);
+			calcProperties.calcClass = calcProperties.calc.getClass();
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+				calcProperties.classSpecificPropertyNames = gaussianProperties;
+				calcProperties.classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KOZ_LEO)) {
+				calcProperties.classSpecificPropertyNames = klProperties;
+				calcProperties.classSpecificPropertiesFieldNames = klPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = klPropertyDescriptions;
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
+				calcProperties.classSpecificPropertyNames = kernelProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kernelPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kernelPropertyDescriptions;
+			} else {
+				calcProperties = null;
+				throw new Exception("No recognised calculator selected: " +
+						selectedCalcType);
+			}
+		}
+		return calcProperties;
+	}
+
+	/**
+	 * Method to assign and initialise our continuous calculator class
+	 */
+	@Override
+	protected EntropyCalculator assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+			return new EntropyCalculatorGaussian();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KOZ_LEO)) {
+			return new EntropyCalculatorMultiVariateKozachenko();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
+			return new EntropyCalculatorKernel();
+		} else {
+			throw new Exception("No recognised continuous calculator selected: " +
+					selectedCalcType);
+		}
+
+	}
+
+	/**
+	 * Method to assign and initialise our discrete calculator class
+	 */
+	protected DiscreteCalcAndArguments assignCalcObjectDiscrete() throws Exception {
+		String basePropValueStr;
+		try {
+			basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot find a value for property " + DISCRETE_PROPNAME_BASE);
+			return null;
+		}
+		int base = Integer.parseInt(basePropValueStr);
+		
+		return new DiscreteCalcAndArguments(
+				new EntropyCalculatorDiscrete(base),
+				base,
+				Integer.toString(base));
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		new AutoAnalyserEntropy();
+	}
+}

From 24c66cc46040bcfbeee6171b158c4b9fce2e2541 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 13:08:53 +1000
Subject: [PATCH 46/78] Make sure the auto generated Java code compiles before
 it is run

---
 demos/AutoAnalyser/runAutoGenerated.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/demos/AutoAnalyser/runAutoGenerated.sh b/demos/AutoAnalyser/runAutoGenerated.sh
index 2908464..30bcdb1 100755
--- a/demos/AutoAnalyser/runAutoGenerated.sh
+++ b/demos/AutoAnalyser/runAutoGenerated.sh
@@ -3,6 +3,8 @@
 # Make sure the latest example source file is compiled.
 javac -classpath "../java:../../infodynamics.jar" "../java/infodynamics/demos/autoanalysis/GeneratedCalculator.java"
 
-# Run the example:
-java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.GeneratedCalculator
+if [ $? == 0 ]; then
+	# Run the example:
+	java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.GeneratedCalculator
+fi
 

From 26b15ccf2d4631496bcec398d378a589836fbc49 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 13:37:52 +1000
Subject: [PATCH 47/78] Added binned calculator type to AutoAnalyser; including
 it for all current AutoAnalyser classes. Only includes equal-bin size binning
 right now, we can incorporate max entropy binning later. Also fixed error
 checking on the base property.

---
 .../demos/autoanalysis/AutoAnalyser.java      | 115 +++++++++++++-----
 .../AutoAnalyserChannelCalculator.java        |  11 +-
 .../autoanalysis/AutoAnalyserEntropy.java     |  27 +++-
 .../demos/autoanalysis/AutoAnalyserMI.java    |  14 +--
 .../demos/autoanalysis/AutoAnalyserTE.java    |   4 +-
 5 files changed, 128 insertions(+), 43 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 54edbcf..44ea6d5 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -90,6 +90,7 @@ public abstract class AutoAnalyser extends JFrame
 	
 	// Set options for the Calculator type:
 	public final static String CALC_TYPE_DISCRETE = "Discrete";
+	public final static String CALC_TYPE_BINNED = "Binned";
 	// Child classes must define options for other calculator types
 	//  and initialise the calcTypes array and unitsForEachCalc array
 	protected String[] calcTypes;
@@ -277,7 +278,7 @@ public abstract class AutoAnalyser extends JFrame
 		calcTypeLabel.setToolTipText("Select estimator type. \"Discrete\" is for discrete or pre-binned data; all others for continuous data.");
 		calcTypeLabel.setBorder(BorderFactory.createEmptyBorder(0,0,10,0));
 		calcTypeComboBox = (JComboBox) new JComboBox(calcTypes);
-		calcTypeComboBox.setSelectedIndex(2); // Select 2nd continuous estimator by default
+		calcTypeComboBox.setSelectedIndex(3); // Select 2nd continuous estimator by default
 		calcTypeComboBox.addActionListener(this);
 		// Don't set an empty border as it becomes clickable as well,
 		//  we'll use an empty label instead
@@ -714,7 +715,8 @@ public abstract class AutoAnalyser extends JFrame
 			javaCode.append("import infodynamics.utils.EmpiricalMeasurementDistribution;\n");
 		}
 		javaCode.append("import infodynamics.utils.MatrixUtils;\n\n");
-		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+				selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 			// Cover all children:
 			javaCode.append("import infodynamics.measures.discrete.*;\n");
 		} else {
@@ -754,7 +756,8 @@ public abstract class AutoAnalyser extends JFrame
 			String javaConstructorLine = null;
 			String pythonPreConstructorLine = null;
 			String matlabConstructorLine = null;
-			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+					selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 				// Defer our processing for this to below ...
 			} else {
 				calcContinuous = assignCalcObjectContinuous(selectedCalcType);
@@ -815,6 +818,17 @@ public abstract class AutoAnalyser extends JFrame
 					}
 					javaCode.append("\n");
 				}
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+				javaCode.append("    double[][] data = afr.getDouble2DMatrix();\n");
+				if (! allCombosCheckBox.isSelected()) {
+					for (int i=0; i < numVariables; i++) {
+						javaCode.append("    int[] " +  variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.discretise(\n        " +
+								"MatrixUtils.selectColumn(data, " + singleCalcColumns[i] + "), " +
+								propertyValues.get(DISCRETE_PROPNAME_BASE) + ");\n");
+					}
+					javaCode.append("\n");
+				}
 			} else {
 				javaCode.append("    double[][] data = afr.getDouble2DMatrix();\n");
 				if (! allCombosCheckBox.isSelected()) {
@@ -840,14 +854,19 @@ public abstract class AutoAnalyser extends JFrame
 						pythonCode.append(variableColNumLabels[i].toLowerCase() + " = JArray(JInt, 1)(data[:," +
 								singleCalcColumns[i] + "].tolist())\n");
 					}
-					pythonCode.append("\n");
+				} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+					pythonCode.append("mUtils = JPackage('infodynamics.utils').MatrixUtils\n");
+					for (int i=0; i < numVariables; i++) {
+						pythonCode.append(variableColNumLabels[i].toLowerCase() + " = mUtils.discretise(JArray(JDouble, 1)(data[:," +
+								singleCalcColumns[i] + "].tolist()), " + propertyValues.get(DISCRETE_PROPNAME_BASE) + ")\n");
+					}
 				} else {
 					for (int i=0; i < numVariables; i++) {
 						pythonCode.append(variableColNumLabels[i].toLowerCase() + " = data[:," +
 								singleCalcColumns[i] + "]\n");
 					}
-					pythonCode.append("\n");
 				}
+				pythonCode.append("\n");
 			}
 			// 3. Matlab
 			matlabCode.append("% " + loadDataComment);
@@ -859,14 +878,19 @@ public abstract class AutoAnalyser extends JFrame
 						matlabCode.append(variableColNumLabels[i].toLowerCase() + " = octaveToJavaIntArray(data(:," +
 								(singleCalcColumns[i]+1) + "));\n");
 					}
-					matlabCode.append("\n");
+				} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+					matlabCode.append("mUtils = javaObject('infodynamics.utils.MatrixUtils');\n");
+					for (int i=0; i < numVariables; i++) {
+						matlabCode.append(variableColNumLabels[i].toLowerCase() + " = mUtils.discretise(octaveToJavaDoubleArray(data(:," +
+								(singleCalcColumns[i]+1) + ")), " + propertyValues.get(DISCRETE_PROPNAME_BASE) + ");\n");
+					}
 				} else {
 					for (int i=0; i < numVariables; i++) {
 						matlabCode.append(variableColNumLabels[i].toLowerCase() + " = octaveToJavaDoubleArray(data(:," +
 								(singleCalcColumns[i]+1) + "));\n");
 					}
-					matlabCode.append("\n");
 				}
+				matlabCode.append("\n");
 			}
 
 			// Construct the calculator and set relevant properties:
@@ -874,7 +898,8 @@ public abstract class AutoAnalyser extends JFrame
 			javaCode.append("    // " + constructComment);
 			pythonCode.append("# " + constructComment);
 			matlabCode.append("% " + constructComment);
-			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+					selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 				DiscreteCalcAndArguments dcaa = assignCalcObjectDiscrete();
 				if (dcaa == null) {
 					return;
@@ -883,16 +908,19 @@ public abstract class AutoAnalyser extends JFrame
 				int base = dcaa.base;
 				String args = dcaa.arguments;
 
-				// Now check that the data is ok:
-				int minInData = MatrixUtils.min(dataDiscrete);
-				int maxInData = MatrixUtils.max(dataDiscrete);
-				if ((minInData < 0) || (maxInData >= base)) {
-					JOptionPane.showMessageDialog(this,
-							"Values in data file (in range " + minInData +
-							":" + maxInData + ") lie outside the expected range 0:" +
-							(base-1) + " for base " + base);
-					resultsLabel.setText(" ");
-					return;
+				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+					// Now check that the data is ok:
+					//  (no need to do this for binned)
+					int minInData = MatrixUtils.min(dataDiscrete);
+					int maxInData = MatrixUtils.max(dataDiscrete);
+					if ((minInData < 0) || (maxInData >= base)) {
+						JOptionPane.showMessageDialog(this,
+								"Values in data file (in range " + minInData +
+								":" + maxInData + ") lie outside the expected range 0:" +
+								(base-1) + " for base " + base);
+						resultsLabel.setText(" ");
+						return;
+					}
 				}
 				
 				// 1. Java
@@ -972,6 +1000,13 @@ public abstract class AutoAnalyser extends JFrame
 			StringBuffer extraFormatTerms = new StringBuffer();
 			if (variableCombinations.size() > 1) {
 				// We're doing all combinations.
+				
+				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+					// Set up the use of utilities for binning if required
+					pythonCode.append("mUtils = JPackage('infodynamics.utils').MatrixUtils\n");
+					matlabCode.append("mUtils = javaObject('infodynamics.utils.MatrixUtils');\n");
+				}
+				
 				// Set up the loops for these:
 				String[] columnVariables = setUpLoopsForAllCombos(javaCode, pythonCode, matlabCode);
 				
@@ -994,7 +1029,20 @@ public abstract class AutoAnalyser extends JFrame
 						matlabCode.append(matlabPrefix + variableColNumLabels[i].toLowerCase() +
 								" = octaveToJavaIntArray(data(:, " + columnVariables[i] + "));\n");
 						pythonCode.append(pythonPrefix + variableColNumLabels[i].toLowerCase() +
-								" = JArray(JInt, 1)(data[:, " + columnVariables[i] + "].tolist())\n");						
+								" = JArray(JInt, 1)(data[:, " + columnVariables[i] + "].tolist())\n");
+					}
+				} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+					for (int i = 0; i < numVariables; i++) {
+						javaCode.append(javaPrefix + "int[] " + variableColNumLabels[i].toLowerCase() +
+								" = MatrixUtils.discretise(\n" + javaPrefix + "    " +
+								"MatrixUtils.selectColumn(data, " + columnVariables[i] + "), " +
+								propertyValues.get(DISCRETE_PROPNAME_BASE) + ");\n");
+						matlabCode.append(matlabPrefix + variableColNumLabels[i].toLowerCase() +
+								" = mUtils.discretise(octaveToJavaDoubleArray(data(:," +
+								columnVariables[i] + ")), " + propertyValues.get(DISCRETE_PROPNAME_BASE) + ");\n");
+						pythonCode.append(pythonPrefix + variableColNumLabels[i].toLowerCase() +
+								" = mUtils.discretise(JArray(JDouble, 1)(data[:," +
+								columnVariables[i] + "].tolist()), " + propertyValues.get(DISCRETE_PROPNAME_BASE) + ")\n");
 					}
 				} else {
 					for (int i = 0; i < numVariables; i++) {
@@ -1026,7 +1074,8 @@ public abstract class AutoAnalyser extends JFrame
 			// Set observations
 			String supplyDataComment = "4. Supply the sample data:\n";
 			String setObservationsMethod;
-			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+					selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 				setObservationsMethod = "addObservations";
 			} else {
 				setObservationsMethod = "setObservations";
@@ -1113,7 +1162,8 @@ public abstract class AutoAnalyser extends JFrame
 				
 				// And tell the user to see results in the console:
 				resultsLabel.setText("See console for all pairs calculation results");
-				if (!selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+				if (!(selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+						selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED))) {
 					System.out.println("Property values not read back into GUI for all pairs calculation");
 				}
 			}
@@ -1157,7 +1207,8 @@ public abstract class AutoAnalyser extends JFrame
 					}
 					
 					// Initialise
-					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+							selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 						calcDiscrete.initialise();
 					} else {
 						calcContinuous.initialise();
@@ -1167,7 +1218,8 @@ public abstract class AutoAnalyser extends JFrame
 										
 					// Compute the result:
 					double result;
-					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+					if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+							selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 						result = calcDiscrete.computeAverageLocalOfObservations();
 					} else {
 						result = calcContinuous.computeAverageLocalOfObservations();
@@ -1179,7 +1231,8 @@ public abstract class AutoAnalyser extends JFrame
 						if (statSigAnalyticCheckBox.isSelected()) {
 							// analytic check of statistical significance:
 							AnalyticNullDistributionComputer analyticCalc;
-							if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+							if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+									selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 								analyticCalc =
 										(AnalyticNullDistributionComputer) calcDiscrete;
 							} else {
@@ -1195,7 +1248,8 @@ public abstract class AutoAnalyser extends JFrame
 							// permutation test of statistical significance:
 							EmpiricalMeasurementDistribution measDist;
 							EmpiricalNullDistributionComputer empiricalNullDistCalc;
-							if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+							if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+									selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 								empiricalNullDistCalc =
 										(EmpiricalNullDistributionComputer) calcDiscrete;
 							} else {
@@ -1234,7 +1288,8 @@ public abstract class AutoAnalyser extends JFrame
 				}
 	
 				if ((variableCombinations.size() == 1) && 
-						!selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+						!(selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+						 (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)))) {
 					// Read the current property values back out (in case of 
 					//  automated parameter assignment)
 					for (String propName : propertyNames) {
@@ -1508,7 +1563,8 @@ public abstract class AutoAnalyser extends JFrame
 		propertyFieldNames = new Vector();
 		propertyDescriptions = new Vector();
 		
-		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+				selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 			// Simply add the properties for this estimator, along
 			//  with their default values
 			int i = 0;
@@ -1604,13 +1660,16 @@ public abstract class AutoAnalyser extends JFrame
 	protected CalcProperties assignCalcProperties(String selectedCalcType) 
 		throws Exception {
 		
-		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
+				selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
 			CalcProperties calcProperties = new CalcProperties();
 			calcProperties.calcClass = discreteClass;
 			calcProperties.calc = null; // Not used
 			calcProperties.classSpecificPropertyNames = discreteProperties;
 			calcProperties.classSpecificPropertiesFieldNames = null; // Not used
 			calcProperties.classSpecificPropertyDescriptions = discretePropertyDescriptions;
+			// TODO Later can add binning method to the properties for 
+			//  binned calculator here.
 			return calcProperties;
 		} else {
 			// We don't handle any other calculators here
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
index 17632da..93b8438 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
@@ -161,10 +161,19 @@ public abstract class AutoAnalyserChannelCalculator extends AutoAnalyser {
 		
 		// Set observations
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-			ChannelCalculatorDiscrete cc = (ChannelCalculatorDiscrete) calcDiscrete; 
+			ChannelCalculatorDiscrete cc = (ChannelCalculatorDiscrete) calcDiscrete;
 			cc.addObservations(
 					MatrixUtils.selectColumn(dataDiscrete, sourceColumn),
 					MatrixUtils.selectColumn(dataDiscrete, destColumn));
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+			ChannelCalculatorDiscrete cc = (ChannelCalculatorDiscrete) calcDiscrete;
+			cc.addObservations(
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, sourceColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))),
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, destColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))));
 		} else {
 			ChannelCalculator cc = (ChannelCalculator) calcContinuous;
 			cc.setObservations(
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
index 3756082..b9ceb52 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
@@ -72,12 +72,12 @@ public class AutoAnalyserEntropy extends AutoAnalyser {
 
 		// Set up the properties for Entropy:
 		measureAcronym = "H";
-		appletTitle = "JIDT Entropy Auto-Analyser"; 
+		appletTitle = "JIDT Entropy Auto-Analyser";
 		
 		calcTypes = new String[] {
-				CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
 				CALC_TYPE_KOZ_LEO, CALC_TYPE_KERNEL};
-		unitsForEachCalc = new String[] {"bits", "nats", "nats", "bits"};
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats", "bits"};
 		
 		// Discrete:
 		discreteClass = MutualInformationCalculatorDiscrete.class;
@@ -211,9 +211,16 @@ public class AutoAnalyserEntropy extends AutoAnalyser {
 		
 		// Set observations
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
-			EntropyCalculatorDiscrete calc = (EntropyCalculatorDiscrete) calcDiscrete; 
+			EntropyCalculatorDiscrete calc = (EntropyCalculatorDiscrete) calcDiscrete;
 			calc.addObservations(
 					MatrixUtils.selectColumn(dataDiscrete, variableColumn));
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+			EntropyCalculatorDiscrete calc = (EntropyCalculatorDiscrete) calcDiscrete;
+			calc.addObservations(
+					MatrixUtils.discretise(
+							MatrixUtils.selectColumn(data, variableColumn),
+							// Should be no parse error on the alphabet size by now
+							Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))));
 		} else {
 			EntropyCalculator calc = (EntropyCalculator) calcContinuous;
 			calc.setObservations(
@@ -282,7 +289,17 @@ public class AutoAnalyserEntropy extends AutoAnalyser {
 			resultsLabel.setText("Cannot find a value for property " + DISCRETE_PROPNAME_BASE);
 			return null;
 		}
-		int base = Integer.parseInt(basePropValueStr);
+		int base;
+		try {
+			base = Integer.parseInt(basePropValueStr);
+		} catch (NumberFormatException nfe) {
+			JOptionPane.showMessageDialog(this,
+					"Cannot parse number for property " + DISCRETE_PROPNAME_BASE
+					+ ": " + nfe.getMessage());
+			resultsLabel.setText("Cannot parse number for property " +
+					DISCRETE_PROPNAME_BASE + ": " + nfe.getMessage());
+			return null;
+		}
 		
 		return new DiscreteCalcAndArguments(
 				new EntropyCalculatorDiscrete(base),
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
index 13afaeb..b62db45 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserMI.java
@@ -65,10 +65,10 @@ public class AutoAnalyserMI extends AutoAnalyserChannelCalculator
 		appletTitle = "JIDT Mutual Information Auto-Analyser"; 
 		
 		calcTypes = new String[] {
-				CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
 				CALC_TYPE_KRASKOV_ALG1, CALC_TYPE_KRASKOV_ALG2,
 				CALC_TYPE_KERNEL};
-		unitsForEachCalc = new String[] {"bits", "nats", "nats", "nats", "bits"};
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats", "nats", "bits"};
 		
 		// Discrete:
 		discreteClass = MutualInformationCalculatorDiscrete.class;
@@ -181,9 +181,10 @@ public class AutoAnalyserMI extends AutoAnalyserChannelCalculator
 	 * Method to assign and initialise our discrete calculator class
 	 */
 	protected DiscreteCalcAndArguments assignCalcObjectDiscrete() throws Exception {
-		String timeDiffPropValueStr, basePropValueStr;
+		int timeDiff, base;
 		try {
-			timeDiffPropValueStr = propertyValues.get(DISCRETE_PROPNAME_TIME_DIFF);
+			String timeDiffPropValueStr = propertyValues.get(DISCRETE_PROPNAME_TIME_DIFF);
+			timeDiff = Integer.parseInt(timeDiffPropValueStr);
 		} catch (Exception ex) {
 			JOptionPane.showMessageDialog(this,
 					ex.getMessage());
@@ -191,15 +192,14 @@ public class AutoAnalyserMI extends AutoAnalyserChannelCalculator
 			return null;
 		}
 		try {
-			basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+			String basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+			base = Integer.parseInt(basePropValueStr);
 		} catch (Exception ex) {
 			JOptionPane.showMessageDialog(this,
 					ex.getMessage());
 			resultsLabel.setText("Cannot find a value for property " + DISCRETE_PROPNAME_BASE);
 			return null;
 		}
-		int timeDiff = Integer.parseInt(timeDiffPropValueStr);
-		int base = Integer.parseInt(basePropValueStr);
 		
 		return new DiscreteCalcAndArguments(
 				new MutualInformationCalculatorDiscrete(base, timeDiff),
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
index 29ee653..e7a03b2 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserTE.java
@@ -66,9 +66,9 @@ public class AutoAnalyserTE extends AutoAnalyserChannelCalculator
 		appletTitle = "JIDT Transfer Entropy Auto-Analyser"; 
 		
 		calcTypes = new String[] {
-				CALC_TYPE_DISCRETE, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
 				CALC_TYPE_KRASKOV, CALC_TYPE_KERNEL};
-		unitsForEachCalc = new String[] {"bits", "nats", "nats", "bits"};
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats", "bits"};
 		
 		// Discrete:
 		discreteClass = TransferEntropyCalculatorDiscrete.class;

From 7972e0342083b56e8ae80bebfa5e678857bc0625 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 13:40:21 +1000
Subject: [PATCH 48/78] Adding scripts to run Entropy AutoAnalyser

---
 demos/AutoAnalyser/runHAutoAnalyser.bat | 8 ++++++++
 demos/AutoAnalyser/runHAutoAnalyser.sh  | 8 ++++++++
 2 files changed, 16 insertions(+)
 create mode 100755 demos/AutoAnalyser/runHAutoAnalyser.bat
 create mode 100755 demos/AutoAnalyser/runHAutoAnalyser.sh

diff --git a/demos/AutoAnalyser/runHAutoAnalyser.bat b/demos/AutoAnalyser/runHAutoAnalyser.bat
new file mode 100755
index 0000000..ded36d1
--- /dev/null
+++ b/demos/AutoAnalyser/runHAutoAnalyser.bat
@@ -0,0 +1,8 @@
+@ECHO OFF
+
+REM Make sure the latest example source file is compiled.
+javac -classpath "..\java;..\..\infodynamics.jar" "..\java\infodynamics\demos\autoanalysis\AutoAnalyserEntropy.java"
+
+REM Run the example:
+java -classpath "..\java;..\..\infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserEntropy
+
diff --git a/demos/AutoAnalyser/runHAutoAnalyser.sh b/demos/AutoAnalyser/runHAutoAnalyser.sh
new file mode 100755
index 0000000..c64f427
--- /dev/null
+++ b/demos/AutoAnalyser/runHAutoAnalyser.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Make sure the latest example source file is compiled.
+javac -classpath "../java:../../infodynamics.jar" "../java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java"
+
+# Run the example:
+java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserEntropy
+

From 4db008ce3386dbcd4e5b49aa6fb21c91aa33df9b Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 21:26:44 +1000
Subject: [PATCH 49/78] For Conditional MI calculators (sontinuous), adding
 setObservations and addObservations method signatures which take univariate
 arrays (only work if initialised dimensionalities are univariate) into the
 interface, and the common base class. This is in preparation for the CMI
 AutoAnalyser

---
 ...ionalMutualInfoCalculatorMultiVariate.java | 48 +++++++++++++++++++
 ...nditionalMutualInfoMultiVariateCommon.java | 26 ++++++++++
 2 files changed, 74 insertions(+)

diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
index a86ce64..6c1e03c 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java
@@ -114,6 +114,29 @@ public interface ConditionalMutualInfoCalculatorMultiVariate
 	public void setObservations(double[][] var1, double[][] var2,
 			double[][] cond) throws Exception;
 
+	/**
+	 * Sets a single series from which to compute the PDF.
+	 * Cannot be called in conjunction with
+	 * {@link #startAddObservations()} / {@link #addObservations(double[][], double[][], double[][])} or
+	 * {@link #addObservations(double[][], double[][], double[][], int, int)} /
+	 * {@link #finaliseAddObservations()}.
      
+	 * 
+	 * 
      The supplied series may be (multivariate) time-series or 
+	 * simply a set of separate observations without a time interpretation.
+	 * 
+	 * 
      This method can only be used where dimensions of all
+	 * variables have been set to 1.
      
+	 * 
+	 * @param var1 univariate observations for variable 1
+	 * @param var2 univariate observations for variable 2
+	 *  Length must match var1, and their indices must correspond.
+	 * @param cond univariate observations for the conditional
+	 *  Length must match var1, and their indices must correspond.
+	 * @throws Exception
+	 */
+	public void setObservations(double[] var1, double[] var2,
+			double[] cond) throws Exception;
+
 	/**
 	 * Sets a single series from which to compute the PDF,
 	 *  where all the various observations are valid.
@@ -210,6 +233,31 @@ public interface ConditionalMutualInfoCalculatorMultiVariate
 	public void addObservations(double[][] var1, double[][] var2,
 			double[][] cond) throws Exception;
 	
+	/**
+	 * 
      Adds a new set of observations to update the PDFs with - is
+	 * intended to be called multiple times.
+	 * Must be called after {@link #startAddObservations()}; call
+	 * {@link #finaliseAddObservations()} once all observations have
+	 * been supplied.
      
+	 * 
+	 * 
      Note that the arrays must not be over-written by the user
+	 *  until after finaliseAddObservations() has been called
+	 *  (they are not copied by this method necessarily, but the method
+	 *  may simply hold a pointer to them).
      
+	 * 
+	 * 
      This method can only be used where dimensions of all
+	 * variables have been set to 1.
      
+	 * 
+	 * @param var1 univariate observations for variable 1
+	 * @param var2 univariate observations for variable 2
+	 *  Length must match var1, and their indices must correspond.
+	 * @param cond univariate observations for the conditional
+	 *  Length must match var1, and their indices must correspond.
+	 * @throws Exception
+	 */
+	public void addObservations(double[] var1, double[] var2,
+			double[] cond) throws Exception;
+	
 	/**
 	 * 
      Adds a new sub-series of observations to update the PDFs with - is
 	 * intended to be called multiple times.
diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
index dd5460e..557ca24 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java
@@ -237,6 +237,15 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements
 		addedMoreThanOneObservationSet = false;
 	}
 
+	@Override
+	public void setObservations(double[] var1, double[] var2,
+			double[] cond) throws Exception {
+		startAddObservations();
+		addObservations(var1, var2, cond);
+		finaliseAddObservations();
+		addedMoreThanOneObservationSet = false;
+	}
+
 	@Override
 	public void startAddObservations() {
 		vectorOfVar1Observations = new Vector();
@@ -276,6 +285,23 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements
 		}
 	}
 
+	@Override
+	public void addObservations(double[] var1, double[] var2,
+			double[] cond) throws Exception {
+		if ((dimensionsVar1 != 1) || (dimensionsVar2 != 1) ||
+				(dimensionsCond != 1)) {
+			throw new Exception("The number of dimensions for each variable (having been initialised to " +
+					dimensionsVar1 + ", " + dimensionsVar2 + " & " +
+					dimensionsCond + ") can only be 1 when " +
+					"the univariate addObservations(double[],double[],double[]) and " + 
+					"setObservations(double[],double[],double[]) methods are called");
+		}
+		addObservations(MatrixUtils.reshape(var1, var1.length, 1),
+						MatrixUtils.reshape(var2, var2.length, 1),
+						MatrixUtils.reshape(cond, cond.length, 1));
+
+	}
+	
 	@Override
 	public void addObservations(double[][] var1, double[][] var2,
 			double[][] cond,

From 7c837bca686310c143244d2c67220b85edefa175 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 21:28:57 +1000
Subject: [PATCH 50/78] Adding Conditional MI AutoAnalyser java implementation
 and shell scripts to run it. Includes some minor changes to AutoAnalyser to
 implement it (allowing selective disabling of column selection fields under
 the all combinations option, and variable indenting for all combos option),
 and to other child classes of it to follow suit. Also adds detection of
 column number inputs being out of bounds or mangled.

---
 demos/AutoAnalyser/runCMIAutoAnalyser.bat     |   8 +
 demos/AutoAnalyser/runCMIAutoAnalyser.sh      |   8 +
 .../demos/autoanalysis/AutoAnalyser.java      |  50 ++-
 .../demos/autoanalysis/AutoAnalyserCMI.java   | 366 ++++++++++++++++++
 .../AutoAnalyserChannelCalculator.java        |   3 +
 .../autoanalysis/AutoAnalyserEntropy.java     |   4 +-
 6 files changed, 421 insertions(+), 18 deletions(-)
 create mode 100755 demos/AutoAnalyser/runCMIAutoAnalyser.bat
 create mode 100755 demos/AutoAnalyser/runCMIAutoAnalyser.sh
 create mode 100644 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java

diff --git a/demos/AutoAnalyser/runCMIAutoAnalyser.bat b/demos/AutoAnalyser/runCMIAutoAnalyser.bat
new file mode 100755
index 0000000..07dbda8
--- /dev/null
+++ b/demos/AutoAnalyser/runCMIAutoAnalyser.bat
@@ -0,0 +1,8 @@
+@ECHO OFF
+
+REM Make sure the latest example source file is compiled.
+javac -classpath "..\java;..\..\infodynamics.jar" "..\java\infodynamics\demos\autoanalysis\AutoAnalyserCMI.java"
+
+REM Run the example:
+java -classpath "..\java;..\..\infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserCMI
+
diff --git a/demos/AutoAnalyser/runCMIAutoAnalyser.sh b/demos/AutoAnalyser/runCMIAutoAnalyser.sh
new file mode 100755
index 0000000..5c508a4
--- /dev/null
+++ b/demos/AutoAnalyser/runCMIAutoAnalyser.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Make sure the latest example source file is compiled.
+javac -classpath "../java:../../infodynamics.jar" "../java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java"
+
+# Run the example:
+java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserCMI
+
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 44ea6d5..8157e16 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -121,6 +121,9 @@ public abstract class AutoAnalyser extends JFrame
 	//  Should be overridden by the child classes.
 	protected String[] variableColNumLabels = null;
 	
+	protected boolean[] disableVariableColTextFieldsForAllCombos = null;
+	protected int indentsForAllCombos = 1;
+	
 	// Store which calculator type we're using:
 	@SuppressWarnings("rawtypes")
 	protected Class calcClass = null;
@@ -688,22 +691,30 @@ public abstract class AutoAnalyser extends JFrame
 		
 		int[] singleCalcColumns = new int[numVariables];
 		Vector variableCombinations = new Vector();
-		if (allCombosCheckBox.isSelected()) {
-			// We're doing all combinations
-			fillOutAllCombinations(variableCombinations);
-		} else {
-			// we're doing a single combination
-			for (int i = 0; i < numVariables; i++) {
-				singleCalcColumns[i] = Integer.parseInt(variableColTextFields[i].getText());
-				if ((singleCalcColumns[i] < 0) || (singleCalcColumns[i] >= dataColumns)) {
-					JOptionPane.showMessageDialog(this,
-							String.format("%s column must be between 0 and %d for this data set",
-									variableColNumLabels[i], dataColumns-1));
-					resultsLabel.setText(" ");				
-					return;
+		try {
+			if (allCombosCheckBox.isSelected()) {
+				// We're doing all combinations
+				fillOutAllCombinations(variableCombinations);
+			} else {
+				// we're doing a single combination
+				for (int i = 0; i < numVariables; i++) {
+					singleCalcColumns[i] = Integer.parseInt(variableColTextFields[i].getText());
+					if ((singleCalcColumns[i] < 0) || (singleCalcColumns[i] >= dataColumns)) {
+						JOptionPane.showMessageDialog(this,
+								String.format("%s column must be between 0 and %d for this data set",
+										variableColNumLabels[i], dataColumns-1));
+						resultsLabel.setText(" ");
+						return;
+					}
 				}
+				variableCombinations.add(singleCalcColumns);
 			}
-			variableCombinations.add(singleCalcColumns);
+		} catch (Exception e) {
+			// Catches number format exception, and column number out of bounds
+			JOptionPane.showMessageDialog(this,
+					e.getMessage());
+			resultsLabel.setText("Cannot parse a column number from input: " + e.getMessage());
+			return;
 		}
 
 		// Generate headers:
@@ -1013,10 +1024,13 @@ public abstract class AutoAnalyser extends JFrame
 				// Get the indents right from here on, 
 				//  and prepare a string of the column labels for
 				//  each variable, to be used in later formatting.
-				for (int i = 0; i < numVariables; i++) {
+				for (int i = 0; i < indentsForAllCombos; i++) {
 					javaPrefix += "    ";
 					pythonPrefix += "\t";
 					matlabPrefix += "\t";
+				}
+				
+				for (int i = 0; i < numVariables; i++) {
 					extraFormatTerms.append(columnVariables[i] + ", ");
 				}
 				
@@ -1324,7 +1338,7 @@ public abstract class AutoAnalyser extends JFrame
 	 *  
 	 * @param variableCombinations
 	 */
-	protected abstract void fillOutAllCombinations(Vector variableCombinations);
+	protected abstract void fillOutAllCombinations(Vector variableCombinations) throws Exception;
 	
 	/**
 	 * Method to allow child classes to set up the loops over all combinations of
@@ -1748,7 +1762,9 @@ public abstract class AutoAnalyser extends JFrame
 			// "All pairs" checkbox -- update in case changed:
 			if (allCombosCheckBox.isSelected()) {
 				for (int i = 0; i < numVariables; i++) {
-					variableColTextFields[i].setEnabled(false);
+					if (disableVariableColTextFieldsForAllCombos[i]) {
+						variableColTextFields[i].setEnabled(false);
+					}
 				}
 			} else {
 				for (int i = 0; i < numVariables; i++) {
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
new file mode 100644
index 0000000..526c9ff
--- /dev/null
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
@@ -0,0 +1,366 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2015, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.demos.autoanalysis;
+
+import infodynamics.measures.continuous.ConditionalMutualInfoCalculatorMultiVariate;
+import infodynamics.measures.continuous.ConditionalMutualInfoMultiVariateCommon;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.continuous.gaussian.ConditionalMutualInfoCalculatorMultiVariateGaussian;
+import infodynamics.measures.continuous.kraskov.ConditionalMutualInfoCalculatorMultiVariateKraskov;
+import infodynamics.measures.continuous.kraskov.ConditionalMutualInfoCalculatorMultiVariateKraskov1;
+import infodynamics.measures.continuous.kraskov.ConditionalMutualInfoCalculatorMultiVariateKraskov2;
+import infodynamics.measures.discrete.ConditionalMutualInformationCalculatorDiscrete;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
+import infodynamics.utils.MatrixUtils;
+
+import javax.swing.JOptionPane;
+import javax.swing.event.DocumentListener;
+
+import java.awt.event.ActionListener;
+import java.awt.event.MouseListener;
+import java.util.Vector;
+
+/**
+ * This class provides a GUI to build a simple conditional mutual information calculation,
+ *  and supply the code to execute it.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class AutoAnalyserCMI extends AutoAnalyser
+	implements ActionListener, DocumentListener, MouseListener {
+
+	/**
+	 * Need serialVersionUID to be serializable
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	// Property names for specific continuous calculators:
+	protected String[] gaussianProperties;
+	protected String[] gaussianPropertiesFieldNames;
+	protected String[] gaussianPropertyDescriptions;
+	protected String[] kraskovProperties;
+	protected String[] kraskovPropertiesFieldNames;
+	protected String[] kraskovPropertyDescriptions;
+
+	protected static final String DISCRETE_PROPNAME_TIME_DIFF = "time difference";
+	
+	protected static final String CALC_TYPE_KRASKOV_ALG1 = CALC_TYPE_KRASKOV + " alg. 1";
+	protected static final String CALC_TYPE_KRASKOV_ALG2 = CALC_TYPE_KRASKOV + " alg. 2";
+	
+	/**
+	 * Constructor to initialise the GUI for MI
+	 */
+	protected void makeSpecificInitialisations() {
+		
+		numVariables = 3;
+		variableColNumLabels = new String[] {"Source", "Destination", "Conditional"};
+		useAllCombosCheckBox = true;
+		useStatSigCheckBox = true;
+		wordForCombinations = "pairs";
+		variableRelationshipFormatString = "col_%d -> col_%d | col_%d";
+		disableVariableColTextFieldsForAllCombos = new boolean[] {true, true, false};
+		indentsForAllCombos = 2;
+		
+		// Set up the properties for CMI:
+		measureAcronym = "CMI";
+		appletTitle = "JIDT Conditional MI Auto-Analyser"; 
+		
+		calcTypes = new String[] {
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_KRASKOV_ALG1, CALC_TYPE_KRASKOV_ALG2};
+				// No kernel calculator defined for CMI (yet, and unlikely to happen)
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats", "nats"};
+		
+		// Discrete:
+		discreteClass = ConditionalMutualInformationCalculatorDiscrete.class;
+		discreteProperties = new String[] {
+				DISCRETE_PROPNAME_BASE
+		};
+		discretePropertyDefaultValues = new String[] {
+				"2"
+		};
+		discretePropertyDescriptions = new String[] {
+				"Number of discrete states available for each variable (i.e. 2 for binary).
      " +
+				"Can be set individually for each variable -- see code."
+		};
+		
+		// Continuous:
+		abstractContinuousClass = ConditionalMutualInfoCalculatorMultiVariate.class;
+		// Common properties for all continuous calcs:
+		commonContPropertyNames = new String[] {
+				// None
+		};
+		commonContPropertiesFieldNames = new String[] {
+				// None
+		};
+		commonContPropertyDescriptions = new String[] {
+				// None
+		};
+		// Gaussian properties:
+		gaussianProperties = new String[] {
+		};
+		gaussianPropertiesFieldNames = new String[] {
+		};
+		gaussianPropertyDescriptions = new String[] {
+		};
+		// KSG (Kraskov):
+		kraskovProperties = new String[] {
+				ConditionalMutualInfoMultiVariateCommon.PROP_NORMALISE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_K,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS,
+		};
+		kraskovPropertiesFieldNames = new String[] {
+				"ConditionalMutualInfoMultiVariateCommon.PROP_NORMALISE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_K",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS",
+		};
+		kraskovPropertyDescriptions = new String[] {
+				"(boolean) whether to normalise 
      each incoming time-series to mean 0, standard deviation 1, or not (recommended)",
+				"Number of k nearest neighbours to use 
      in the full joint kernel space in the KSG algorithm",
+				"Standard deviation for an amount 
      of random Gaussian noise to add to each variable, " +
+						"to avoid having neighbourhoods with artificially large counts. 
      " +
+						"(\"false\" may be used to indicate \"0\".). The amount is added in after any normalisation.",
+				"Dynamic correlation exclusion time or 
      Theiler window (see Kantz and Schreiber); " +
+						"0 (default) means no dynamic exclusion window",
+				"
      Norm type to use in KSG algorithm between the points in each marginal space. 
      Options are: " +
+						"\"MAX_NORM\" (default), otherwise \"EUCLIDEAN\" or \"EUCLIDEAN_SQUARED\" (both equivalent here)",
+				"Number of parallel threads to use 
      in computation: an integer > 0 or \"USE_ALL\" " +
+						"(default, to indicate to use all available processors)",
+		};
+		
+	}
+
+	@Override
+	protected void fillOutAllCombinations(Vector variableCombinations) throws Exception {
+		// All combinations here means all pairs of sources and destinations,
+		//  with the conditional fixed.
+		int conditional = Integer.parseInt(variableColTextFields[2].getText());
+		if (conditional >= dataColumns) {
+			throw new Exception(String.format("%s column must be between 0 and %d for this data set",
+					variableColNumLabels[2], dataColumns-1));
+		}
+		for (int s = 0; s < dataColumns; s++) {
+			for (int d = 0; d < dataColumns; d++) {
+				variableCombinations.add(new int[] {s, d, conditional});
+			}
+		}
+	}
+
+	@Override
+	protected String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		// Set up loops in the code:
+		int conditional = Integer.parseInt(variableColTextFields[2].getText());
+		// 1. Java code
+		javaCode.append("    \n");
+		javaCode.append("    int c = " + conditional + ";\n");
+		javaCode.append("    // Compute for all source-destination pairs:\n");
+		javaCode.append("    for (int s = 0; s < " + dataColumns +
+					"; s++) {\n");
+		javaCode.append("        for (int d = 0; d < " + dataColumns +
+				"; d++) {\n");
+		String javaPrefix = "            ";
+		javaCode.append(javaPrefix + "// For each source-dest pair (given conditional):\n");
+		javaCode.append(javaPrefix + "if ((s == d) || (s == c) || (d == c)) {\n");
+		javaCode.append(javaPrefix + "    continue;\n");
+		javaCode.append(javaPrefix + "}\n");
+		// 2. Python code
+		pythonCode.append("\n");
+		pythonCode.append("c = " + conditional + "\n");
+		pythonCode.append("# Compute for all pairs:\n");
+		pythonCode.append("for s in range(" + dataColumns + "):\n");
+		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "\t\t";
+		pythonCode.append(pythonPrefix+ "# For each source-dest pair (given conditional):\n");
+		pythonCode.append(pythonPrefix + "if ((s == d) or (s == c) or (d == c)):\n");
+		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		// 3. Matlab code
+		matlabCode.append("\n");
+		matlabCode.append("c = " + (conditional+1) + ";\n");
+		matlabCode.append("% Compute for all pairs:\n");
+		matlabCode.append("for s = 1:" + dataColumns + "\n");
+		matlabCode.append("\tfor d = 1:" + dataColumns + "\n");
+		String matlabPrefix = "\t\t";
+		matlabCode.append(matlabPrefix + "% For each source-dest pair (given conditional):\n");
+		matlabCode.append(matlabPrefix + "if ((s == d) || (s == c) || (d == c))\n");
+		matlabCode.append(matlabPrefix + "\tcontinue;\n");
+		matlabCode.append(matlabPrefix + "end\n");
+		
+		// Return the variables to index each column:
+		return new String[] {"s", "d", "c"}; 
+	}
+
+	@Override
+	protected void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		
+		// 1. Java code
+		javaCode.append("        }\n");
+		javaCode.append("    }\n");
+		// 2. Python code
+		// Nothing to do
+		// 3. Matlab code
+		matlabCode.append("\tend\n");
+		matlabCode.append("end\n");
+	}
+	
+	@Override
+	protected String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers) {
+		// We format the source and target variable numbers into the
+		//  return string here:
+		return String.format(formatStr,
+				columnNumbers[0], columnNumbers[1], columnNumbers[2]);
+	}
+	
+	@Override
+	protected boolean skipColumnCombo(int[] columnCombo) {
+		if ((columnCombo[0] == columnCombo[1]) ||
+				(columnCombo[0] == columnCombo[2]) ||
+				(columnCombo[1] == columnCombo[2])) {
+			// Two columns are the same here,
+			//  so don't compute the conditional MI
+			return true;
+		}
+		return false;
+	}
+	
+	@Override
+	protected void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception {
+		
+		String selectedCalcType = (String)
+			calcTypeComboBox.getSelectedItem();
+		
+		int sourceColumn = columnCombo[0];
+		int destColumn = columnCombo[1];
+		int condColumn = columnCombo[2];
+		
+		// Set observations
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			ConditionalMutualInformationCalculatorDiscrete cmiCalc =
+					(ConditionalMutualInformationCalculatorDiscrete) calcDiscrete;
+			cmiCalc.addObservations(
+					MatrixUtils.selectColumn(dataDiscrete, sourceColumn),
+					MatrixUtils.selectColumn(dataDiscrete, destColumn),
+					MatrixUtils.selectColumn(dataDiscrete, condColumn));
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+			ConditionalMutualInformationCalculatorDiscrete cmiCalc =
+					(ConditionalMutualInformationCalculatorDiscrete) calcDiscrete;
+			cmiCalc.addObservations(
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, sourceColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))),
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, destColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))),
+					MatrixUtils.discretise(
+							MatrixUtils.selectColumn(data, condColumn),
+							Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))));
+		} else {
+			ConditionalMutualInfoCalculatorMultiVariate cmiCalc =
+					(ConditionalMutualInfoCalculatorMultiVariate) calcContinuous;
+			cmiCalc.setObservations(
+					MatrixUtils.selectColumn(data, sourceColumn),
+					MatrixUtils.selectColumn(data, destColumn),
+					MatrixUtils.selectColumn(data, condColumn));
+		}
+	}
+	
+	protected CalcProperties assignCalcProperties(String selectedCalcType)
+			throws Exception {
+		// Let the super class handle discrete calculators
+		CalcProperties calcProperties = super.assignCalcProperties(selectedCalcType);
+		if (calcProperties == null) {
+			// We need to assign properties for a continuous calculator
+			calcProperties = new CalcProperties();
+			calcProperties.calc = assignCalcObjectContinuous(selectedCalcType);
+			calcProperties.calcClass = calcProperties.calc.getClass();
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+				calcProperties.classSpecificPropertyNames = gaussianProperties;
+				calcProperties.classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
+			} else if (selectedCalcType.startsWith(CALC_TYPE_KRASKOV)) {
+				// The if statement will work for both MI Kraskov calculators
+				calcProperties.classSpecificPropertyNames = kraskovProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kraskovPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kraskovPropertyDescriptions;
+			} else {
+				calcProperties = null;
+				throw new Exception("No recognised calculator selected: " +
+						selectedCalcType);
+			}
+		}
+		return calcProperties;
+	}
+
+	/**
+	 * Method to assign and initialise our continuous calculator class
+	 */
+	@Override
+	protected ConditionalMutualInfoCalculatorMultiVariate assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+			return new ConditionalMutualInfoCalculatorMultiVariateGaussian();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV_ALG1)) {
+			return new ConditionalMutualInfoCalculatorMultiVariateKraskov1();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV_ALG2)) {
+			return new ConditionalMutualInfoCalculatorMultiVariateKraskov2();
+		} else {
+			throw new Exception("No recognised continuous calculator selected: " +
+					selectedCalcType);
+		}
+
+	}
+
+	/**
+	 * Method to assign and initialise our discrete calculator class
+	 */
+	protected DiscreteCalcAndArguments assignCalcObjectDiscrete() throws Exception {
+		int base;
+		try {
+			String basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+			base = Integer.parseInt(basePropValueStr);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot find a value for property " + DISCRETE_PROPNAME_BASE);
+			return null;
+		}
+		
+		return new DiscreteCalcAndArguments(
+				new ConditionalMutualInformationCalculatorDiscrete(base, base, base),
+				base,
+				base + ", " + base + ", " + base);
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		new AutoAnalyserCMI();
+	}
+}
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
index 93b8438..d365b90 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
@@ -64,6 +64,9 @@ public abstract class AutoAnalyserChannelCalculator extends AutoAnalyser {
 		useStatSigCheckBox = true;
 		wordForCombinations = "pairs";
 		variableRelationshipFormatString = "col_%d -> col_%d";
+		
+		disableVariableColTextFieldsForAllCombos = new boolean[] {true, true};
+		indentsForAllCombos = 2;
 	}
 
 	@Override
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
index b9ceb52..3131cae 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
@@ -69,7 +69,9 @@ public class AutoAnalyserEntropy extends AutoAnalyser {
 		useStatSigCheckBox = false;
 		wordForCombinations = "variables";
 		variableRelationshipFormatString = "col_%d";
-
+		disableVariableColTextFieldsForAllCombos = new boolean[] {true};
+		indentsForAllCombos = 1;
+		
 		// Set up the properties for Entropy:
 		measureAcronym = "H";
 		appletTitle = "JIDT Entropy Auto-Analyser";

From 4bc411654f5462fbd6c7aeb205882477448413c0 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 22 Aug 2017 21:40:27 +1000
Subject: [PATCH 51/78] Removed superfluous "time difference" parameter from
 AutoAnalyerCMI class.

---
 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java | 2 --
 1 file changed, 2 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
index 526c9ff..dc1d09b 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
@@ -60,8 +60,6 @@ public class AutoAnalyserCMI extends AutoAnalyser
 	protected String[] kraskovPropertiesFieldNames;
 	protected String[] kraskovPropertyDescriptions;
 
-	protected static final String DISCRETE_PROPNAME_TIME_DIFF = "time difference";
-	
 	protected static final String CALC_TYPE_KRASKOV_ALG1 = CALC_TYPE_KRASKOV + " alg. 1";
 	protected static final String CALC_TYPE_KRASKOV_ALG2 = CALC_TYPE_KRASKOV + " alg. 2";
 	

From 0e40ec2401ea9cd7d08d6eb69db65adf58d92dd2 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 23 Aug 2017 00:18:41 +1000
Subject: [PATCH 52/78] Initialising conditional TE calculator to have one
 conditional dimension by default instead of none. Also fixing the way the
 conditional parameters are printed via getProperty() when they are null.

---
 ...ferEntropyCalculatorViaCondMutualInfo.java | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/java/source/infodynamics/measures/continuous/ConditionalTransferEntropyCalculatorViaCondMutualInfo.java b/java/source/infodynamics/measures/continuous/ConditionalTransferEntropyCalculatorViaCondMutualInfo.java
index d49faa3..acb7bf5 100755
--- a/java/source/infodynamics/measures/continuous/ConditionalTransferEntropyCalculatorViaCondMutualInfo.java
+++ b/java/source/infodynamics/measures/continuous/ConditionalTransferEntropyCalculatorViaCondMutualInfo.java
@@ -98,17 +98,17 @@ public class ConditionalTransferEntropyCalculatorViaCondMutualInfo implements
 	 * Array of embedding lengths for each conditional variable.
 	 *  Can be an empty array or null if there are no conditional variables.
 	 */
-	protected int[] condEmbedDims = null;
+	protected int[] condEmbedDims = new int[] {1};
 	/**
 	 * Array of embedding delays for the conditional variables.
 	 *  Must be same length as condEmbedDims array.
 	 */
-	protected int[] cond_taus = null;
+	protected int[] cond_taus = new int[] {1};
 	/**
 	 * Array of time lags between last element of each conditional variable
 	 *  and destination next value.
 	 */
-	protected int[] condDelays = null;
+	protected int[] condDelays = new int[] {1};
 	
 	/**
 	 * Time index of the last point in the destination embedding of the first
@@ -380,11 +380,23 @@ public class ConditionalTransferEntropyCalculatorViaCondMutualInfo implements
 		} else if (propertyName.equalsIgnoreCase(DELAY_PROP_NAME)) {
 			return Integer.toString(delay);
 		} else if (propertyName.equalsIgnoreCase(COND_EMBED_LENGTHS_PROP_NAME)) {
-			return MatrixUtils.arrayToString(condEmbedDims);
+			if (condEmbedDims == null) {
+				return "";
+			} else {
+				return MatrixUtils.arrayToString(condEmbedDims);
+			}
 		} else if (propertyName.equalsIgnoreCase(COND_EMBED_DELAYS_PROP_NAME)) {
-			return MatrixUtils.arrayToString(cond_taus);
+			if (cond_taus == null) {
+				return "";
+			} else {
+				return MatrixUtils.arrayToString(cond_taus);
+			}
 		} else if (propertyName.equalsIgnoreCase(COND_DELAYS_PROP_NAME)) {
-			return MatrixUtils.arrayToString(condDelays);
+			if (condDelays == null) {
+				return "";
+			} else {
+				return MatrixUtils.arrayToString(condDelays);
+			}
 		} else {
 			// No property matches for this class, assume it is for the underlying
 			//  conditional MI calculator

From b2e3ae1ca09c94952377938fd812695c1b8d5839 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 23 Aug 2017 00:19:09 +1000
Subject: [PATCH 53/78] Added missing getProperty() method to Conditional TE
 calculator Kraskov

---
 .../ConditionalTransferEntropyCalculatorKraskov.java   | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/java/source/infodynamics/measures/continuous/kraskov/ConditionalTransferEntropyCalculatorKraskov.java b/java/source/infodynamics/measures/continuous/kraskov/ConditionalTransferEntropyCalculatorKraskov.java
index f224229..0728a85 100755
--- a/java/source/infodynamics/measures/continuous/kraskov/ConditionalTransferEntropyCalculatorKraskov.java
+++ b/java/source/infodynamics/measures/continuous/kraskov/ConditionalTransferEntropyCalculatorKraskov.java
@@ -232,4 +232,14 @@ public class ConditionalTransferEntropyCalculatorKraskov
 			props.put(propertyName, propertyValue); // This will keep properties for the super class as well as the cond MI calculator, but this is ok
 		}
 	}
+	
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(PROP_KRASKOV_ALG_NUM)) {
+			return Integer.toString(kraskovAlgorithmNumber);
+		} else {
+			// Assume it was a property for the parent class or underlying conditional MI calculator
+			return super.getProperty(propertyName);
+		}
+	}
 }

From c59d932a44453b31fd59ac4d47293bd0895c5352 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 23 Aug 2017 00:19:43 +1000
Subject: [PATCH 54/78] Misc spacing/comment patches

---
 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java | 2 +-
 .../discrete/ConditionalTransferEntropyCalculatorDiscrete.java  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
index dc1d09b..7c5a173 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
@@ -64,7 +64,7 @@ public class AutoAnalyserCMI extends AutoAnalyser
 	protected static final String CALC_TYPE_KRASKOV_ALG2 = CALC_TYPE_KRASKOV + " alg. 2";
 	
 	/**
-	 * Constructor to initialise the GUI for MI
+	 * Constructor to initialise the GUI for CMI
 	 */
 	protected void makeSpecificInitialisations() {
 		
diff --git a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
index d16c8d7..fe032bb 100644
--- a/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/ConditionalTransferEntropyCalculatorDiscrete.java
@@ -226,7 +226,7 @@ public class ConditionalTransferEntropyCalculatorDiscrete
 		super(base);
 		
 		k = history;
-    base_others = base; // If unspecified, the base of the conditional variables is the same as src and tgt
+		base_others = base; // If unspecified, the base of the conditional variables is the same as src and tgt
 		this.numOtherInfoContributors = numOtherInfoContributors;
 		base_power_k = MathsUtils.power(base, k);
 		base_power_num_others = MathsUtils.power(base, numOtherInfoContributors);

From 9f4ce0236384f2c41358a006f2388e1ae262e020 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 23 Aug 2017 00:37:28 +1000
Subject: [PATCH 55/78] Added Conditional Transfer Entropy AutoAnalyser source
 code, and startup scripts.

---
 demos/AutoAnalyser/runCTEAutoAnalyser.bat     |   8 +
 demos/AutoAnalyser/runCTEAutoAnalyser.sh      |   8 +
 .../demos/autoanalysis/AutoAnalyserCTE.java   | 397 ++++++++++++++++++
 3 files changed, 413 insertions(+)
 create mode 100755 demos/AutoAnalyser/runCTEAutoAnalyser.bat
 create mode 100755 demos/AutoAnalyser/runCTEAutoAnalyser.sh
 create mode 100644 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java

diff --git a/demos/AutoAnalyser/runCTEAutoAnalyser.bat b/demos/AutoAnalyser/runCTEAutoAnalyser.bat
new file mode 100755
index 0000000..7f106e1
--- /dev/null
+++ b/demos/AutoAnalyser/runCTEAutoAnalyser.bat
@@ -0,0 +1,8 @@
+@ECHO OFF
+
+REM Make sure the latest example source file is compiled.
+javac -classpath "..\java;..\..\infodynamics.jar" "..\java\infodynamics\demos\autoanalysis\AutoAnalyserCTE.java"
+
+REM Run the example:
+java -classpath "..\java;..\..\infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserCTE
+
diff --git a/demos/AutoAnalyser/runCTEAutoAnalyser.sh b/demos/AutoAnalyser/runCTEAutoAnalyser.sh
new file mode 100755
index 0000000..81d113f
--- /dev/null
+++ b/demos/AutoAnalyser/runCTEAutoAnalyser.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Make sure the latest example source file is compiled.
+javac -classpath "../java:../../infodynamics.jar" "../java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java"
+
+# Run the example:
+java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserCTE
+
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java
new file mode 100644
index 0000000..915e068
--- /dev/null
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java
@@ -0,0 +1,397 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2015, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.demos.autoanalysis;
+
+import infodynamics.measures.continuous.ConditionalMutualInfoMultiVariateCommon;
+import infodynamics.measures.continuous.ConditionalTransferEntropyCalculator;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.continuous.gaussian.ConditionalTransferEntropyCalculatorGaussian;
+import infodynamics.measures.continuous.kraskov.ConditionalMutualInfoCalculatorMultiVariateKraskov;
+import infodynamics.measures.continuous.kraskov.ConditionalTransferEntropyCalculatorKraskov;
+import infodynamics.measures.discrete.ConditionalTransferEntropyCalculatorDiscrete;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
+import infodynamics.utils.MatrixUtils;
+
+import javax.swing.JOptionPane;
+import javax.swing.event.DocumentListener;
+
+import java.awt.event.ActionListener;
+import java.awt.event.MouseListener;
+import java.util.Vector;
+
+/**
+ * This class provides a GUI to build a simple conditional transfer entropy calculation,
+ *  and supply the code to execute it.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class AutoAnalyserCTE extends AutoAnalyser
+	implements ActionListener, DocumentListener, MouseListener {
+
+	/**
+	 * Need serialVersionUID to be serializable
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	protected static final String DISCRETE_PROPNAME_K = "k_HISTORY";
+
+	// Property names for specific continuous calculators:
+	protected String[] gaussianProperties;
+	protected String[] gaussianPropertiesFieldNames;
+	protected String[] gaussianPropertyDescriptions;
+	protected String[] kraskovProperties;
+	protected String[] kraskovPropertiesFieldNames;
+	protected String[] kraskovPropertyDescriptions;
+
+	/**
+	 * Constructor to initialise the GUI for Conditional TE
+	 */
+	protected void makeSpecificInitialisations() {
+		
+		numVariables = 3;
+		variableColNumLabels = new String[] {"Source", "Destination", "Conditional"};
+		useAllCombosCheckBox = true;
+		useStatSigCheckBox = true;
+		wordForCombinations = "pairs";
+		variableRelationshipFormatString = "col_%d -> col_%d | col_%d";
+		disableVariableColTextFieldsForAllCombos = new boolean[] {true, true, false};
+		indentsForAllCombos = 2;
+		
+		// Set up the properties for CTE:
+		measureAcronym = "CTE";
+		appletTitle = "JIDT Conditional TE Auto-Analyser"; 
+		
+		calcTypes = new String[] {
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_KRASKOV};
+				// No kernel calculator defined for CMI (yet, and unlikely to happen)
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats"};
+		
+		// Discrete:
+		discreteClass = ConditionalTransferEntropyCalculatorDiscrete.class;
+		discreteProperties = new String[] {
+				DISCRETE_PROPNAME_BASE,
+				DISCRETE_PROPNAME_K
+		};
+		discretePropertyDefaultValues = new String[] {
+				"2",
+				"1"
+		};
+		discretePropertyDescriptions = new String[] {
+				"Number of discrete states available for each variable (i.e. 2 for binary).
      " +
+				"Can be set individually for each variable -- see code.",
+				"Destination history embedding length (k_HISTORY)",
+		};
+		
+		// Continuous:
+		abstractContinuousClass = ConditionalTransferEntropyCalculator.class;
+		// Common properties for all continuous calcs:
+		commonContPropertyNames = new String[] {
+				ConditionalTransferEntropyCalculator.K_PROP_NAME,
+				ConditionalTransferEntropyCalculator.K_TAU_PROP_NAME,
+				ConditionalTransferEntropyCalculator.L_PROP_NAME,
+				ConditionalTransferEntropyCalculator.L_TAU_PROP_NAME,
+				ConditionalTransferEntropyCalculator.DELAY_PROP_NAME,
+				ConditionalTransferEntropyCalculator.COND_EMBED_LENGTHS_PROP_NAME,
+				ConditionalTransferEntropyCalculator.COND_EMBED_DELAYS_PROP_NAME,
+				ConditionalTransferEntropyCalculator.COND_DELAYS_PROP_NAME
+		};
+		commonContPropertiesFieldNames = new String[] {
+				"K_PROP_NAME",
+				"K_TAU_PROP_NAME",
+				"L_PROP_NAME",
+				"L_TAU_PROP_NAME",
+				"DELAY_PROP_NAME",
+				"COND_EMBED_LENGTHS_PROP_NAME",
+				"COND_EMBED_DELAYS_PROP_NAME",
+				"COND_DELAYS_PROP_NAME"
+		};
+		commonContPropertyDescriptions = new String[] {
+				"Destination history embedding length (k_HISTORY)",
+				"Destination history embedding delay (k_TAU)",
+				"Source history embedding length (l_HISTORY)",
+				"Source history embeding delay (l_TAU)",
+				"Delay from source to destination (in time steps)",
+				"Conditional history embedding length",
+				"Conditional history embeding delay",
+				"Delay from conditional to destination (in time steps)"
+		};
+		// Gaussian properties:
+		gaussianProperties = new String[] {
+		};
+		gaussianPropertiesFieldNames = new String[] {
+		};
+		gaussianPropertyDescriptions = new String[] {
+		};
+		// KSG (Kraskov):
+		kraskovProperties = new String[] {
+				ConditionalMutualInfoMultiVariateCommon.PROP_NORMALISE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_K,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE,
+				ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS,
+				ConditionalTransferEntropyCalculatorKraskov.PROP_KRASKOV_ALG_NUM
+		};
+		kraskovPropertiesFieldNames = new String[] {
+				"ConditionalMutualInfoMultiVariateCommon.PROP_NORMALISE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_K",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE",
+				"ConditionalMutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS",
+				"PROP_KRASKOV_ALG_NUM"
+		};
+		kraskovPropertyDescriptions = new String[] {
+				"(boolean) whether to normalise 
      each incoming time-series to mean 0, standard deviation 1, or not (recommended)",
+				"Number of k nearest neighbours to use 
      in the full joint kernel space in the KSG algorithm",
+				"Standard deviation for an amount 
      of random Gaussian noise to add to each variable, " +
+						"to avoid having neighbourhoods with artificially large counts. 
      " +
+						"(\"false\" may be used to indicate \"0\".). The amount is added in after any normalisation.",
+				"Dynamic correlation exclusion time or 
      Theiler window (see Kantz and Schreiber); " +
+						"0 (default) means no dynamic exclusion window",
+				"
      Norm type to use in KSG algorithm between the points in each marginal space. 
      Options are: " +
+						"\"MAX_NORM\" (default), otherwise \"EUCLIDEAN\" or \"EUCLIDEAN_SQUARED\" (both equivalent here)",
+				"Number of parallel threads to use 
      in computation: an integer > 0 or \"USE_ALL\" " +
+						"(default, to indicate to use all available processors)",
+				"Which KSG algorithm to use (1 or 2)",
+		};
+		
+	}
+
+	@Override
+	protected void fillOutAllCombinations(Vector variableCombinations) throws Exception {
+		// All combinations here means all pairs of sources and destinations,
+		//  with the conditional fixed.
+		int conditional = Integer.parseInt(variableColTextFields[2].getText());
+		if (conditional >= dataColumns) {
+			throw new Exception(String.format("%s column must be between 0 and %d for this data set",
+					variableColNumLabels[2], dataColumns-1));
+		}
+		for (int s = 0; s < dataColumns; s++) {
+			for (int d = 0; d < dataColumns; d++) {
+				variableCombinations.add(new int[] {s, d, conditional});
+			}
+		}
+	}
+
+	@Override
+	protected String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		// Set up loops in the code:
+		int conditional = Integer.parseInt(variableColTextFields[2].getText());
+		// 1. Java code
+		javaCode.append("    \n");
+		javaCode.append("    int c = " + conditional + ";\n");
+		javaCode.append("    // Compute for all source-destination pairs:\n");
+		javaCode.append("    for (int s = 0; s < " + dataColumns +
+					"; s++) {\n");
+		javaCode.append("        for (int d = 0; d < " + dataColumns +
+				"; d++) {\n");
+		String javaPrefix = "            ";
+		javaCode.append(javaPrefix + "// For each source-dest pair (given conditional):\n");
+		javaCode.append(javaPrefix + "if ((s == d) || (s == c) || (d == c)) {\n");
+		javaCode.append(javaPrefix + "    continue;\n");
+		javaCode.append(javaPrefix + "}\n");
+		// 2. Python code
+		pythonCode.append("\n");
+		pythonCode.append("c = " + conditional + "\n");
+		pythonCode.append("# Compute for all pairs:\n");
+		pythonCode.append("for s in range(" + dataColumns + "):\n");
+		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "\t\t";
+		pythonCode.append(pythonPrefix+ "# For each source-dest pair (given conditional):\n");
+		pythonCode.append(pythonPrefix + "if ((s == d) or (s == c) or (d == c)):\n");
+		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		// 3. Matlab code
+		matlabCode.append("\n");
+		matlabCode.append("c = " + (conditional+1) + ";\n");
+		matlabCode.append("% Compute for all pairs:\n");
+		matlabCode.append("for s = 1:" + dataColumns + "\n");
+		matlabCode.append("\tfor d = 1:" + dataColumns + "\n");
+		String matlabPrefix = "\t\t";
+		matlabCode.append(matlabPrefix + "% For each source-dest pair (given conditional):\n");
+		matlabCode.append(matlabPrefix + "if ((s == d) || (s == c) || (d == c))\n");
+		matlabCode.append(matlabPrefix + "\tcontinue;\n");
+		matlabCode.append(matlabPrefix + "end\n");
+		
+		// Return the variables to index each column:
+		return new String[] {"s", "d", "c"}; 
+	}
+
+	@Override
+	protected void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		
+		// 1. Java code
+		javaCode.append("        }\n");
+		javaCode.append("    }\n");
+		// 2. Python code
+		// Nothing to do
+		// 3. Matlab code
+		matlabCode.append("\tend\n");
+		matlabCode.append("end\n");
+	}
+	
+	@Override
+	protected String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers) {
+		// We format the source and target variable numbers into the
+		//  return string here:
+		return String.format(formatStr,
+				columnNumbers[0], columnNumbers[1], columnNumbers[2]);
+	}
+	
+	@Override
+	protected boolean skipColumnCombo(int[] columnCombo) {
+		if ((columnCombo[0] == columnCombo[1]) ||
+				(columnCombo[0] == columnCombo[2]) ||
+				(columnCombo[1] == columnCombo[2])) {
+			// Two columns are the same here,
+			//  so don't compute the conditional TE
+			return true;
+		}
+		return false;
+	}
+	
+	@Override
+	protected void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception {
+		
+		String selectedCalcType = (String)
+			calcTypeComboBox.getSelectedItem();
+		
+		int sourceColumn = columnCombo[0];
+		int destColumn = columnCombo[1];
+		int condColumn = columnCombo[2];
+		
+		// Set observations
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			ConditionalTransferEntropyCalculatorDiscrete cteCalc =
+					(ConditionalTransferEntropyCalculatorDiscrete) calcDiscrete;
+			cteCalc.addObservations(
+					MatrixUtils.selectColumn(dataDiscrete, sourceColumn),
+					MatrixUtils.selectColumn(dataDiscrete, destColumn),
+					MatrixUtils.selectColumn(dataDiscrete, condColumn));
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+			ConditionalTransferEntropyCalculatorDiscrete cteCalc =
+					(ConditionalTransferEntropyCalculatorDiscrete) calcDiscrete;
+			cteCalc.addObservations(
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, sourceColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))),
+					MatrixUtils.discretise(
+						MatrixUtils.selectColumn(data, destColumn),
+						Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))),
+					MatrixUtils.discretise(
+							MatrixUtils.selectColumn(data, condColumn),
+							Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))));
+		} else {
+			ConditionalTransferEntropyCalculator cteCalc =
+					(ConditionalTransferEntropyCalculator) calcContinuous;
+			cteCalc.setObservations(
+					MatrixUtils.selectColumn(data, sourceColumn),
+					MatrixUtils.selectColumn(data, destColumn),
+					MatrixUtils.selectColumn(data, condColumn));
+		}
+	}
+	
+	protected CalcProperties assignCalcProperties(String selectedCalcType)
+			throws Exception {
+		// Let the super class handle discrete calculators
+		CalcProperties calcProperties = super.assignCalcProperties(selectedCalcType);
+		if (calcProperties == null) {
+			// We need to assign properties for a continuous calculator
+			calcProperties = new CalcProperties();
+			calcProperties.calc = assignCalcObjectContinuous(selectedCalcType);
+			calcProperties.calcClass = calcProperties.calc.getClass();
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+				calcProperties.classSpecificPropertyNames = gaussianProperties;
+				calcProperties.classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
+			} else if (selectedCalcType.startsWith(CALC_TYPE_KRASKOV)) {
+				// The if statement will work for both MI Kraskov calculators
+				calcProperties.classSpecificPropertyNames = kraskovProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kraskovPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kraskovPropertyDescriptions;
+			} else {
+				calcProperties = null;
+				throw new Exception("No recognised calculator selected: " +
+						selectedCalcType);
+			}
+		}
+		return calcProperties;
+	}
+
+	/**
+	 * Method to assign and initialise our continuous calculator class
+	 */
+	@Override
+	protected ConditionalTransferEntropyCalculator assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+			return new ConditionalTransferEntropyCalculatorGaussian();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV)) {
+			return new ConditionalTransferEntropyCalculatorKraskov();
+		} else {
+			throw new Exception("No recognised continuous calculator selected: " +
+					selectedCalcType);
+		}
+
+	}
+
+	/**
+	 * Method to assign and initialise our discrete calculator class
+	 */
+	protected DiscreteCalcAndArguments assignCalcObjectDiscrete() throws Exception {
+		int base, k;
+		try {
+			String basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+			base = Integer.parseInt(basePropValueStr);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot find a value for property " + DISCRETE_PROPNAME_BASE);
+			return null;
+		}
+		try {
+			String kPropValueStr = propertyValues.get(DISCRETE_PROPNAME_K);
+			k = Integer.parseInt(kPropValueStr);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot read a value for property " + DISCRETE_PROPNAME_K);
+			return null;
+		}
+
+		return new DiscreteCalcAndArguments(
+				// Initialise always with one other info contributor for now
+				new ConditionalTransferEntropyCalculatorDiscrete(base, k, 1),
+				base,
+				base + ", " + k + ", " + 1);
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		new AutoAnalyserCTE();
+	}
+}

From 19f2c5abbce33786e8628beddf79e205ccb9908a Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 28 Aug 2017 00:36:24 +1000
Subject: [PATCH 56/78] Performance tweaks for TE Discrete, which should make a
 difference when the state space is large in comparison to the number of
 samples.

---
 .../TransferEntropyCalculatorDiscrete.java    | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
index e3d04d0..98ffbb0 100755
--- a/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
+++ b/java/source/infodynamics/measures/discrete/TransferEntropyCalculatorDiscrete.java
@@ -1107,14 +1107,17 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 		for (int pastVal = 0; pastVal < base_power_k; pastVal++) {
 			// compute p(past)
 			// double p_past = (double) pastCount[pastVal] / (double) observations;
+			if (pastCount[pastVal] == 0) {
+				continue;
+			}
 			for (int destVal = 0; destVal < base; destVal++) {
 				// compute p(dest,past)
 				// double p_dest_past = (double) destPastCount[destVal][pastVal] / (double) observations;
+				if (nextPastCount[destVal][pastVal] == 0) {
+					continue;
+				}
+				double denom = (double) nextPastCount[destVal][pastVal] / (double) pastCount[pastVal];
 				for (int sourceVal = 0; sourceVal < base_power_l; sourceVal++) {
-					// compute p(source,dest,past)
-					double p_source_dest_past = (double) sourceNextPastCount[sourceVal][destVal][pastVal] / (double) observations;
-					// compute p(source,past)
-					// double p_source_past = (double) sourcePastCount[sourceVal][pastVal] / (double) observations;
 					// Compute TE contribution:
 					if (sourceNextPastCount[sourceVal][destVal][pastVal] != 0) {
 						/* Double check: should never happen
@@ -1125,9 +1128,13 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 						}
 						*/
 						
+						// compute p(source,dest,past)
+						double p_source_dest_past = (double) sourceNextPastCount[sourceVal][destVal][pastVal] / (double) observations;
+						// compute p(source,past)
+						// double p_source_past = (double) sourcePastCount[sourceVal][pastVal] / (double) observations;
 						double logTerm = ((double) sourceNextPastCount[sourceVal][destVal][pastVal] / (double) sourcePastCount[sourceVal][pastVal]) /
-						 	((double) nextPastCount[destVal][pastVal] / (double) pastCount[pastVal]);
-						double localValue = Math.log(logTerm) / log_2;
+						 	(denom);
+						double localValue = Math.log(logTerm); // We'll / log_2 later, to save a floating pt op;
 						teCont = p_source_dest_past * localValue;
 						if (localValue > max) {
 							max = localValue;
@@ -1145,6 +1152,11 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 			}
 		}
 		
+		te /= log_2;
+		max /= log_2;
+		min /= log_2;
+		meanSqLocals /= (log_2 * log_2);
+		
 		average = te;
 		std = Math.sqrt(meanSqLocals - average * average);
 		estimateComputed = true;
@@ -1389,6 +1401,7 @@ public class TransferEntropyCalculatorDiscrete extends ContextOfPastMeasureCalcu
 							(double) sourcePastCount[thisSourceVal][thisPastVal]) /
 			 		((double) nextPastCount[destVal][thisPastVal] / (double) pastCount[thisPastVal]);
 			localTE[t] = Math.log(logTerm) / log_2;
+			average += localTE[t];
 			if (localTE[t] > max) {
 				max = localTE[t];
 			} else if (localTE[t] < min) {

From 5dcb89bed719bdcc47a3dd000f5df136b3a7b5f8 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 28 Aug 2017 09:45:05 +1000
Subject: [PATCH 57/78] Adding AutoAnalyser for Active information storage,
 including run scripts in demo folder

---
 demos/AutoAnalyser/runAISAutoAnalyser.bat     |   8 +
 demos/AutoAnalyser/runAISAutoAnalyser.sh      |   8 +
 .../demos/autoanalysis/AutoAnalyserAIS.java   | 370 ++++++++++++++++++
 3 files changed, 386 insertions(+)
 create mode 100755 demos/AutoAnalyser/runAISAutoAnalyser.bat
 create mode 100755 demos/AutoAnalyser/runAISAutoAnalyser.sh
 create mode 100644 demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java

diff --git a/demos/AutoAnalyser/runAISAutoAnalyser.bat b/demos/AutoAnalyser/runAISAutoAnalyser.bat
new file mode 100755
index 0000000..7794f3c
--- /dev/null
+++ b/demos/AutoAnalyser/runAISAutoAnalyser.bat
@@ -0,0 +1,8 @@
+@ECHO OFF
+
+REM Make sure the latest example source file is compiled.
+javac -classpath "..\java;..\..\infodynamics.jar" "..\java\infodynamics\demos\autoanalysis\AutoAnalyserAIS.java"
+
+REM Run the example:
+java -classpath "..\java;..\..\infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserAIS
+
diff --git a/demos/AutoAnalyser/runAISAutoAnalyser.sh b/demos/AutoAnalyser/runAISAutoAnalyser.sh
new file mode 100755
index 0000000..0439262
--- /dev/null
+++ b/demos/AutoAnalyser/runAISAutoAnalyser.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Make sure the latest example source file is compiled.
+javac -classpath "../java:../../infodynamics.jar" "../java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java"
+
+# Run the example:
+java -classpath "../java:../../infodynamics.jar" infodynamics.demos.autoanalysis.AutoAnalyserAIS
+
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java
new file mode 100644
index 0000000..46879ae
--- /dev/null
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java
@@ -0,0 +1,370 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2015, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see .
+ */
+
+package infodynamics.demos.autoanalysis;
+
+import infodynamics.measures.continuous.ActiveInfoStorageCalculator;
+import infodynamics.measures.continuous.InfoMeasureCalculatorContinuous;
+import infodynamics.measures.continuous.gaussian.ActiveInfoStorageCalculatorGaussian;
+import infodynamics.measures.continuous.kernel.ActiveInfoStorageCalculatorKernel;
+import infodynamics.measures.continuous.kernel.MutualInfoCalculatorMultiVariateKernel;
+import infodynamics.measures.continuous.kraskov.ActiveInfoStorageCalculatorKraskov;
+import infodynamics.measures.continuous.kraskov.MutualInfoCalculatorMultiVariateKraskov;
+import infodynamics.measures.discrete.ActiveInformationCalculatorDiscrete;
+import infodynamics.measures.discrete.InfoMeasureCalculatorDiscrete;
+import infodynamics.utils.MatrixUtils;
+
+import java.util.Vector;
+
+import javax.swing.JOptionPane;
+
+/**
+ * This class provides a GUI to build a simple calculation
+ *  of active information storage,
+ *  and supply the code to execute it.
+ * 
+ * 
+ * @author Joseph Lizier
+ *
+ */
+public class AutoAnalyserAIS extends AutoAnalyser {
+
+	/**
+	 * Need serialVersionUID to be serializable
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	protected static final String DISCRETE_PROPNAME_K = "k_HISTORY";
+
+	// Property names for specific continuous calculators:
+	protected String[] gaussianProperties;
+	protected String[] gaussianPropertiesFieldNames;
+	protected String[] gaussianPropertyDescriptions;
+	protected String[] kernelProperties;
+	protected String[] kernelPropertiesFieldNames;
+	protected String[] kernelPropertyDescriptions;
+	protected String[] kraskovProperties;
+	protected String[] kraskovPropertiesFieldNames;
+	protected String[] kraskovPropertyDescriptions;
+
+	/**
+	 * Constructor to initialise the GUI for a channel calculator
+	 */
+	protected void makeSpecificInitialisations() {
+		numVariables = 1;
+		variableColNumLabels = new String[] {"Variable"};
+		useAllCombosCheckBox = true;
+		useStatSigCheckBox = true;
+		wordForCombinations = "variables";
+		variableRelationshipFormatString = "col_%d";
+		disableVariableColTextFieldsForAllCombos = new boolean[] {true};
+		indentsForAllCombos = 1;
+		
+		// Set up the properties for Entropy:
+		measureAcronym = "AIS";
+		appletTitle = "JIDT Active Information Storage Auto-Analyser";
+		
+		calcTypes = new String[] {
+				CALC_TYPE_DISCRETE, CALC_TYPE_BINNED, CALC_TYPE_GAUSSIAN,
+				CALC_TYPE_KRASKOV, CALC_TYPE_KERNEL};
+		unitsForEachCalc = new String[] {"bits", "bits", "nats", "nats", "bits"};
+		
+		// Discrete:
+		discreteClass = ActiveInformationCalculatorDiscrete.class;
+		discreteProperties = new String[] {
+				DISCRETE_PROPNAME_BASE,
+				DISCRETE_PROPNAME_K
+		};
+		discretePropertyDefaultValues = new String[] {
+				"2",
+				"1"
+		};
+		discretePropertyDescriptions = new String[] {
+				"Number of discrete states available for each variable (i.e. 2 for binary)",
+				"History embedding length (k_HISTORY)"
+		};
+		
+		// Continuous:
+		abstractContinuousClass = ActiveInfoStorageCalculator.class;
+		// Common properties for all continuous calcs:
+		commonContPropertyNames = new String[] {
+				ActiveInfoStorageCalculator.K_PROP_NAME,
+				ActiveInfoStorageCalculator.TAU_PROP_NAME
+		};
+		commonContPropertiesFieldNames = new String[] {
+				"K_PROP_NAME",
+				"TAU_PROP_NAME"
+		};
+		commonContPropertyDescriptions = new String[] {
+				"History embedding length (k_HISTORY)",
+				"History embedding delay (k_TAU)"
+		};
+		// Gaussian properties:
+		gaussianProperties = new String[] {
+		};
+		gaussianPropertiesFieldNames = new String[] {
+		};
+		gaussianPropertyDescriptions = new String[] {
+		};
+		// Kernel:
+		kernelProperties = new String[] {
+				MutualInfoCalculatorMultiVariateKernel.KERNEL_WIDTH_PROP_NAME,
+				MutualInfoCalculatorMultiVariateKernel.DYN_CORR_EXCL_TIME_NAME,
+				MutualInfoCalculatorMultiVariateKernel.NORMALISE_PROP_NAME,			
+		};
+		kernelPropertiesFieldNames = new String[] {
+				"KERNEL_WIDTH_PROP_NAME",
+				"DYN_CORR_EXCL_TIME_NAME",
+				"NORMALISE_PROP_NAME"			
+		};
+		kernelPropertyDescriptions = new String[] {
+				"Kernel width to be used in the calculation. 
      If the property " +
+						MutualInfoCalculatorMultiVariateKernel.NORMALISE_PROP_NAME +
+						" is set, then this is a number of standard deviations; " +
+						"otherwise it is an absolute value.",
+				"Dynamic correlation exclusion time or 
      Theiler window (see Kantz and Schreiber); " +
+						"0 (default) means no dynamic exclusion window",
+				"(boolean) whether to normalise 
      each incoming time-series to mean 0, standard deviation 1, or not  (recommended)",
+		};
+		// KSG (Kraskov):
+		kraskovProperties = new String[] {
+				MutualInfoCalculatorMultiVariateKraskov.PROP_NORMALISE,
+				MutualInfoCalculatorMultiVariateKraskov.PROP_K,
+				MutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE,
+				MutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME,
+				MutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE,
+				MutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS,
+				ActiveInfoStorageCalculatorKraskov.PROP_AUTO_EMBED_METHOD,
+				ActiveInfoStorageCalculatorKraskov.PROP_K_SEARCH_MAX,
+				ActiveInfoStorageCalculatorKraskov.PROP_TAU_SEARCH_MAX,
+				ActiveInfoStorageCalculatorKraskov.PROP_RAGWITZ_NUM_NNS,
+		};
+		kraskovPropertiesFieldNames = new String[] {
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_NORMALISE",
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_K",
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_ADD_NOISE",
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_DYN_CORR_EXCL_TIME",
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_NORM_TYPE",
+				"MutualInfoCalculatorMultiVariateKraskov.PROP_NUM_THREADS",
+				"PROP_AUTO_EMBED_METHOD",
+				"PROP_K_SEARCH_MAX",
+				"PROP_TAU_SEARCH_MAX",
+				"PROP_RAGWITZ_NUM_NNS"				
+		};
+		kraskovPropertyDescriptions = new String[] {
+				"(boolean) whether to normalise 
      each incoming time-series to mean 0, standard deviation 1, or not (recommended)",
+				"Number of k nearest neighbours to use 
      in the full joint kernel space in the KSG algorithm",
+				"Standard deviation for an amount 
      of random Gaussian noise to add to each variable, " +
+						"to avoid having neighbourhoods with artificially large counts. 
      " +
+						"(\"false\" may be used to indicate \"0\".). The amount is added in after any normalisation.",
+				"Dynamic correlation exclusion time or 
      Theiler window (see Kantz and Schreiber); " +
+						"0 (default) means no dynamic exclusion window",
+				"
      Norm type to use in KSG algorithm between the points in each marginal space. 
      Options are: " +
+						"\"MAX_NORM\" (default), otherwise \"EUCLIDEAN\" or \"EUCLIDEAN_SQUARED\" (both equivalent here)",
+				"Number of parallel threads to use 
      in computation: an integer > 0 or \"USE_ALL\" " +
+						"(default, to indicate to use all available processors)",
+				"Method to automatically determine embedding length (k_HISTORY)
       and delay (k_TAU) for " +
+						"the samples. Default is \"" + ActiveInfoStorageCalculatorKraskov.AUTO_EMBED_METHOD_NONE +
+						"\" meaning values are set manually; other values include: 
        -- \"" + ActiveInfoStorageCalculatorKraskov.AUTO_EMBED_METHOD_RAGWITZ +
+						"\" for use of the Ragwitz criteria for both source and destination (searching up to \"" + ActiveInfoStorageCalculatorKraskov.PROP_K_SEARCH_MAX +
+						"\" and \"" + ActiveInfoStorageCalculatorKraskov.PROP_TAU_SEARCH_MAX + "\"); 
        -- \"" + ActiveInfoStorageCalculatorKraskov.AUTO_EMBED_METHOD_MAX_CORR_AIS +
+						"\" for maximising the (bias corrected) Active Info Storage (searching up to \"" + ActiveInfoStorageCalculatorKraskov.PROP_K_SEARCH_MAX +
+						"\" and \"" + ActiveInfoStorageCalculatorKraskov.PROP_TAU_SEARCH_MAX + "\"); 
      Use of values other than \"" + ActiveInfoStorageCalculatorKraskov.AUTO_EMBED_METHOD_NONE +
+						"\" leads to any previous settings for embedding lengths and delays to be overwritten after observations are supplied",
+				"Max. embedding length to search to 
      if auto embedding (as determined by " + ActiveInfoStorageCalculatorKraskov.PROP_AUTO_EMBED_METHOD + ")",
+				"Max. embedding delay to search to 
      if auto embedding (as determined by " + ActiveInfoStorageCalculatorKraskov.PROP_AUTO_EMBED_METHOD + ")",
+				"Number of k nearest neighbours for 
      Ragwitz auto embedding (if used; defaults to match property \"k\")"
+		};
+	}
+
+	@Override
+	protected void fillOutAllCombinations(Vector variableCombinations) {
+		// All combinations here means all variables
+		for (int s = 0; s < dataColumns; s++) {
+			variableCombinations.add(new int[] {s});
+		}
+	}
+
+	@Override
+	protected String[] setUpLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		// Set up loops in the code:
+		// 1. Java code
+		javaCode.append("    \n");
+		javaCode.append("    // Compute for all variables:\n");
+		javaCode.append("    for (int v = 0; v < " + dataColumns +
+					"; v++) {\n");
+		String javaPrefix = "        ";
+		javaCode.append(javaPrefix + "// For each variable:\n");
+		// 2. Python code
+		pythonCode.append("\n");
+		pythonCode.append("# Compute for all variables:\n");
+		pythonCode.append("for v in range(" + dataColumns + "):\n");
+		String pythonPrefix = "\t";
+		pythonCode.append(pythonPrefix+ "# For each variable:\n");
+		// 3. Matlab code
+		matlabCode.append("\n");
+		matlabCode.append("% Compute for all variables:\n");
+		matlabCode.append("for v = 1:" + dataColumns + "\n");
+		String matlabPrefix = "\t";
+		matlabCode.append(matlabPrefix + "% For each variable:\n");
+		
+		// Return the variables to index each column:
+		return new String[] {"v"}; 
+	}
+
+	@Override
+	protected void finaliseLoopsForAllCombos(StringBuffer javaCode,
+			StringBuffer pythonCode, StringBuffer matlabCode) {
+		
+		// 1. Java code
+		javaCode.append("    }\n");
+		// 2. Python code
+		// Nothing to do
+		// 3. Matlab code
+		matlabCode.append("end\n");
+	}
+	
+	@Override
+	protected String formatStringWithColumnNumbers(String formatStr, int[] columnNumbers) {
+		// We format the variable number into the
+		//  return string here:
+		return String.format(formatStr,
+				columnNumbers[0]);
+	}
+	
+	@Override
+	protected boolean skipColumnCombo(int[] columnCombo) {
+		// No reason to skip any columns here
+		return false;
+	}
+	
+	@Override
+	protected void setObservations(InfoMeasureCalculatorDiscrete calcDiscrete,
+			InfoMeasureCalculatorContinuous calcContinuous,
+			int[] columnCombo) throws Exception {
+		
+		String selectedCalcType = (String)
+			calcTypeComboBox.getSelectedItem();
+		
+		int variableColumn = columnCombo[0];
+		
+		// Set observations
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
+			ActiveInformationCalculatorDiscrete calc = (ActiveInformationCalculatorDiscrete) calcDiscrete;
+			calc.addObservations(
+					MatrixUtils.selectColumn(dataDiscrete, variableColumn));
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
+			ActiveInformationCalculatorDiscrete calc = (ActiveInformationCalculatorDiscrete) calcDiscrete;
+			calc.addObservations(
+					MatrixUtils.discretise(
+							MatrixUtils.selectColumn(data, variableColumn),
+							// Should be no parse error on the alphabet size by now
+							Integer.parseInt(propertyValues.get(DISCRETE_PROPNAME_BASE))));
+		} else {
+			ActiveInfoStorageCalculator calc = (ActiveInfoStorageCalculator) calcContinuous;
+			calc.setObservations(
+					MatrixUtils.selectColumn(data, variableColumn));
+		}
+	}
+	
+	protected CalcProperties assignCalcProperties(String selectedCalcType)
+			throws Exception {
+		// Let the super class handle discrete calculators
+		CalcProperties calcProperties = super.assignCalcProperties(selectedCalcType);
+		if (calcProperties == null) {
+			// We need to assign properties for a continuous calculator
+			calcProperties = new CalcProperties();
+			calcProperties.calc = assignCalcObjectContinuous(selectedCalcType);
+			calcProperties.calcClass = calcProperties.calc.getClass();
+			if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+				calcProperties.classSpecificPropertyNames = gaussianProperties;
+				calcProperties.classSpecificPropertiesFieldNames = gaussianPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = gaussianPropertyDescriptions;
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV)) {
+				calcProperties.classSpecificPropertyNames = kraskovProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kraskovPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kraskovPropertyDescriptions;
+			} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
+				calcProperties.classSpecificPropertyNames = kernelProperties;
+				calcProperties.classSpecificPropertiesFieldNames = kernelPropertiesFieldNames;
+				calcProperties.classSpecificPropertyDescriptions = kernelPropertyDescriptions;
+			} else {
+				calcProperties = null;
+				throw new Exception("No recognised calculator selected: " +
+						selectedCalcType);
+			}
+		}
+		return calcProperties;
+	}
+
+	/**
+	 * Method to assign and initialise our continuous calculator class
+	 */
+	@Override
+	protected ActiveInfoStorageCalculator assignCalcObjectContinuous(String selectedCalcType) throws Exception {
+		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_GAUSSIAN)) {
+			return new ActiveInfoStorageCalculatorGaussian();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KRASKOV)) {
+			return new ActiveInfoStorageCalculatorKraskov();
+		} else if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_KERNEL)) {
+			return new ActiveInfoStorageCalculatorKernel();
+		} else {
+			throw new Exception("No recognised continuous calculator selected: " +
+					selectedCalcType);
+		}
+
+	}
+
+	/**
+	 * Method to assign and initialise our discrete calculator class
+	 */
+	protected DiscreteCalcAndArguments assignCalcObjectDiscrete() throws Exception {
+		int base, k;
+		try {
+			String basePropValueStr = propertyValues.get(DISCRETE_PROPNAME_BASE);
+			base = Integer.parseInt(basePropValueStr);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot read a value for property " + DISCRETE_PROPNAME_BASE);
+			return null;
+		}
+		try {
+			String kPropValueStr = propertyValues.get(DISCRETE_PROPNAME_K);
+			k = Integer.parseInt(kPropValueStr);
+		} catch (Exception ex) {
+			JOptionPane.showMessageDialog(this,
+					ex.getMessage());
+			resultsLabel.setText("Cannot read a value for property " + DISCRETE_PROPNAME_K);
+			return null;
+		}
+		
+		return new DiscreteCalcAndArguments(
+				new ActiveInformationCalculatorDiscrete(base, k),
+				base,
+				base + ", " + k);
+	}
+
+	/**
+	 * @param args
+	 */
+	public static void main(String[] args) {
+		new AutoAnalyserAIS();
+	}
+}

From 5196c204a94bbe11f006fb25b3560035a2a77e61 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Mon, 28 Aug 2017 13:31:38 +1000
Subject: [PATCH 58/78] Fixed Matlab paths to only have one variable to change,
 and fixed Python spacing in AutoAnalyser

---
 .../infodynamics/demos/autoanalysis/AutoAnalyser.java    | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 8157e16..76793e7 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -753,9 +753,10 @@ public abstract class AutoAnalyser extends JFrame
 		// 3. Matlab:
 		StringBuffer matlabCode = new StringBuffer();
 		matlabCode.append("% Add JIDT jar library to the path\n");
-		matlabCode.append("javaaddpath('../../infodynamics.jar');\n");
+		matlabCode.append("jidtPath = '../../';\n");
+		matlabCode.append("javaaddpath([jidtPath, '/infodynamics.jar']);\n");
 		matlabCode.append("% Add utilities to the path\n");
-		matlabCode.append("addpath('../octave');\n\n");
+		matlabCode.append("addpath([jidtPath, '/demos/octave']);\n\n");
 		
 		try{
 			// Create both discrete and continuous calculators to make
@@ -1026,7 +1027,7 @@ public abstract class AutoAnalyser extends JFrame
 				//  each variable, to be used in later formatting.
 				for (int i = 0; i < indentsForAllCombos; i++) {
 					javaPrefix += "    ";
-					pythonPrefix += "\t";
+					pythonPrefix += "    ";
 					matlabPrefix += "\t";
 				}
 				
@@ -1161,7 +1162,7 @@ public abstract class AutoAnalyser extends JFrame
 					extraFormatTerms + "result" + statSigFormatTerms + ");\n");
 			// 2. Python
 			pythonCode.append("\n" + pythonPrefix + "print(\"" + resultsPrefixString +
-					"%.4f " + units + resultsSuffixString + "\" %\n\t" + pythonPrefix + "(" +
+					"%.4f " + units + resultsSuffixString + "\" %\n    " + pythonPrefix + "(" +
 					extraFormatTerms + "result" + statSigFormatTerms + "))\n");
 			// 3. Matlab
 			matlabCode.append("\n" + matlabPrefix + "fprintf('" + resultsPrefixString +

From b7154fab62b231f0f75d23c7aa8fb4d7c4e3ab73 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 29 Aug 2017 14:51:56 +1000
Subject: [PATCH 59/78] Patched issue where Analytic stat sig checkbox was
 still generating Empirical in the generated code (even though it was
 computing it properly)

---
 .../demos/autoanalysis/AutoAnalyser.java       | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 76793e7..c90ce05 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -1138,13 +1138,23 @@ public abstract class AutoAnalyser extends JFrame
 			String statSigFormatTerms = "";
 			if (statSigCheckBox.isSelected()) {
 				// Compute statistical significance
-				String statSigComment = "6. Compute the (statistical significance via) null distribution (e.g. 100 permutations):\n";
+				String statSigComment = "6. Compute the (statistical significance via) null distribution";
+				String javaReturnClass, numPermutationsToPrint;
+				if (statSigAnalyticCheckBox.isSelected()) {
+					javaReturnClass = "AnalyticMeasurementDistribution";
+					numPermutationsToPrint = ""; // Missing argument will signal analytic calculation
+					statSigComment = statSigComment + " analytically:\n";
+				} else {
+					javaReturnClass = "EmpiricalMeasurementDistribution";
+					numPermutationsToPrint = Integer.toString(numPermutationsToCheck);
+					statSigComment = statSigComment + " empirically (e.g. with 100 permutations):\n";
+				}
 				javaCode.append(javaPrefix + "// " + statSigComment);
-				javaCode.append(javaPrefix + "EmpiricalMeasurementDistribution measDist = calc.computeSignificance(100);\n");
+				javaCode.append(javaPrefix + javaReturnClass + " measDist = calc.computeSignificance(" + numPermutationsToPrint + ");\n");
 				pythonCode.append(pythonPrefix + "# " + statSigComment);
-				pythonCode.append(pythonPrefix + "measDist = calc.computeSignificance(100)\n");
+				pythonCode.append(pythonPrefix + "measDist = calc.computeSignificance(" + numPermutationsToPrint + ")\n");
 				matlabCode.append(matlabPrefix + "% " + statSigComment);
-				matlabCode.append(matlabPrefix + "measDist = calc.computeSignificance(100);\n");
+				matlabCode.append(matlabPrefix + "measDist = calc.computeSignificance(" + numPermutationsToPrint + ");\n");
 				if (statSigAnalyticCheckBox.isSelected()) {
 					resultsSuffixString = " (analytic p(surrogate > measured)=%.5f)";
 					statSigFormatTerms = ", measDist.pValue";

From 1f236dee4990db36e949bd909cd72c32e48a09a0 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 29 Aug 2017 14:55:44 +1000
Subject: [PATCH 60/78] Added AnalyticMeasurementDistribution to the generated
 Java headers for AutoAnalyser

---
 .../java/infodynamics/demos/autoanalysis/AutoAnalyser.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index c90ce05..55d55e5 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -723,7 +723,11 @@ public abstract class AutoAnalyser extends JFrame
 		javaCode.append("package infodynamics.demos.autoanalysis;\n\n");
 		javaCode.append("import infodynamics.utils.ArrayFileReader;\n");
 		if (statSigCheckBox.isSelected()) {
-			javaCode.append("import infodynamics.utils.EmpiricalMeasurementDistribution;\n");
+			if (statSigAnalyticCheckBox.isSelected()) {
+				javaCode.append("import infodynamics.utils.AnalyticMeasurementDistribution;\n");
+			} else {
+				javaCode.append("import infodynamics.utils.EmpiricalMeasurementDistribution;\n");
+			}
 		}
 		javaCode.append("import infodynamics.utils.MatrixUtils;\n\n");
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||

From bba00f07fccc615075670e49335e80467c98ee0b Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 29 Aug 2017 15:08:50 +1000
Subject: [PATCH 61/78] In AutoAnalyser switching to store calculator as its
 actual class rather than superclass, since otherwise you can't access
 features like analytic surrogates without casting to the correct interface.

---
 .../java/infodynamics/demos/autoanalysis/AutoAnalyser.java  | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 55d55e5..78a1e3c 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -958,7 +958,11 @@ public abstract class AutoAnalyser extends JFrame
 			} else {
 				// Construct the calculator:
 				// 1. Java
-				javaCode.append("    " + abstractContinuousClass.getSimpleName() + " calc;\n");
+				// If we wanted to create as superclass:
+				// javaCode.append("    " + abstractContinuousClass.getSimpleName() + " calc;\n");
+				// But that gives problems when we use interfaces such as AnalyticNullDistributionComputer,
+				//  so we'll store it as the instantiated class:
+				javaCode.append("    " + calcContinuous.getClass().getSimpleName() + " calc;\n");
 				javaCode.append(javaConstructorLine);
 				// 2. Python
 				pythonCode.append(pythonPreConstructorLine);

From 49afa343fc1f846312951967740a4d7fe595f658 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Tue, 29 Aug 2017 23:55:10 +0100
Subject: [PATCH 62/78] Removed old variables and functions from KSG mixed,
 added docs.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 72 +++++++++----------
 1 file changed, 33 insertions(+), 39 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index f311140..06a3a1c 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -65,16 +65,27 @@ import java.util.Arrays;
  */
 public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements MutualInfoCalculatorMultiVariateWithDiscrete, Cloneable {
 
-	// Multiplier used in hueristic for determining whether to use a linear search
-	//  for min kth element or a binary search.
-	protected static final double CUTOFF_MULTIPLIER = 1.5;
-	
 	/**
 	 * we compute distances to the kth neighbour
 	 */
 	protected int k = 4;
+	/**
+	 * The set of continuous data observations, retained in case the user wants
+   * to retrieve the local entropy values of these.
+	 * They're held in the order in which they were supplied in the
+	 *  {@link addObservations(double[][], int[])} functions.
+	 */
 	protected double[][] continuousData;
+	/**
+	 * The set of discrete data observations, retained in case the user wants
+   * to retrieve the local entropy values of these.
+	 * They're held in the order in which they were supplied in the
+	 *  {@link addObservations(double[][], int[])} functions.
+	 */
 	protected int[] discreteData;
+  /**
+   * Counts of how many observations belong to each discrete bin.
+   */
 	protected int[] counts;
 	/**
 	 * Number of possible states of the discrete variable
@@ -130,20 +141,28 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
    * The norm type in use (see {@link #PROP_NORM_TYPE})
    */
   protected int normType = EuclideanUtils.NORM_MAX_NORM;
-	
-	protected EuclideanUtils normCalculator;
-	// Storage for the norms from each observation to each other one
-	protected double[][] xNorms;
-	// Keep the norms each time (making reordering very quick)
-	//  (Should only be set to false for testing)
-	public static boolean tryKeepAllPairsNorms = true;
-	public static int MAX_DATA_SIZE_FOR_KEEP_ALL_PAIRS_NORM = 2000;
-	
+  /**
+   * Property name for the number of K nearest neighbours used in
+   * the KSG algorithm in the full joint space (default 4).
+   */
 	public final static String PROP_K = "k";
+  /**
+   * Property name for what type of norm to use between data points
+   *  for each marginal variable -- Options are defined by 
+   *  {@link KdTree#setNormType(String)} and the
+   *  default is {@link EuclideanUtils#NORM_MAX_NORM}.
+   */
 	public final static String PROP_NORM_TYPE = "NORM_TYPE";	
+  /**
+   * Property name for whether to normalise the incoming data to 
+   * mean 0, standard deviation 1 (default true)
+   */
 	public static final String PROP_NORMALISE = "NORMALISE";
+	/**
+	 * Property name for time difference between source and destination (0 by default,
+	 * must be >= 0)
+	 */ 
 	public static final String PROP_TIME_DIFF = "TIME_DIFF";
-	
 	/**
 	 * Track whether we're going to normalise the joint variables individually
 	 */
@@ -167,7 +186,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 
 	public MutualInfoCalculatorMultiVariateWithDiscreteKraskov() {
 		super();
-		normCalculator = new EuclideanUtils(EuclideanUtils.NORM_MAX_NORM);
 	}
 
 	/**
@@ -180,7 +198,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		mi = 0.0;
 		miComputed = false;
     totalObservations = 0;
-		xNorms = null;
 		continuousData = null;
 		means = null;
 		stds = null;
@@ -335,29 +352,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
     finaliseAddObservations();
 	}
 
-
-	/**
-	 * Compute the norms for each marginal time series
-	 *
-	 */
-	protected void computeNorms() {
-		int N = continuousData.length; // number of observations
-		
-		xNorms = new double[N][N];
-		for (int t = 0; t < N; t++) {
-			// Compute the norms from t to all other time points
-			for (int t2 = 0; t2 < N; t2++) {
-				if (t2 == t) {
-					xNorms[t][t2] = Double.POSITIVE_INFINITY;
-					continue;
-				}
-				// Compute norm in the continuous space
-				xNorms[t][t2] = normCalculator.norm(continuousData[t], continuousData[t2]);
-			}
-		}
-	}
-
-
   /**
    * Internal method to ensure that the Kd-tree data structures to represent the
    * observational data have been constructed (should be called prior to attempting

From e11218631eab9dcffdce70faf9b17acaa96a0ffe Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Tue, 29 Aug 2017 23:56:28 +0100
Subject: [PATCH 63/78] Added myself as author of KSG mixed calculator.

---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 06a3a1c..880e18c 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -62,6 +62,8 @@ import java.util.Arrays;
  * 
  * @author Joseph Lizier (email,
  * www)
+ * @author Pedro A.M. Mediano (email,
+ * www)
  */
 public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements MutualInfoCalculatorMultiVariateWithDiscrete, Cloneable {
 

From 626293b84d063096157243b3ae5dba63d9b31649 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 30 Aug 2017 01:19:54 +0100
Subject: [PATCH 64/78] More docs for KSG mixed calc.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 116 +++++++++++++++---
 1 file changed, 98 insertions(+), 18 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 880e18c..edc60ba 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -186,6 +186,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 */
 	protected int timeDiff = 0;
 
+  /**
+   * Construct an instance of the KSG mixed MI calculator
+   */
 	public MutualInfoCalculatorMultiVariateWithDiscreteKraskov() {
 		super();
 	}
@@ -277,11 +280,26 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		throw new RuntimeException("Not implemented yet");
 	}
 
+  /**
+   * Prepare the calculator to receive observations.
+   *
+   * Must be called between {@link #initialise()} and any addObservations
+   * method. Does not need to be called if
+   * {@link setObservations(double[][], int[])} is used.
+   */
 	public void startAddObservations() {
 		vectorOfContinuousObservations = new Vector();
 		vectorOfDiscreteObservations   = new Vector();
 	}
 
+  /**
+   * Prepare observations previously added through any addObservations method
+   * for calculations.
+   *
+   * Must be called between any addObservations method and any compute method.
+   * Does not need to be called if {@link setObservations(double[][], int[])}
+   * is used.
+   */
 	public void finaliseAddObservations() throws Exception {
 		if (vectorOfContinuousObservations.size() < 1) {
 			throw new Exception("Cannot compute MI with a null set of data");
@@ -408,15 +426,51 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		return miSurrogateCalculator.computeAverageLocalOfObservations();
 	}
 
+  /**
+   * Calculate the average MI in nats.
+   */
   public double computeAverageLocalOfObservations() throws Exception {
     return computeFromObservations(false)[0];
   }
 
+  /**
+   * 
      Computes the local values of the MI,
+   *  for each valid observation in the previously supplied observations
+   *  (with PDFs computed using all of the previously supplied observation sets).
      
+   *  
+   * 
      If the samples were supplied via a single call such as
+   * {@link #setObservations(double[][], int[])},
+   * then the return value is a single time-series of local
+   * channel measure values corresponding to these samples.
      
+   * 
+   * 
      Otherwise where disjoint time-series observations were supplied using several 
+   *  calls such as {@link #addObservations(double[][], int[])}
+   *  then the local values for each disjoint observation set will be appended here
+   *  to create a single "time-series" return array.
      
+   * 
+   * @return the "time-series" of local MIs in nats
+   * @throws Exception
+   */
   public double[] computeLocalOfPreviousObservations() throws Exception {
     return computeFromObservations(true);
   }
 
-	public double[] computeFromObservations(boolean returnLocals) throws Exception {
+  /**
+   * This protected method handles the computation of either the average or
+   * local MI (over parts of the total observations).
+   * 
+   * 
      The method returns:
        
+   *  
      1. for (returnLocals == false), an array of size 1,
+   *      containing the average MI 
        2. 
+   *  
      2. for local MIs (returnLocals == true), the array of local MI values
        3. 
+   *  
      
+   * 
+   * @param returnLocals whether to return an array or local values, or else
+   *  sums of these values
+   * @return either the average MI, or array of local MI value, in nats not bits
+   * @throws Exception
+   */
+	protected double[] computeFromObservations(boolean returnLocals) throws Exception {
 
     // FIXME
     int dynCorrExclTime = 0;
@@ -500,17 +554,20 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 
 
 	/**
-	 * Compute the significance of the mutual information of the previously supplied observations.
-	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
-	 *  significant this mutual information actually was.
+	 * Compute the significance of the mutual information of the previously
+   * supplied observations.
+   *
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y)
+   * marginals, to check how significant this mutual information actually was.
 	 *  
-	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
-	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
-	 *  which was performed for Transfer entropy.
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of
+   * nonlinear causality: application to epileptic EEG signals", Journal of
+   * Neuroscience Methods 124 (2003) 113-128 which was performed for transfer
+   * entropy.
 	 * 
 	 * @param numPermutationsToCheck
-	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
-	 *  (i.e. 1 - CDF of our score)
+	 * @return the proportion of MI scores from the distribution which have
+   * higher or equal MIs to ours. (i.e. 1 - CDF of our score)
 	 */
 	public synchronized EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception {
 		// Generate the re-ordered indices:
@@ -521,16 +578,20 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	}
 	
 	/**
-	 * Compute the significance of the mutual information of the previously supplied observations.
-	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y) marginals, to check how
-	 *  significant this mutual information actually was.
+	 * Compute the significance of the mutual information of the previously
+   * supplied observations.
+   *
+	 * We destroy the p(x,y) correlations, while retaining the p(x), p(y)
+   * marginals, to check how significant this mutual information actually was.
 	 *  
-	 * This is in the spirit of Chavez et. al., "Statistical assessment of nonlinear causality:
-	 *  application to epileptic EEG signals", Journal of Neuroscience Methods 124 (2003) 113-128
-	 *  which was performed for Transfer entropy.
+	 * This is in the spirit of Chavez et. al., "Statistical assessment of
+   * nonlinear causality: application to epileptic EEG signals", Journal of
+   * Neuroscience Methods 124 (2003) 113-128 which was performed for transfer
+   * entropy.
 	 * 
 	 * @param newOrderings the specific new orderings to use
-	 * @return the proportion of MI scores from the distribution which have higher or equal MIs to ours.
+	 * @return the proportion of MI scores from the distribution which have
+   * higher or equal MIs to ours.
 	 */
 	public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception {
 		
@@ -577,8 +638,8 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 *  it should be ignored, since the data point was not counted in its
 	 *  own counts in the standard version anyway.
      
 	 *  
-	 *  
      Importantly, the supplied observations are intended to be new observations,
-	 *  not those fed in to compute the PDFs from. There would be
+	 *  
      Importantly, the supplied observations are intended to be new
+   *  observations, not those fed in to compute the PDFs from. There would be
 	 *  some subtle changes necessary to accomodate computing locals on
 	 *  the same data set (e.g. not counting the current point as one
 	 *  of those within eps_x etc.). 
@@ -643,10 +704,29 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		this.debug = debug;
 	}
 
+	/**
+	 * Return the MI last calculated in a call to {@link #computeAverageLocalOfObservations()}
+	 * or {@link #computeLocalOfPreviousObservations()} after the previous
+	 * {@link #initialise()} call.
+	 * 
+	 * @return the last computed average mutual information
+	 */
 	public double getLastAverage() {
 		return mi;
 	}
 	
+	/**
+	 * Get the number of samples to be used for the PDFs here 
+	 * which have been supplied by calls to
+	 * {@link #setObservations(double[][], int[])},
+	 * {@link #addObservations(double[][], int[])}
+	 * etc.
+	 * 
+	 * 
      Note that the number of samples may not be equal to the length of time-series
+	 * supplied (i.e. where a {@link PROP_TIME_DIFF}
+	 * is set).
+	 * 
      
+	 */
 	public int getNumObservations() {
 		return totalObservations;
 	}

From c2b1f7ef25f8a9665b232aa92047d1ba042359fe Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 30 Aug 2017 12:06:02 +1000
Subject: [PATCH 65/78] Altered code generated by AutoAnalyser to generate
 absolute paths to jar and Matlab/Python utilities, so that it doesn't matter
 if the user moves the generated code to another location. Also added warning
 suppression in Matlab for attempting to add the jar to the path multiple
 times.

---
 .../demos/autoanalysis/AutoAnalyser.java      | 32 ++++++++++++++-----
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 78a1e3c..0d68648 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -68,6 +68,7 @@ import java.awt.event.MouseEvent;
 import java.awt.event.MouseListener;
 import java.io.File;
 import java.io.FileWriter;
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Vector;
 
@@ -739,28 +740,43 @@ public abstract class AutoAnalyser extends JFrame
 			//  used for property names
 			javaCode.append("import infodynamics.measures.continuous.*;\n");
 		}
+		// Work out full path of the jar and code utilities:
+		String jarLocation, pythonDemosLocation, matlabDemosLocation;
+		try {
+			File jarLocationFile = new File(System.getProperty("user.dir") + "/../../infodynamics.jar");
+			jarLocation = jarLocationFile.getCanonicalPath();
+			File pythonDemosLocationFile = new File(System.getProperty("user.dir") + "/../python");
+			pythonDemosLocation = pythonDemosLocationFile.getCanonicalPath();
+			File matlabDemosLocationFile = new File(System.getProperty("user.dir") + "/../octave");
+			matlabDemosLocation = matlabDemosLocationFile.getCanonicalPath();
+		} catch (IOException ioex) {
+			JOptionPane.showMessageDialog(this,
+					ioex.getMessage());
+			resultsLabel.setText("Cannot find jar and Matlab/Python utility locations: " + ioex.getMessage());
+			return;
+		}
 		// 2. Python
 		StringBuffer pythonCode = new StringBuffer();
 		pythonCode.append("from jpype import *\n");
 		pythonCode.append("import numpy\n");
-		pythonCode.append("# I think this is a bit of a hack, python users will do better on this:\n");
-		pythonCode.append("sys.path.append(\"../python\")\n");
+		pythonCode.append("# Our python data file readers are a bit of a hack, python users will do better on this:\n");
+		pythonCode.append("sys.path.append(\"" + pythonDemosLocation + "\")\n");
 		if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE)) {
 			pythonCode.append("import readIntsFile\n\n");
 		} else {
 			pythonCode.append("import readFloatsFile\n\n");
 		}
-		pythonCode.append("# Add JIDT jar library to the path\n\n");
-		pythonCode.append("jarLocation = \"../../infodynamics.jar\"\n");
+		pythonCode.append("# Add JIDT jar library to the path\n");
+		pythonCode.append("jarLocation = \"" + jarLocation + "\"\n");
 		pythonCode.append("# Start the JVM (add the \"-Xmx\" option with say 1024M if you get crashes due to not enough memory space)\n");
 		pythonCode.append("startJVM(getDefaultJVMPath(), \"-ea\", \"-Djava.class.path=\" + jarLocation)\n\n");
 		// 3. Matlab:
 		StringBuffer matlabCode = new StringBuffer();
-		matlabCode.append("% Add JIDT jar library to the path\n");
-		matlabCode.append("jidtPath = '../../';\n");
-		matlabCode.append("javaaddpath([jidtPath, '/infodynamics.jar']);\n");
+		matlabCode.append("% Add JIDT jar library to the path, and disable warnings that it's already there:\n");
+		matlabCode.append("warning('off','MATLAB:Java:DuplicateClass');\n");
+		matlabCode.append("javaaddpath('" + jarLocation + "');\n");
 		matlabCode.append("% Add utilities to the path\n");
-		matlabCode.append("addpath([jidtPath, '/demos/octave']);\n\n");
+		matlabCode.append("addpath('" + matlabDemosLocation + "');\n\n");
 		
 		try{
 			// Create both discrete and continuous calculators to make

From 257ef285d3cb4430c4e978a90cdc16a678c49d98 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 30 Aug 2017 15:39:59 +1000
Subject: [PATCH 66/78] Added a Matlab utility function to save CA data in a
 (relatively) efficient data format, since Matlab will only save doubles for
 ascii format.

---
 demos/octave/CellularAutomata/saveCA.m | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 demos/octave/CellularAutomata/saveCA.m

diff --git a/demos/octave/CellularAutomata/saveCA.m b/demos/octave/CellularAutomata/saveCA.m
new file mode 100644
index 0000000..34d0f9d
--- /dev/null
+++ b/demos/octave/CellularAutomata/saveCA.m
@@ -0,0 +1,16 @@
+%
+% Utility function to save raw CA data to a text file format.
+% Unfortunately MAtlab's -ascii format only writes full doubles, making much larger files than we need.
+% So I wrote this.
+
+function saveCA(filename, caStates)
+
+	warning('off','MATLAB:Java:DuplicateClass');
+	javaaddpath('../../../infodynamics.jar');
+	addpath('..');
+
+	arrayWriter = javaObject('infodynamics.utils.ArrayFileWriter');
+	arrayWriter.makeIntMatrixFile(octaveToJavaIntMatrix(caStates), filename);
+	
+end
+

From d36c803e413d7a8bfd56cce0b6865f9110640997 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 30 Aug 2017 16:24:17 +1000
Subject: [PATCH 67/78] Added utilities to convert between estimate values and
 p-values for an analytic null distribution in bulk (array calls)

---
 .../AnalyticMeasurementDistribution.java      | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java b/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java
index e547349..cdef850 100755
--- a/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java
+++ b/java/source/infodynamics/utils/AnalyticMeasurementDistribution.java
@@ -76,6 +76,21 @@ public abstract class AnalyticMeasurementDistribution extends MeasurementDistrib
 	 */
 	public abstract double computePValueForGivenEstimate(double estimate);
 
+	/**
+	 * Computes p-values corresponding to a set of estimates, each done via
+	 * {@link #computePValueForGivenEstimate(double)}
+	 * 
+	 * @param estimates array of estimates to return corresponding p-values for.
+	 * @return
+	 */
+	public double[] computePValuesForGivenEstimates(double[] estimates) {
+		double[] pValues = new double[estimates.length];
+		for (int i = 0; i < estimates.length; i++) {
+			pValues[i] = computePValueForGivenEstimate(estimates[i]);
+		}
+		return estimates;
+	}
+
 	/**
 	 * Compute the estimated observed measured value corresponding to
 	 * a given p-value
@@ -107,9 +122,25 @@ public abstract class AnalyticMeasurementDistribution extends MeasurementDistrib
 	 * 		Physical Review Letters, 109, p. 138105+ (2012).
      * 
 	 * 
+	 * @param pValue the sample p-value for the given channel measure score under this null hypothesis.
 	 * @return the estimate of the channel measure score corresponding to
 	 * the given p-value under this null hypothesis.
 	 * @throws Exception
 	 */
 	public abstract double computeEstimateForGivenPValue(double pValue);
+	
+	/**
+	 * Computes estimates corresponding to a set of p-values, each done via
+	 * {@link #computeEstimateForGivenPValue(double)}
+	 * 
+	 * @param pValues array of p-values to return corresponding estimates for.
+	 * @return
+	 */
+	public double[] computeEstimatesForGivenPValues(double[] pValues) {
+		double[] estimates = new double[pValues.length];
+		for (int i = 0; i < pValues.length; i++) {
+			estimates[i] = computeEstimateForGivenPValue(pValues[i]);
+		}
+		return estimates;
+	}
 }

From 9aea9c4a0294914cc4a4af681f85c236a0021161 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 30 Aug 2017 09:14:56 +0100
Subject: [PATCH 68/78] Added some tests for KSG mixed setProperty.

---
 ...MultiVariateWithDiscreteKraskovTester.java | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index 49d7580..4c735c6 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -212,4 +212,46 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
       assertEquals(analytical, estimated, 0.05);
     }
   }
+
+  public void testProperties() throws Exception {
+    MutualInfoCalculatorMultiVariateWithDiscreteKraskov miCalc =
+        new MutualInfoCalculatorMultiVariateWithDiscreteKraskov();
+
+    // Generate data with unit Gaussians separated by fixed distance
+    double separation = 4.0;
+    double true_val = 0.6327;
+    int N = 1000;
+    RandomGenerator rg = new RandomGenerator();
+    double[][] contData = rg.generateNormalData(N, 1, 0, 1);
+    int[] discData = rg.generateRandomInts(N, 2);
+    for (int i = 0; i < N; i++) {
+      if (discData[i] > 0) {
+        contData[i][0] += separation;
+      }
+    }
+
+    // Test we get correct result with default properties
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res1 = miCalc.computeAverageLocalOfObservations();
+    assertEquals(true_val, res1, 0.05);
+
+    // Test that timeDiff has an effect
+    miCalc.setProperty("TIME_DIFF", "1");
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res2 = miCalc.computeAverageLocalOfObservations();
+    assertEquals(0.0, res2, 0.05);
+    miCalc.setProperty("TIME_DIFF", "0");
+
+    // Test that K has an effect
+    miCalc.setProperty("K", "2");
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res3 = miCalc.computeAverageLocalOfObservations();
+    assertTrue(Math.abs(res3 - res1) > 0.001);
+    miCalc.setProperty("k", "4");
+
+
+  }
 }

From 5961fb624610c33b596a8dbcc1d292bcc255588e Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 30 Aug 2017 12:17:02 +0100
Subject: [PATCH 69/78] Add Theiler window and jitter to KSG mixed calc.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 74 ++++++++++++++++++-
 1 file changed, 71 insertions(+), 3 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index edc60ba..1a6f15c 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -30,6 +30,7 @@ import infodynamics.utils.KdTree;
 import java.util.Iterator;
 import java.util.Vector;
 import java.util.Arrays;
+import java.util.Random;
 
 /**
  * 
      Compute the Mutual Information between a vector of continuous variables and discrete
@@ -165,6 +166,17 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 * must be >= 0)
 	 */ 
 	public static final String PROP_TIME_DIFF = "TIME_DIFF";
+  /**
+   * Property name for an amount of random Gaussian noise to be
+   *  added to the data (default is 1e-8, matching the MILCA toolkit).
+   */
+  public static final String PROP_ADD_NOISE = "NOISE_LEVEL_TO_ADD";
+  /**
+   * Property name for a dynamics exclusion time window 
+   * otherwise known as Theiler window (see Kantz and Schreiber).
+   * Default is 0 which means no dynamic exclusion window.
+   */
+  public static final String PROP_DYN_CORR_EXCL_TIME = "DYN_CORR_EXCL"; 
 	/**
 	 * Track whether we're going to normalise the joint variables individually
 	 */
@@ -185,6 +197,18 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 *  are adjusted so that there is no timeDiff between them).
 	 */
 	protected int timeDiff = 0;
+  /**
+   * Whether to add an amount of random noise to the incoming data
+   */
+  protected boolean addNoise = true;
+  /**
+   * Amount of random Gaussian noise to add to the incoming data
+   */
+  protected double noiseLevel = (double) 1e-8;
+  /**
+   * Size of dynamic correlation exclusion window.
+   */
+  protected int dynCorrExclTime = 0;
 
   /**
    * Construct an instance of the KSG mixed MI calculator
@@ -226,6 +250,18 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 	 *      variables (true by default)
 	 * 	
    18. {@link #PROP_TIME_DIFF} - Time difference between source and
    *     	destination (0 by default). Must be >= 0.
      19. 
+   *  
    19. {@link #PROP_ADD_NOISE} -- a standard deviation for an amount of
+   *    random Gaussian noise to add to
+   *      each variable, to avoid having neighbourhoods with artificially
+   *      large counts. (We also accept "false" to indicate "0".)
+   *      The amount is added in after any normalisation,
+   *      so can be considered as a number of standard deviations of the data.
+   *      (Recommended by Kraskov. MILCA uses 1e-8; but adds in
+   *      a random amount of noise in [0,noiseLevel) ).
+   *      Default 1e-8 to match the noise order in MILCA toolkit.
      20. 
+   *  
    20. {@link #PROP_DYN_CORR_EXCL_TIME} -- a dynamics exclusion time window,
+   *      also known as Theiler window (see Kantz and Schreiber);
+   *      default is 0 which means no dynamic exclusion window.
      21. 
 	 * 
 	 * 
 	 * @param propertyName name of the property to set
@@ -239,6 +275,17 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
       normType = KdTree.validateNormType(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
 			normalise = Boolean.parseBoolean(propertyValue);
+    } else if (propertyName.equalsIgnoreCase(PROP_DYN_CORR_EXCL_TIME)) {
+      dynCorrExclTime = Integer.parseInt(propertyValue);
+    } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
+      if (propertyValue.equals("0") ||
+          propertyValue.equalsIgnoreCase("false")) {
+        addNoise = false;
+        noiseLevel = 0;
+      } else {
+        addNoise = true;
+        noiseLevel = Double.parseDouble(propertyValue);
+      }
 		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
 			int val = Integer.parseInt(propertyValue);
 			if (val < 0) {
@@ -358,6 +405,17 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 			}
 		}
 
+    if (addNoise) {
+      Random random = new Random();
+      // Add Gaussian noise of std dev noiseLevel to the data
+      for (int r = 0; r < totalObservations; r++) {
+        for (int c = 0; c < dimensions; c++) {
+          continuousData[r][c] +=
+              random.nextGaussian()*noiseLevel;
+        }
+      }
+    }
+
     digammaN = MathsUtils.digamma(totalObservations);
     digammaK = MathsUtils.digamma(k);
 
@@ -472,9 +530,6 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
    */
 	protected double[] computeFromObservations(boolean returnLocals) throws Exception {
 
-    // FIXME
-    int dynCorrExclTime = 0;
-
     int[] cumcount = new int[base];
     Arrays.fill(cumcount, 0);
 
@@ -496,8 +551,21 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
       int b = discreteData[t];
       double eps_x = kdTreeBins[b].findKNearestNeighbours(k, cumcount[b]++).poll().distance;
       int n_x = kdTreeJoint.countPointsWithinOrOnR(t, eps_x, dynCorrExclTime);
+
+      // Number of points in discrete bin b, not counting itself
       int n_y = counts[b] - 1;
 
+      if (dynCorrExclTime > 0) {
+        // Add one here because later we are going to subtract the point itself
+        // again in the for-loop when tt == t
+        n_y++;
+        for (int tt = t - dynCorrExclTime; tt <= t + dynCorrExclTime; tt++) {
+          if (discreteData[tt] == b) {
+            n_y--;
+          }
+        }
+      }
+
 			avNx += n_x;
 			avNy += n_y;
 			// And take the digamma before adding into the 

From f56ac6e69e90b573915e7dc7157a12854ae1414b Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 30 Aug 2017 13:30:30 +0100
Subject: [PATCH 70/78] Fix bug with Theiler window in KSG mixed calc.

---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 1a6f15c..9e176a4 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -560,6 +560,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
         // again in the for-loop when tt == t
         n_y++;
         for (int tt = t - dynCorrExclTime; tt <= t + dynCorrExclTime; tt++) {
+          if ((tt < 0) || (tt >= totalObservations)) {
+            continue;
+          }
           if (discreteData[tt] == b) {
             n_y--;
           }

From 9b7d7c70c8630d49fbcbd4a314acf8151a05b7f9 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Wed, 30 Aug 2017 13:31:13 +0100
Subject: [PATCH 71/78] Add unittests for noise level and Theiler window in KSG
 mixed calc.

---
 ...lInfoMultiVariateWithDiscreteKraskovTester.java | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index 4c735c6..63c428e 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -252,6 +252,20 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
     assertTrue(Math.abs(res3 - res1) > 0.001);
     miCalc.setProperty("k", "4");
 
+    // Test that noiseLevel has an effect
+    miCalc.setProperty("NOISE_LEVEL_TO_ADD", "20.0");
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res4 = miCalc.computeAverageLocalOfObservations();
+    assertEquals(0.0, res4, 0.05);
+    miCalc.setProperty("NOISE_LEVEL_TO_ADD", "false");
+    
+    // Test that dynCorrExcl has an effect
+    miCalc.setProperty("DYN_CORR_EXCL", "10");
+    miCalc.initialise(1, 2);
+    miCalc.setObservations(contData, discData);
+    double res5 = miCalc.computeAverageLocalOfObservations();
+    assertTrue(Math.abs(res5 - res1) > 0.001);
 
   }
 }

From 0e27dbcbc27a352636009f11876a41108e40ff5b Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Sat, 2 Sep 2017 20:18:36 +1000
Subject: [PATCH 72/78] Fixed default value of dynamic exclusion window in
 Kernel MI calculator to be 0. Technically it is zero (because dynCorrExcl
 boolean is set to false, but gets reported as 100 because the integer value
 of dynCorrExclTime was not set to 0 itself)

---
 .../kernel/MutualInfoCalculatorMultiVariateKernel.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateKernel.java b/java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateKernel.java
index 4959b9f..012f305 100755
--- a/java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateKernel.java
+++ b/java/source/infodynamics/measures/continuous/kernel/MutualInfoCalculatorMultiVariateKernel.java
@@ -65,7 +65,7 @@ public class MutualInfoCalculatorMultiVariateKernel
 	public static final String NORMALISE_PROP_NAME = "NORMALISE";
 	
 	private boolean dynCorrExcl = false;
-	private int dynCorrExclTime = 100;
+	private int dynCorrExclTime = 0;
 	/**
 	 * Property name for a dynamics exclusion time window (see Kantz and Schreiber),
 	 * default is 0 which means no dynamic exclusion window.

From 6a481128182189e88024dcec0ba089b36d6b08c0 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Wed, 6 Sep 2017 13:51:34 +1000
Subject: [PATCH 73/78] In AutoAnalyser, limiting the use of
 variableCombinations to when we're actually making a calculation. (It chews
 up memory for large combinations, which makes AutoAnalyser take a long time
 even if we're not computing)

---
 .../demos/autoanalysis/AutoAnalyser.java      | 57 ++++++++++---------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
index 0d68648..9278bad 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyser.java
@@ -692,30 +692,33 @@ public abstract class AutoAnalyser extends JFrame
 		
 		int[] singleCalcColumns = new int[numVariables];
 		Vector variableCombinations = new Vector();
-		try {
-			if (allCombosCheckBox.isSelected()) {
-				// We're doing all combinations
-				fillOutAllCombinations(variableCombinations);
-			} else {
-				// we're doing a single combination
-				for (int i = 0; i < numVariables; i++) {
-					singleCalcColumns[i] = Integer.parseInt(variableColTextFields[i].getText());
-					if ((singleCalcColumns[i] < 0) || (singleCalcColumns[i] >= dataColumns)) {
-						JOptionPane.showMessageDialog(this,
-								String.format("%s column must be between 0 and %d for this data set",
-										variableColNumLabels[i], dataColumns-1));
-						resultsLabel.setText(" ");
-						return;
+		if (computeResultCheckBox.isSelected()) {
+			// Only need variableCombinations filled out if we're computing
+			try {
+				if (allCombosCheckBox.isSelected()) {
+					// We're doing all combinations
+					fillOutAllCombinations(variableCombinations);
+				} else {
+					// we're doing a single combination
+					for (int i = 0; i < numVariables; i++) {
+						singleCalcColumns[i] = Integer.parseInt(variableColTextFields[i].getText());
+						if ((singleCalcColumns[i] < 0) || (singleCalcColumns[i] >= dataColumns)) {
+							JOptionPane.showMessageDialog(this,
+									String.format("%s column must be between 0 and %d for this data set",
+											variableColNumLabels[i], dataColumns-1));
+							resultsLabel.setText(" ");
+							return;
+						}
 					}
+					variableCombinations.add(singleCalcColumns);
 				}
-				variableCombinations.add(singleCalcColumns);
+			} catch (Exception e) {
+				// Catches number format exception, and column number out of bounds
+				JOptionPane.showMessageDialog(this,
+						e.getMessage());
+				resultsLabel.setText("Cannot parse a column number from input: " + e.getMessage());
+				return;
 			}
-		} catch (Exception e) {
-			// Catches number format exception, and column number out of bounds
-			JOptionPane.showMessageDialog(this,
-					e.getMessage());
-			resultsLabel.setText("Cannot parse a column number from input: " + e.getMessage());
-			return;
 		}
 
 		// Generate headers:
@@ -1034,7 +1037,7 @@ public abstract class AutoAnalyser extends JFrame
 			String pythonPrefix = "";
 			String matlabPrefix = "";
 			StringBuffer extraFormatTerms = new StringBuffer();
-			if (variableCombinations.size() > 1) {
+			if (allCombosCheckBox.isSelected()) {
 				// We're doing all combinations.
 				
 				if (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)) {
@@ -1147,7 +1150,7 @@ public abstract class AutoAnalyser extends JFrame
 			matlabCode.append(matlabPrefix + "result = calc.computeAverageLocalOfObservations();\n");
 			
 			String resultsPrefixString;
-			if (variableCombinations.size() > 1) {
+			if (allCombosCheckBox.isSelected()) {
 				resultsPrefixString = String.format(measureAcronym + "_%s(%s) = ",
 						selectedCalcType, variableRelationshipFormatString);
 			} else {
@@ -1203,7 +1206,7 @@ public abstract class AutoAnalyser extends JFrame
 					"%.4f " + units + resultsSuffixString + "\\n', ...\n\t" + matlabPrefix +
 					extraFormatTerms + "result" + statSigFormatTerms + ");\n");
 			
-			if (variableCombinations.size() > 1) {
+			if (allCombosCheckBox.isSelected()) {
 				// We're doing all combos
 				
 				// Finalise the loops in the code:
@@ -1249,7 +1252,7 @@ public abstract class AutoAnalyser extends JFrame
 				//  button to cancel the calculation whilst it's running
 				for (int[] columnCombo : variableCombinations) {
 					
-					if ((variableCombinations.size() > 1) && skipColumnCombo(columnCombo)) {
+					if (allCombosCheckBox.isSelected() && skipColumnCombo(columnCombo)) {
 						System.out.print("Skipping column combo ");
 						MatrixUtils.printArray(System.out, columnCombo);
 						continue;
@@ -1316,7 +1319,7 @@ public abstract class AutoAnalyser extends JFrame
 					}
 					
 					String resultsText;
-					if (variableCombinations.size() > 1) {
+					if (allCombosCheckBox.isSelected()) {
 						// We're doing all pairs
 						// OLD:
 						// resultsText = String.format(
@@ -1336,7 +1339,7 @@ public abstract class AutoAnalyser extends JFrame
 					System.out.println(resultsText);
 				}
 	
-				if ((variableCombinations.size() == 1) && 
+				if ((allCombosCheckBox.isSelected()) && 
 						!(selectedCalcType.equalsIgnoreCase(CALC_TYPE_DISCRETE) ||
 						 (selectedCalcType.equalsIgnoreCase(CALC_TYPE_BINNED)))) {
 					// Read the current property values back out (in case of 

From 0c5bed62d6d8f54aee6fc65066776d3bb38b8967 Mon Sep 17 00:00:00 2001
From: jlizier 
Date: Tue, 24 Oct 2017 11:34:42 +1100
Subject: [PATCH 74/78] Fixing tab/spaces mix in Python code for all combos for
 loops generated by AutoAnalyser calculators

---
 .../infodynamics/demos/autoanalysis/AutoAnalyserAIS.java    | 2 +-
 .../infodynamics/demos/autoanalysis/AutoAnalyserCMI.java    | 6 +++---
 .../infodynamics/demos/autoanalysis/AutoAnalyserCTE.java    | 6 +++---
 .../demos/autoanalysis/AutoAnalyserChannelCalculator.java   | 6 +++---
 .../demos/autoanalysis/AutoAnalyserEntropy.java             | 2 +-
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java
index 46879ae..46eee68 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserAIS.java
@@ -215,7 +215,7 @@ public class AutoAnalyserAIS extends AutoAnalyser {
 		pythonCode.append("\n");
 		pythonCode.append("# Compute for all variables:\n");
 		pythonCode.append("for v in range(" + dataColumns + "):\n");
-		String pythonPrefix = "\t";
+		String pythonPrefix = "    ";
 		pythonCode.append(pythonPrefix+ "# For each variable:\n");
 		// 3. Matlab code
 		matlabCode.append("\n");
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
index 7c5a173..4eb12db 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCMI.java
@@ -191,11 +191,11 @@ public class AutoAnalyserCMI extends AutoAnalyser
 		pythonCode.append("c = " + conditional + "\n");
 		pythonCode.append("# Compute for all pairs:\n");
 		pythonCode.append("for s in range(" + dataColumns + "):\n");
-		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
-		String pythonPrefix = "\t\t";
+		pythonCode.append("    for d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "        ";
 		pythonCode.append(pythonPrefix+ "# For each source-dest pair (given conditional):\n");
 		pythonCode.append(pythonPrefix + "if ((s == d) or (s == c) or (d == c)):\n");
-		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		pythonCode.append(pythonPrefix + "    continue\n");
 		// 3. Matlab code
 		matlabCode.append("\n");
 		matlabCode.append("c = " + (conditional+1) + ";\n");
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java
index 915e068..9c4161a 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserCTE.java
@@ -216,11 +216,11 @@ public class AutoAnalyserCTE extends AutoAnalyser
 		pythonCode.append("c = " + conditional + "\n");
 		pythonCode.append("# Compute for all pairs:\n");
 		pythonCode.append("for s in range(" + dataColumns + "):\n");
-		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
-		String pythonPrefix = "\t\t";
+		pythonCode.append("    for d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "        ";
 		pythonCode.append(pythonPrefix+ "# For each source-dest pair (given conditional):\n");
 		pythonCode.append(pythonPrefix + "if ((s == d) or (s == c) or (d == c)):\n");
-		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		pythonCode.append(pythonPrefix + "    continue\n");
 		// 3. Matlab code
 		matlabCode.append("\n");
 		matlabCode.append("c = " + (conditional+1) + ";\n");
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
index d365b90..e53fd69 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserChannelCalculator.java
@@ -99,11 +99,11 @@ public abstract class AutoAnalyserChannelCalculator extends AutoAnalyser {
 		pythonCode.append("\n");
 		pythonCode.append("# Compute for all pairs:\n");
 		pythonCode.append("for s in range(" + dataColumns + "):\n");
-		pythonCode.append("\tfor d in range(" + dataColumns + "):\n");
-		String pythonPrefix = "\t\t";
+		pythonCode.append("    for d in range(" + dataColumns + "):\n");
+		String pythonPrefix = "        ";
 		pythonCode.append(pythonPrefix+ "# For each source-dest pair:\n");
 		pythonCode.append(pythonPrefix + "if (s == d):\n");
-		pythonCode.append(pythonPrefix + "\tcontinue\n");
+		pythonCode.append(pythonPrefix + "    continue\n");
 		// 3. Matlab code
 		matlabCode.append("\n");
 		matlabCode.append("% Compute for all pairs:\n");
diff --git a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
index 3131cae..5e8ac79 100644
--- a/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
+++ b/demos/java/infodynamics/demos/autoanalysis/AutoAnalyserEntropy.java
@@ -162,7 +162,7 @@ public class AutoAnalyserEntropy extends AutoAnalyser {
 		pythonCode.append("\n");
 		pythonCode.append("# Compute for all variables:\n");
 		pythonCode.append("for v in range(" + dataColumns + "):\n");
-		String pythonPrefix = "\t";
+		String pythonPrefix = "    ";
 		pythonCode.append(pythonPrefix+ "# For each variable:\n");
 		// 3. Matlab code
 		matlabCode.append("\n");

From d40024e1cd045844f14ca0ffa41ed386661c910d Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Mon, 20 Nov 2017 14:02:00 +0000
Subject: [PATCH 75/78] Allow negative timeDiff in KSG mixed calc.

---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 9e176a4..4c949eb 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -287,11 +287,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
         noiseLevel = Double.parseDouble(propertyValue);
       }
 		} else if (propertyName.equalsIgnoreCase(PROP_TIME_DIFF)) {
-			int val = Integer.parseInt(propertyValue);
-			if (val < 0) {
-				throw new Exception("Time difference must be >= 0. Flip data1 and data2 around if required.");
-			}
-      timeDiff = val;
+			timeDiff = Integer.parseInt(propertyValue);
 		}
 	}
 
@@ -309,7 +305,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		if (continuousObservations[0].length != dimensions) {
 			throw new Exception("The continuous observations do not have the expected number of variables (" + dimensions + ")");
 		}
-    if (continuousObservations.length > timeDiff) {
+    if (continuousObservations.length > Math.abs(timeDiff)) {
       vectorOfContinuousObservations.add(continuousObservations);
       vectorOfDiscreteObservations.add(discreteObservations);
     }
@@ -355,7 +351,7 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		// First work out the size to allocate the joint vectors, and do the allocation:
 		totalObservations = 0;
 		for (double[][] destination : vectorOfContinuousObservations) {
-			totalObservations += destination.length - timeDiff;
+			totalObservations += destination.length - Math.abs(timeDiff);
 		}
 		continuousData = new double[totalObservations][dimensions];
 		discreteData   = new int[totalObservations];
@@ -367,12 +363,24 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		for (int[] dct : vectorOfDiscreteObservations) {
 			double[][] cnt = iterator.next();
 			// Copy the data from these given observations into our master 
-			//  array, aligning them incorporating the timeDiff:
+			//  array, aligning them incorporating the timeDiff. Depending on whether
+      //  timeDiff >= 0, we have to put first the elements of one array or the
+      //  other.
+      if (timeDiff >= 0) {
 			MatrixUtils.arrayCopy(cnt, 0, 0,
 					continuousData, startObservation, 0,
 					cnt.length - timeDiff, dimensions);
       System.arraycopy(dct, timeDiff, discreteData, startObservation, dct.length - timeDiff);
 			startObservation += cnt.length - timeDiff;
+
+      } else {
+        MatrixUtils.arrayCopy(cnt, Math.abs(timeDiff), 0,
+            continuousData, startObservation, 0,
+            cnt.length - Math.abs(timeDiff), dimensions);
+        System.arraycopy(dct, 0, discreteData, startObservation, dct.length - Math.abs(timeDiff));
+        startObservation += cnt.length - Math.abs(timeDiff);
+      }
+
 		}
 		// We don't need to keep the vectors of observation sets anymore:
 		vectorOfContinuousObservations = null;

From 8f169171e110752185f4766d2f1a398e07dfa131 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Mon, 20 Nov 2017 14:14:49 +0000
Subject: [PATCH 76/78] Fixed small bug in KSG mixed calc surrogates.

Surrogate calculator was applying timeDiff twice, so surrogate
dataset had abs(timeDiff) less observations than it should.
---
 .../MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 4c949eb..2ad758e 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -484,6 +484,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		
 		// Generate a new re-ordered data set
 		int[] shuffledDiscreteData = MatrixUtils.extractSelectedTimePoints(discreteData, newOrdering);
+    // Since the observations in the original calculator have already been
+    // shifted according to timeDiff, set timeDiff = 0 in the surrogate calculator.
+    miSurrogateCalculator.setProperty(PROP_TIME_DIFF, "0");
 		// Perform new initialisations
 		miSurrogateCalculator.initialise(dimensions, base);
 		// Set new observations

From 25df49fa5a1806e1c132277ad730a04c9994ee16 Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Mon, 20 Nov 2017 14:26:25 +0000
Subject: [PATCH 77/78] KSG mixed calc now allocates memory for locals only if
 requested.

---
 ...ualInfoCalculatorMultiVariateWithDiscreteKraskov.java | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 2ad758e..5f0815f 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -552,7 +552,10 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
 		
 		double testSum = 0.0; // Used for debugging prints
     int N = totalObservations;
-    double[] locals = new double[N];
+    double[] locals = null;
+    if (returnLocals) {
+      locals = new double[N];
+    }
 
 		for (int t = 0; t < N; t++) {
 
@@ -593,7 +596,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
       // double localValue = MathsUtils.digamma(k) - 1.0/(double)k - localSum + MathsUtils.digamma(N);
       // Instead do:
       double localValue = digammaK + digammaN - localSum;
-      locals[t] = localValue;
+      if (returnLocals) {
+        locals[t] = localValue;
+      }
 
 			if (debug) {
 				testSum += localValue;

From ecda70f8e92859364c88b82153c6ab31cabdf91f Mon Sep 17 00:00:00 2001
From: Pedro Martinez Mediano 
Date: Mon, 20 Nov 2017 14:47:00 +0000
Subject: [PATCH 78/78] Hard-setting dynCorrExclTime to 0 in KSG mixed calc.

Previous implementation was incomplete due to a mismatch between the
sample indices in the full and marginal KdTrees. Some code has been left
to guide future development.
---
 ...ulatorMultiVariateWithDiscreteKraskov.java | 55 +++++++++++--------
 ...MultiVariateWithDiscreteKraskovTester.java | 13 +++--
 2 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
index 5f0815f..65b3c24 100755
--- a/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
+++ b/java/source/infodynamics/measures/mixed/kraskov/MutualInfoCalculatorMultiVariateWithDiscreteKraskov.java
@@ -171,12 +171,15 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
    *  added to the data (default is 1e-8, matching the MILCA toolkit).
    */
   public static final String PROP_ADD_NOISE = "NOISE_LEVEL_TO_ADD";
-  /**
-   * Property name for a dynamics exclusion time window 
-   * otherwise known as Theiler window (see Kantz and Schreiber).
-   * Default is 0 which means no dynamic exclusion window.
-   */
-  public static final String PROP_DYN_CORR_EXCL_TIME = "DYN_CORR_EXCL"; 
+
+  // TODO: properly implement dynCorrExclTime
+  // /**
+  //  * Property name for a dynamics exclusion time window 
+  //  * otherwise known as Theiler window (see Kantz and Schreiber).
+  //  * Default is 0 which means no dynamic exclusion window.
+  //  */
+  // public static final String PROP_DYN_CORR_EXCL_TIME = "DYN_CORR_EXCL"; 
+
 	/**
 	 * Track whether we're going to normalise the joint variables individually
 	 */
@@ -206,7 +209,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
    */
   protected double noiseLevel = (double) 1e-8;
   /**
-   * Size of dynamic correlation exclusion window.
+   * Size of dynamic correlation exclusion window. By default is set to 0.
+   *
+   * NOTE non-zero exclusion windows are not yet supported in mixed calculators.
    */
   protected int dynCorrExclTime = 0;
 
@@ -275,8 +280,9 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
       normType = KdTree.validateNormType(propertyValue);
 		} else if (propertyName.equalsIgnoreCase(PROP_NORMALISE)) {
 			normalise = Boolean.parseBoolean(propertyValue);
-    } else if (propertyName.equalsIgnoreCase(PROP_DYN_CORR_EXCL_TIME)) {
-      dynCorrExclTime = Integer.parseInt(propertyValue);
+    // TODO: properly implement dynCorrExclTime
+    // } else if (propertyName.equalsIgnoreCase(PROP_DYN_CORR_EXCL_TIME)) {
+    //   dynCorrExclTime = Integer.parseInt(propertyValue);
     } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
       if (propertyValue.equals("0") ||
           propertyValue.equalsIgnoreCase("false")) {
@@ -569,19 +575,24 @@ public class MutualInfoCalculatorMultiVariateWithDiscreteKraskov implements Mutu
       // Number of points in discrete bin b, not counting itself
       int n_y = counts[b] - 1;
 
-      if (dynCorrExclTime > 0) {
-        // Add one here because later we are going to subtract the point itself
-        // again in the for-loop when tt == t
-        n_y++;
-        for (int tt = t - dynCorrExclTime; tt <= t + dynCorrExclTime; tt++) {
-          if ((tt < 0) || (tt >= totalObservations)) {
-            continue;
-          }
-          if (discreteData[tt] == b) {
-            n_y--;
-          }
-        }
-      }
+      // TODO: properly implement dynCorrExclTime. Code left here for future
+      // development. To implement exclusion window in the full joint space we need
+      // to track the sample indices in each point in the kdTreeBins. See comments in
+      // https://github.com/jlizier/jidt/pull/58#pullrequestreview-64865317
+      //
+      // if (dynCorrExclTime > 0) {
+      //   // Add one here because later we are going to subtract the point itself
+      //   // again in the for-loop when tt == t
+      //   n_y++;
+      //   for (int tt = t - dynCorrExclTime; tt <= t + dynCorrExclTime; tt++) {
+      //     if ((tt < 0) || (tt >= totalObservations)) {
+      //       continue;
+      //     }
+      //     if (discreteData[tt] == b) {
+      //       n_y--;
+      //     }
+      //   }
+      // }
 
 			avNx += n_x;
 			avNy += n_y;
diff --git a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
index 63c428e..f692491 100755
--- a/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
+++ b/java/unittests/infodynamics/measures/mixed/kraskov/MutualInfoMultiVariateWithDiscreteKraskovTester.java
@@ -260,12 +260,13 @@ public class MutualInfoMultiVariateWithDiscreteKraskovTester extends TestCase {
     assertEquals(0.0, res4, 0.05);
     miCalc.setProperty("NOISE_LEVEL_TO_ADD", "false");
     
-    // Test that dynCorrExcl has an effect
-    miCalc.setProperty("DYN_CORR_EXCL", "10");
-    miCalc.initialise(1, 2);
-    miCalc.setObservations(contData, discData);
-    double res5 = miCalc.computeAverageLocalOfObservations();
-    assertTrue(Math.abs(res5 - res1) > 0.001);
+    // TEST REMOVED. dynCorrExcl not properly implemented yet
+    // // Test that dynCorrExcl has an effect
+    // miCalc.setProperty("DYN_CORR_EXCL", "10");
+    // miCalc.initialise(1, 2);
+    // miCalc.setObservations(contData, discData);
+    // double res5 = miCalc.computeAverageLocalOfObservations();
+    // assertTrue(Math.abs(res5 - res1) > 0.001);
 
   }
 }