From 3e64fd28bd8b1dd5c0f44bc5e479402dd69e96ed Mon Sep 17 00:00:00 2001 From: jlizier Date: Fri, 18 Aug 2017 15:31:27 +1000 Subject: [PATCH] Altering various continuous calculators to implement the EmpiricalNullDistributionComputer interface (where they're already implementing computeSignificance() etc.). Also altered the EmpiricalNullDistributionComputer methods to throw Exceptions, since the continuous calculators generally do this (and doesn't harm the discrete ones). Also involved implementing the methods in ConditionalMIMultiVariateCommon by selecting to permute the first variable by default. --- .../ActiveInfoStorageCalculator.java | 71 +--------------- ...tiveInfoStorageCalculatorMultiVariate.java | 4 +- ...geCalculatorMultiVariateViaMutualInfo.java | 1 - .../continuous/ChannelCalculatorCommon.java | 66 +-------------- ...ionalMutualInfoCalculatorMultiVariate.java | 82 ++++++++----------- ...nditionalMutualInfoMultiVariateCommon.java | 12 +++ .../continuous/PredictiveInfoCalculator.java | 71 +--------------- ...PredictiveInfoCalculatorViaMutualInfo.java | 4 +- ...alMutualInformationCalculatorDiscrete.java | 4 +- .../EmpiricalNullDistributionComputer.java | 6 +- 10 files changed, 63 insertions(+), 258 deletions(-) diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java index b04b173..01467ac 100644 --- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java +++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculator.java @@ -18,7 +18,7 @@ package infodynamics.measures.continuous; -import infodynamics.utils.EmpiricalMeasurementDistribution; +import infodynamics.utils.EmpiricalNullDistributionComputer; /** * Interface for implementations of @@ -84,7 +84,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution; * @author Joseph Lizier (email, * www) */ -public interface ActiveInfoStorageCalculator { +public interface ActiveInfoStorageCalculator extends EmpiricalNullDistributionComputer { /** * Property name for embedding length k of @@ -294,73 +294,6 @@ public interface ActiveInfoStorageCalculator { */ public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception; - /** - * Generate a bootstrapped distribution of what the AIS would look like, - * under a null hypothesis that the previous k values of our - * samples had no relation to the next value in the time-series. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for AIS - * as a mutual information. Basically, the marginal PDFs - * of the past k values, and that of the next value, - * are preserved, while their joint PDF is destroyed, and the - * distribution of AIS under these conditions is generated.

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int[][])}) - * creates random shufflings of the next values for the surrogate AIS - * calculations.

- * - * @param numPermutationsToCheck number of surrogate samples to bootstrap - * to generate the distribution. - * @return the distribution of AIS scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - * @throws Exception - */ - public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception; - - /** - * Generate a bootstrapped distribution of what the AIS would look like, - * under a null hypothesis that the previous k values of our - * samples had no relation to the next value in the time-series. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for AIS - * as a mutual information. Basically, the marginal PDFs - * of the past k values, and that of the next value, - * are preserved, while their joint PDF is destroyed, and the - * distribution of AIS under these conditions is generated.

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int)}) - * allows the user to specify how to construct the surrogates, - * such that repeatable results may be obtained.

- * - * @param newOrderings a specification of how to shuffle the next values - * to create the surrogates to generate the distribution with. The first - * index is the permutation number (i.e. newOrderings.length is the number - * of surrogate samples we use to bootstrap to generate the distribution here.) - * Each array newOrderings[i] should be an array of length L being - * the value returned by {@link #getNumObservations()}, - * containing a permutation of the values in 0..(L-1). - * @return the distribution of AIS scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - * @throws Exception where the length of each permutation in newOrderings - * is not equal to the number L observations that were previously supplied. - */ - public EmpiricalMeasurementDistribution computeSignificance( - int[][] newOrderings) throws Exception; - /** * Set or clear debug mode for extra debug printing to stdout * diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java index 7b64050..3da0fba 100755 --- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java +++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariate.java @@ -18,8 +18,6 @@ package infodynamics.measures.continuous; -import infodynamics.utils.EmpiricalMeasurementDistribution; - /** * Interface for implementations of Active Information Storage estimators on * multivariate continuous time-series data. That is, it is applied to @@ -103,4 +101,6 @@ public interface ActiveInfoStorageCalculatorMultiVariate { */ public void initialise(int dimensions, int k, int tau) throws Exception; + // TODO We seem to be missing a lot of functionality which should be defined here, + // as implemented in the univariate measure. will have to go back and do this. } diff --git a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java index a6b1ead..adf14b2 100755 --- a/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java +++ b/java/source/infodynamics/measures/continuous/ActiveInfoStorageCalculatorMultiVariateViaMutualInfo.java @@ -21,7 +21,6 @@ package infodynamics.measures.continuous; import java.util.Iterator; import java.util.Vector; -import infodynamics.utils.EmpiricalMeasurementDistribution; import infodynamics.utils.MatrixUtils; /** diff --git a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java index ee2d7ab..afee934 100755 --- a/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java +++ b/java/source/infodynamics/measures/continuous/ChannelCalculatorCommon.java @@ -18,7 +18,7 @@ package infodynamics.measures.continuous; -import infodynamics.utils.EmpiricalMeasurementDistribution; +import infodynamics.utils.EmpiricalNullDistributionComputer; /** * A basic interface for calculators computing measures on a channel from a @@ -76,7 +76,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution; * @see ChannelCalculatorMultiVariate * */ -public abstract interface ChannelCalculatorCommon { +public abstract interface ChannelCalculatorCommon extends EmpiricalNullDistributionComputer { /** * Initialise the calculator for (re-)use, with the existing @@ -166,68 +166,6 @@ public abstract interface ChannelCalculatorCommon { */ public double[] computeLocalOfPreviousObservations() throws Exception; - /** - * Generate a bootstrapped distribution of what the channel measure would look like, - * under a null hypothesis that the source values of our - * samples had no relation to the destination value. - * (Precise null hypothesis varies between MI and TE). - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for MI and TE. - *

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int[][])}) - * creates random shufflings of the next values for the surrogate AIS - * calculations.

- * - * @param numPermutationsToCheck number of surrogate samples to bootstrap - * to generate the distribution. - * @return the distribution of channel measure scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - */ - public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception; - - /** - * Generate a bootstrapped distribution of what the channel measure would look like, - * under a null hypothesis that the source values of our - * samples had no relation to the destination value. - * (Precise null hypothesis varies between MI and TE). - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for MI and TE. - *

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int)}) - * allows the user to specify how to construct the surrogates, - * such that repeatable results may be obtained.

- * - * @param newOrderings a specification of how to shuffle the source values - * to create the surrogates to generate the distribution with. The first - * index is the permutation number (i.e. newOrderings.length is the number - * of surrogate samples we use to bootstrap to generate the distribution here.) - * Each array newOrderings[i] should be an array of length N (where - * would be the value returned by {@link #getNumObservations()}), - * containing a permutation of the values in 0..(N-1). - * @return the distribution of channel measure scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - * @throws Exception where the length of each permutation in newOrderings - * is not equal to the number N samples that were previously supplied. - */ - public EmpiricalMeasurementDistribution computeSignificance( - int[][] newOrderings) throws Exception; - /** * Set or clear debug mode for extra debug printing to stdout * diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java index dfa1762..f66da14 100755 --- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java +++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoCalculatorMultiVariate.java @@ -19,6 +19,7 @@ package infodynamics.measures.continuous; import infodynamics.utils.EmpiricalMeasurementDistribution; +import infodynamics.utils.EmpiricalNullDistributionComputer; /** *

Interface for implementations of the conditional mutual information, @@ -75,7 +76,8 @@ import infodynamics.utils.EmpiricalMeasurementDistribution; * @see "T. M. Cover and J. A. Thomas, 'Elements of Information Theory' (John Wiley & Sons, New York, 1991)." */ -public interface ConditionalMutualInfoCalculatorMultiVariate { +public interface ConditionalMutualInfoCalculatorMultiVariate + extends EmpiricalNullDistributionComputer { /** * Initialise the calculator for (re-)use, clearing PDFs, @@ -303,66 +305,53 @@ public interface ConditionalMutualInfoCalculatorMultiVariate { public double[] computeLocalOfPreviousObservations() throws Exception; /** - * Generate a bootstrapped distribution of what the conditional MI would look like, - * under a null hypothesis that the variable identified by - * variableToReorder had no relation to the - * other variable, given the conditional. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for - * conditional MI. Basically, we shuffle the observations of - * variableToReorder against the other variable - * and the conditional. - * This keeps the marginal and joint PDFs of the unshuffled variable - * and conditional the same - * but destroys any correlation between the named variable and the others. + *

Note -- in contrast to the generic computeSignificance() method + * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship + * between variable 2 and the conditional, and shuffles + * variable 1 with respect to these. + * To shuffle variable 2 instead, call {@link #computeSignificance(int, int)}. *

* - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[][], double[][], double[][])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int, int[][])}) - * creates random shufflings of the next values for the surrogate AIS - * calculations.

+ * @inheritDoc + */ + @Override + public EmpiricalMeasurementDistribution computeSignificance( + int numPermutationsToCheck) throws Exception; + + /** + * Defined as per {@link #computeSignificance(int)} except that this method allows + * the user to specify which variable is shuffled (whilst the other has its + * relationship with the conditional preserved). * * @param variableToReorder which variable to shuffle: * 1 for variable 1, 2 for variable 2. * @param numPermutationsToCheck number of surrogate samples to bootstrap * to generate the distribution. * @return the distribution of channel measure scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." + * @see {@link #computeSignificance(int)} * @throws Exception */ public EmpiricalMeasurementDistribution computeSignificance(int variableToReorder, int numPermutationsToCheck) throws Exception; /** - * Generate a bootstrapped distribution of what the conditional MI would look like, - * under a null hypothesis that the variable identified by - * variableToReorder had no relation to the - * other variable, given the conditional. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for - * conditional MI. Basically, we shuffle the observations of - * variableToReorder against the other variable - * and the conditional. - * This keeps the marginal and joint PDFs of the unshuffled variable - * and conditional the same - * but destroys any correlation between the named variable and the others. + *

Note -- in contrast to the generic computeSignificance() method + * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship + * between variable 2 and the conditional, and shuffles + * variable 1 with respect to these. + * To shuffle variable 2 instead, call {@link #computeSignificance(int, int[][])}. *

* - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[][], double[][], double[][])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int, int)}) - * allows the user to specify how to construct the surrogates, - * such that repeatable results may be obtained.

+ * @inheritDoc + */ + @Override + public EmpiricalMeasurementDistribution computeSignificance( + int[][] newOrderings) throws Exception; + + /** + * Defined as per {@link #computeSignificance(int[][])} except that this method allows + * the user to specify which variable is shuffled (whilst the other has its + * relationship with the conditional preserved). * * @param variableToReorder which variable to shuffle: * 1 for variable 1, 2 for variable 2. @@ -375,8 +364,7 @@ public interface ConditionalMutualInfoCalculatorMultiVariate { * would be the value returned by {@link #getNumObservations()}), * containing a permutation of the values in 0..(N-1). * @return the distribution of channel measure scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." + * @see {@link #computeSignificance(int[][])} except that this method all * @throws Exception where the length of each permutation in newOrderings * is not equal to the number N samples that were previously supplied. */ diff --git a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java index 3c6fc0e..bbb0e46 100755 --- a/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java +++ b/java/source/infodynamics/measures/continuous/ConditionalMutualInfoMultiVariateCommon.java @@ -390,6 +390,12 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements vectorOfCondObservations = null; } + @Override + public EmpiricalMeasurementDistribution computeSignificance( + int numPermutationsToCheck) throws Exception { + return computeSignificance(1, numPermutationsToCheck); + } + @Override public EmpiricalMeasurementDistribution computeSignificance( int variableToReorder, int numPermutationsToCheck) throws Exception { @@ -403,6 +409,12 @@ public abstract class ConditionalMutualInfoMultiVariateCommon implements return computeSignificance(variableToReorder, newOrderings); } + @Override + public EmpiricalMeasurementDistribution computeSignificance( + int[][] newOrderings) throws Exception { + return computeSignificance(1, newOrderings); + } + /** *

As described in * {@link ConditionalMutualInfoCalculatorMultiVariate#computeSignificance(int, int[][])} diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java index 3f08e11..946d00e 100755 --- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java +++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculator.java @@ -18,7 +18,7 @@ package infodynamics.measures.continuous; -import infodynamics.utils.EmpiricalMeasurementDistribution; +import infodynamics.utils.EmpiricalNullDistributionComputer; /** * Interface for implementations of @@ -94,7 +94,7 @@ import infodynamics.utils.EmpiricalMeasurementDistribution; * @author Joseph Lizier (email, * www) */ -public interface PredictiveInfoCalculator { +public interface PredictiveInfoCalculator extends EmpiricalNullDistributionComputer { /** * Property name for embedding length k of @@ -282,73 +282,6 @@ public interface PredictiveInfoCalculator { */ public double[] computeLocalUsingPreviousObservations(double[] newObservations) throws Exception; - /** - * Generate a bootstrapped distribution of what the PI would look like, - * under a null hypothesis that the previous k values of our - * samples had no relation to the next k values in the time-series. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for PI - * as a mutual information. Basically, the marginal PDFs - * of the past k values, and that of the next k values, - * are preserved, while their joint PDF is destroyed, and the - * distribution of PI under these conditions is generated.

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int[][])}) - * creates random shufflings of the next vectors for the surrogate PI - * calculations.

- * - * @param numPermutationsToCheck number of surrogate samples to bootstrap - * to generate the distribution. - * @return the distribution of PI scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - * @throws Exception - */ - public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception; - - /** - * Generate a bootstrapped distribution of what the PI would look like, - * under a null hypothesis that the previous k values of our - * samples had no relation to the next k values in the time-series. - * - *

See Section II.E "Statistical significance testing" of - * the JIDT paper below for a description of how this is done for PI - * as a mutual information. Basically, the marginal PDFs - * of the past k values, and that of the next k values, - * are preserved, while their joint PDF is destroyed, and the - * distribution of PI under these conditions is generated.

- * - *

Note that if several disjoint time-series have been added - * as observations using {@link #addObservations(double[])} etc., - * then these separate "trials" will be mixed up in the generation - * of surrogates here.

- * - *

This method (in contrast to {@link #computeSignificance(int)}) - * allows the user to specify how to construct the surrogates, - * such that repeatable results may be obtained.

- * - * @param newOrderings a specification of how to shuffle the next k values vectors - * to create the surrogates to generate the distribution with. The first - * index is the permutation number (i.e. newOrderings.length is the number - * of surrogate samples we use to bootstrap to generate the distribution here.) - * Each array newOrderings[i] should be an array of length L being - * the value returned by {@link #getNumObservations()}, - * containing a permutation of the values in 0..(L-1). - * @return the distribution of PI scores under this null hypothesis. - * @see "J.T. Lizier, 'JIDT: An information-theoretic - * toolkit for studying the dynamics of complex systems', 2014." - * @throws Exception where the length of each permutation in newOrderings - * is not equal to the number L observations that were previously supplied. - */ - public EmpiricalMeasurementDistribution computeSignificance( - int[][] newOrderings) throws Exception; - /** * Set or clear debug mode for extra debug printing to stdout * diff --git a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java index 835839b..108292a 100755 --- a/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java +++ b/java/source/infodynamics/measures/continuous/PredictiveInfoCalculatorViaMutualInfo.java @@ -417,7 +417,7 @@ public class PredictiveInfoCalculatorViaMutualInfo implements } /* (non-Javadoc) - * @see infodynamics.measures.continuous.PredictiveInfoCalculator#computeSignificance(int) + * @see infodynamics.utils.EmpiricalNullDistributionComputer#computeSignificance(int) */ @Override public EmpiricalMeasurementDistribution computeSignificance( @@ -426,7 +426,7 @@ public class PredictiveInfoCalculatorViaMutualInfo implements } /* (non-Javadoc) - * @see infodynamics.measures.continuous.PredictiveInfoCalculator#computeSignificance(int[][]) + * @see infodynamics.utils.EmpiricalNullDistributionComputer#computeSignificance(int[][]) */ @Override public EmpiricalMeasurementDistribution computeSignificance( diff --git a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java index cbed332..55e6f34 100755 --- a/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java +++ b/java/source/infodynamics/measures/discrete/ConditionalMutualInformationCalculatorDiscrete.java @@ -406,7 +406,7 @@ public class ConditionalMutualInformationCalculatorDiscrete /** *

Note -- in contrast to the generic computeSignificance() method - * described below, this method for a conditional MI calculator currently fixes the relationship + * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship * between variable 2 and the conditional, and shuffles * variable 1 with respect to these. *

@@ -422,7 +422,7 @@ public class ConditionalMutualInformationCalculatorDiscrete /** *

Note -- in contrast to the generic computeSignificance() method - * described below, this method for a conditional MI calculator currently fixes the relationship + * described in the documentation below, this method for a conditional MI calculator currently fixes the relationship * between variable 2 and the conditional, and shuffles * variable 1 with respect to these. *

diff --git a/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java index 66e535d..9ef66bf 100644 --- a/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java +++ b/java/source/infodynamics/utils/EmpiricalNullDistributionComputer.java @@ -58,7 +58,7 @@ public interface EmpiricalNullDistributionComputer { * toolkit for studying the dynamics of complex systems', 2014." * @return the empirical distribution of measure scores under this null hypothesis. */ - public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck); + public EmpiricalMeasurementDistribution computeSignificance(int numPermutationsToCheck) throws Exception; /** * Generate an empirical (bootstrapped) distribution of what the given measure would look like, @@ -96,7 +96,9 @@ public interface EmpiricalNullDistributionComputer { * @see "J.T. Lizier, 'JIDT: An information-theoretic * toolkit for studying the dynamics of complex systems', 2014." * @return the empirical distribution of measure scores under this null hypothesis. + * @throws Exception where e.g. the newOrderings don't supply arrays of the correct + * length matching the number of observations that we have. */ - public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings); + public EmpiricalMeasurementDistribution computeSignificance(int[][] newOrderings) throws Exception; }