From d1bba0e4c1c5b4ddce98c162bbc04c015fd084cb Mon Sep 17 00:00:00 2001 From: jlizier Date: Fri, 27 Apr 2018 11:20:25 +1000 Subject: [PATCH] Adding computeLocalOfPreviousObservations() to continuous Entropy estimator interface and all underlying implementations. Added unit tests that locals should average back ok. Also fixing multivariate Entropy estimator interface to implement the univariate interface also (and fixing underlying implementations to comply) --- .../continuous/EntropyCalculator.java | 15 ++++- .../EntropyCalculatorMultiVariate.java | 15 +---- .../gaussian/EntropyCalculatorGaussian.java | 58 +++++++++++++++++-- ...EntropyCalculatorMultiVariateGaussian.java | 17 +++++- .../kernel/EntropyCalculatorKernel.java | 23 +++++++- .../EntropyCalculatorMultiVariateKernel.java | 17 +++++- ...tropyCalculatorMultiVariateKozachenko.java | 4 +- .../EntropyCalculatorGaussianTest.java | 19 +++++- .../kernel/EntropyCalculatorKernelTest.java | 22 +++++++ 9 files changed, 160 insertions(+), 30 deletions(-) create mode 100644 java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java diff --git a/java/source/infodynamics/measures/continuous/EntropyCalculator.java b/java/source/infodynamics/measures/continuous/EntropyCalculator.java index 747a0c9..ee38a4c 100755 --- a/java/source/infodynamics/measures/continuous/EntropyCalculator.java +++ b/java/source/infodynamics/measures/continuous/EntropyCalculator.java @@ -56,7 +56,20 @@ public interface EntropyCalculator extends InfoMeasureCalculatorContinuous { * observations that are used (they are not accumulated). * * @param observations array of (univariate) samples + * @throws Exception */ - public void setObservations(double observations[]); + public void setObservations(double observations[]) throws Exception; + + /** + * Compute the local entropy values for each of the + * previously-supplied samples. + * + *

PDFs are computed using all of the previously supplied + * observations.

+ * + * @return the array of local entropy values corresponding to each + * previously supplied observation. + */ + public double[] computeLocalOfPreviousObservations() throws Exception; } diff --git a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java index eb4e5f3..919710d 100755 --- a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java +++ b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java @@ -55,7 +55,7 @@ Theory' (John Wiley & Sons, New York, 1991). * www) */ public interface EntropyCalculatorMultiVariate - extends InfoMeasureCalculatorContinuous { + extends EntropyCalculator { /** * Property name for the number of dimensions @@ -82,6 +82,7 @@ public interface EntropyCalculatorMultiVariate * @param propertyValue value of the property * @throws Exception for invalid property values */ + @Override public void setProperty(String propertyName, String propertyValue) throws Exception; /** @@ -122,16 +123,4 @@ public interface EntropyCalculatorMultiVariate */ public double[] computeLocalUsingPreviousObservations(double newObservations[][]) throws Exception; - /** - * Compute the local entropy values for each of the - * previously-supplied samples. - * - *

PDFs are computed using all of the previously supplied - * observations.

- * - * @return the array of local entropy values corresponding to each - * previously supplied observation. - */ - public double[] computeLocalOfPreviousObservations() throws Exception; - } diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java index 4064af9..d6a757b 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java @@ -60,9 +60,10 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { protected boolean debug; /** - * Total number of observations supplied. + * The set of observations, retained in case the user wants to retrieve the local + * entropy values of these */ - protected int totalObservations; + protected double[] observations; /** * Store the last computed average Entropy @@ -78,13 +79,14 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { @Override public void initialise() { - // Nothing to do + observations = null; + variance = 0; } public void setObservations(double[] observations) { variance = MatrixUtils.stdDev(observations); variance *= variance; - totalObservations = observations.length; + this.observations = observations; } /** @@ -94,8 +96,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { * * @param variance the variance of the univariate distribution. */ - public void setVariance(double variance) { + public void setVariance(double variance) throws Exception { this.variance = variance; + if (variance < 0) { + throw new Exception("Cannot have negative variance"); + } + observations = null; } /** @@ -118,6 +124,41 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { return lastAverage; } + /** + * @throws Exception if {@link #setVariance(double)} was used previously instead + * of {@link #setObservations(double[][])} + */ + public double[] computeLocalOfPreviousObservations() throws Exception { + if (observations == null) { + throw new Exception("Cannot compute local values since no observations were supplied"); + } + + // Check that the variance was non-zero: + if (variance == 0) { + throw new Exception("variance is not positive - cannot compute local entropies"); + } + // Now we are clear to take the variance inverse + double invVariance = 1.0 / variance; + double mean = MatrixUtils.mean(observations); + + double[] localValues = new double[observations.length]; + for (int t = 0; t < observations.length; t++) { + double deviationFromMean = observations[t] - mean; + // Computing PDF + // (see the PDF defined at the wikipedia page referenced in the method header) + double jointExpArg = deviationFromMean * deviationFromMean * invVariance; + double pJoint = Math.exp(-0.5 * jointExpArg) / + Math.sqrt(2.0 * Math.PI * variance); + localValues[t] = - Math.log(pJoint); + } + + // Don't set average if this was the previously supplied observations, + // since it won't be the same as what would have been computed + // analytically. + + return localValues; + } + @Override public void setDebug(boolean debug) { this.debug = debug; @@ -144,7 +185,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { @Override public int getNumObservations() throws Exception { - return totalObservations; + if (observations == null) { + throw new Exception("Cannot return number of observations because either " + + "this calculator has not had observations supplied or " + + "the user supplied the variance instead of observations"); + } + return observations.length; } @Override diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java index aaf0504..818fb83 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java @@ -130,6 +130,19 @@ public class EntropyCalculatorMultiVariateGaussian this.observations = observations; } + /** + * @throws Exception where the observations do not match the expected number of + * dimensions, or covariance matrix is not positive definite (reflecting + * redundant variables in the observations) + */ + @Override + public void setObservations(double[] observations) throws Exception { + if (dimensions != 1) { + throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions)); + } + setObservations(MatrixUtils.reshape(observations, observations.length, 1)); + } + /** *

Set the covariance of the distribution for which we will compute the * entropy.

@@ -242,8 +255,10 @@ public class EntropyCalculatorMultiVariateGaussian } /** + * @see Multivariate normal distribution on Wikipedia * @return an array of local values in nats (NOT bits). */ + @Override public double[] computeLocalUsingPreviousObservations(double[][] states) throws Exception { if (means == null) { @@ -268,7 +283,6 @@ public class EntropyCalculatorMultiVariateGaussian double[][] invCovariance = MatrixUtils.solveViaCholeskyResult(L, MatrixUtils.identityMatrix(L.length)); - // If we have a time delay, slide the local values double[] localValues = new double[states.length]; for (int t = 0; t < states.length; t++) { double[] deviationsFromMean = @@ -297,6 +311,7 @@ public class EntropyCalculatorMultiVariateGaussian * {@link #setCovarianceAndMeans(double[][], double[])} were used previously instead * of {@link #setObservations(double[][])} */ + @Override public double[] computeLocalOfPreviousObservations() throws Exception { if (observations == null) { throw new Exception("Cannot compute local values since no observations were supplied"); diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java index d1e8e10..5d20e8b 100755 --- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java +++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java @@ -132,8 +132,8 @@ public class EntropyCalculatorKernel implements EntropyCalculator { double entropy = 0.0; for (int t = 0; t < observations.length; t++) { double prob = svke.getProbability(observations[t]); - double cont = Math.log(prob); - entropy -= cont; + double cont = -Math.log(prob); + entropy += cont; if (debug) { System.out.println(t + ": p(" + observations[t] + ")= " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + @@ -144,6 +144,25 @@ public class EntropyCalculatorKernel implements EntropyCalculator { return lastAverage; } + @Override + public double[] computeLocalOfPreviousObservations() throws Exception { + double[] localEntropies = new double[observations.length]; + double entropy = 0.0; + for (int t = 0; t < observations.length; t++) { + double prob = svke.getProbability(observations[t]); + double cont = -Math.log(prob); + localEntropies[t] = cont / Math.log(2.0); + entropy += cont; + if (debug) { + System.out.println(t + ": p(" + observations[t] + ")= " + + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + + (entropy/Math.log(2.0))); + } + } + lastAverage = entropy / (double) totalObservations / Math.log(2.0); + return localEntropies; + } + @Override public void setDebug(boolean debug) { this.debug = debug; diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java index e039d4f..bbe7b29 100755 --- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java +++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java @@ -19,6 +19,7 @@ package infodynamics.measures.continuous.kernel; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; +import infodynamics.utils.MatrixUtils; /** *

Computes the differential entropy of a given set of observations @@ -141,6 +142,18 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul this.observations = observations; } + /** + * @throws Exception where the observations do not match the expected number of + * dimensions + */ + @Override + public void setObservations(double[] observations) throws Exception { + if (dimensions != 1) { + throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions)); + } + setObservations(MatrixUtils.reshape(observations, observations.length, 1)); + } + @Override public double computeAverageLocalOfObservations() { double entropy = 0.0; @@ -183,13 +196,13 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul for (int b = 0; b < states.length; b++) { double prob = mvke.getProbability(states[b]); double cont = -Math.log(prob); - localEntropy[b] = cont; + localEntropy[b] = cont / Math.log(2.0); entropy += cont; if (debug) { System.out.println(b + ": " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0))); } } - entropy /= (double) totalObservations / Math.log(2.0); + entropy = entropy / (double) totalObservations / Math.log(2.0); // Don't use /= as I'm not sure of the order of operations it applies. if (isPreviousObservations) { lastEntropy = entropy; } diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index ba67fc1..2586fa1 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -19,10 +19,8 @@ package infodynamics.measures.continuous.kozachenko; import java.util.Random; -import java.util.Iterator; import java.util.Vector; -import infodynamics.measures.continuous.EntropyCalculator; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; import infodynamics.utils.EuclideanUtils; import infodynamics.utils.MathsUtils; @@ -62,7 +60,7 @@ import infodynamics.utils.MatrixUtils; * www) */ public class EntropyCalculatorMultiVariateKozachenko - implements EntropyCalculator, EntropyCalculatorMultiVariate { + implements EntropyCalculatorMultiVariate { /** * Total number of observations supplied. diff --git a/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java b/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java index f9d67f9..a1355d7 100755 --- a/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java +++ b/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java @@ -24,7 +24,7 @@ import junit.framework.TestCase; public class EntropyCalculatorGaussianTest extends TestCase { - public void testVarianceSetting() { + public void testVarianceSetting() throws Exception { EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); entcalc.initialise(); double variance = 3.45567; @@ -43,7 +43,7 @@ public class EntropyCalculatorGaussianTest extends TestCase { assertEquals(variance, entcalc.variance); } - public void testEntropyCalculation() { + public void testEntropyCalculation() throws Exception { EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); entcalc.initialise(); double variance = 3.45567; @@ -51,4 +51,19 @@ public class EntropyCalculatorGaussianTest extends TestCase { double expectedEntropy = 0.5 * Math.log(2.0*Math.PI*Math.E*variance); assertEquals(expectedEntropy, entcalc.computeAverageLocalOfObservations()); } + + public void testLocalEntropiesAverage() throws Exception { + RandomGenerator rg = new RandomGenerator(); + double[] data = rg.generateNormalData(100, 0, 1); + EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); + entcalc.initialise(); + entcalc.setObservations(data); + double entropy = entcalc.computeAverageLocalOfObservations(); + double[] localEntropies = entcalc.computeLocalOfPreviousObservations(); + double avgLocal = MatrixUtils.mean(localEntropies); + // There are many sources of numerical noise in the combination of so many + // local entropy calculations here (in comparison to the average calculation + // which only comes from the variance), so we need to leave a wide tolerance + assertEquals(entropy, avgLocal, 0.02); + } } diff --git a/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java b/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java new file mode 100644 index 0000000..a328ca8 --- /dev/null +++ b/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java @@ -0,0 +1,22 @@ +package infodynamics.measures.continuous.kernel; + +import infodynamics.utils.MatrixUtils; +import infodynamics.utils.RandomGenerator; +import junit.framework.TestCase; + +public class EntropyCalculatorKernelTest extends TestCase { + + public void testLocalEntropiesAverage() throws Exception { + RandomGenerator rg = new RandomGenerator(); + double[] data = rg.generateNormalData(100, 0, 1); + EntropyCalculatorKernel entcalc = new EntropyCalculatorKernel(); + entcalc.initialise(0.5); + entcalc.setObservations(data); + double entropy = entcalc.computeAverageLocalOfObservations(); + double[] localEntropies = entcalc.computeLocalOfPreviousObservations(); + double avgLocal = MatrixUtils.mean(localEntropies); + // There are many sources of numerical noise in the combination of so many + // local entropy calculations here, so we need to leave a wide tolerance + assertEquals(entropy, avgLocal, 0.000001); + } +}