PDFs are computed using all of the previously supplied + * observations.
+ * + * @return the array of local entropy values corresponding to each + * previously supplied observation. + */ + public double[] computeLocalOfPreviousObservations() throws Exception; } diff --git a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java index eb4e5f3..919710d 100755 --- a/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java +++ b/java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java @@ -55,7 +55,7 @@ Theory' (John Wiley & Sons, New York, 1991). * www) */ public interface EntropyCalculatorMultiVariate - extends InfoMeasureCalculatorContinuous { + extends EntropyCalculator { /** * Property name for the number of dimensions @@ -82,6 +82,7 @@ public interface EntropyCalculatorMultiVariate * @param propertyValue value of the property * @throws Exception for invalid property values */ + @Override public void setProperty(String propertyName, String propertyValue) throws Exception; /** @@ -122,16 +123,4 @@ public interface EntropyCalculatorMultiVariate */ public double[] computeLocalUsingPreviousObservations(double newObservations[][]) throws Exception; - /** - * Compute the local entropy values for each of the - * previously-supplied samples. - * - *PDFs are computed using all of the previously supplied - * observations.
- * - * @return the array of local entropy values corresponding to each - * previously supplied observation. - */ - public double[] computeLocalOfPreviousObservations() throws Exception; - } diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java index 4064af9..d6a757b 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java @@ -60,9 +60,10 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { protected boolean debug; /** - * Total number of observations supplied. + * The set of observations, retained in case the user wants to retrieve the local + * entropy values of these */ - protected int totalObservations; + protected double[] observations; /** * Store the last computed average Entropy @@ -78,13 +79,14 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { @Override public void initialise() { - // Nothing to do + observations = null; + variance = 0; } public void setObservations(double[] observations) { variance = MatrixUtils.stdDev(observations); variance *= variance; - totalObservations = observations.length; + this.observations = observations; } /** @@ -94,8 +96,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { * * @param variance the variance of the univariate distribution. */ - public void setVariance(double variance) { + public void setVariance(double variance) throws Exception { this.variance = variance; + if (variance < 0) { + throw new Exception("Cannot have negative variance"); + } + observations = null; } /** @@ -118,6 +124,41 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { return lastAverage; } + /** + * @throws Exception if {@link #setVariance(double)} was used previously instead + * of {@link #setObservations(double[][])} + */ + public double[] computeLocalOfPreviousObservations() throws Exception { + if (observations == null) { + throw new Exception("Cannot compute local values since no observations were supplied"); + } + + // Check that the variance was non-zero: + if (variance == 0) { + throw new Exception("variance is not positive - cannot compute local entropies"); + } + // Now we are clear to take the variance inverse + double invVariance = 1.0 / variance; + double mean = MatrixUtils.mean(observations); + + double[] localValues = new double[observations.length]; + for (int t = 0; t < observations.length; t++) { + double deviationFromMean = observations[t] - mean; + // Computing PDF + // (see the PDF defined at the wikipedia page referenced in the method header) + double jointExpArg = deviationFromMean * deviationFromMean * invVariance; + double pJoint = Math.exp(-0.5 * jointExpArg) / + Math.sqrt(2.0 * Math.PI * variance); + localValues[t] = - Math.log(pJoint); + } + + // Don't set average if this was the previously supplied observations, + // since it won't be the same as what would have been computed + // analytically. + + return localValues; + } + @Override public void setDebug(boolean debug) { this.debug = debug; @@ -144,7 +185,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator { @Override public int getNumObservations() throws Exception { - return totalObservations; + if (observations == null) { + throw new Exception("Cannot return number of observations because either " + + "this calculator has not had observations supplied or " + + "the user supplied the variance instead of observations"); + } + return observations.length; } @Override diff --git a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java index aaf0504..818fb83 100755 --- a/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java +++ b/java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java @@ -130,6 +130,19 @@ public class EntropyCalculatorMultiVariateGaussian this.observations = observations; } + /** + * @throws Exception where the observations do not match the expected number of + * dimensions, or covariance matrix is not positive definite (reflecting + * redundant variables in the observations) + */ + @Override + public void setObservations(double[] observations) throws Exception { + if (dimensions != 1) { + throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions)); + } + setObservations(MatrixUtils.reshape(observations, observations.length, 1)); + } + /** *Set the covariance of the distribution for which we will compute the * entropy.
@@ -242,8 +255,10 @@ public class EntropyCalculatorMultiVariateGaussian } /** + * @see Multivariate normal distribution on Wikipedia * @return an array of local values in nats (NOT bits). */ + @Override public double[] computeLocalUsingPreviousObservations(double[][] states) throws Exception { if (means == null) { @@ -268,7 +283,6 @@ public class EntropyCalculatorMultiVariateGaussian double[][] invCovariance = MatrixUtils.solveViaCholeskyResult(L, MatrixUtils.identityMatrix(L.length)); - // If we have a time delay, slide the local values double[] localValues = new double[states.length]; for (int t = 0; t < states.length; t++) { double[] deviationsFromMean = @@ -297,6 +311,7 @@ public class EntropyCalculatorMultiVariateGaussian * {@link #setCovarianceAndMeans(double[][], double[])} were used previously instead * of {@link #setObservations(double[][])} */ + @Override public double[] computeLocalOfPreviousObservations() throws Exception { if (observations == null) { throw new Exception("Cannot compute local values since no observations were supplied"); diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java index d1e8e10..5d20e8b 100755 --- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java +++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java @@ -132,8 +132,8 @@ public class EntropyCalculatorKernel implements EntropyCalculator { double entropy = 0.0; for (int t = 0; t < observations.length; t++) { double prob = svke.getProbability(observations[t]); - double cont = Math.log(prob); - entropy -= cont; + double cont = -Math.log(prob); + entropy += cont; if (debug) { System.out.println(t + ": p(" + observations[t] + ")= " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + @@ -144,6 +144,25 @@ public class EntropyCalculatorKernel implements EntropyCalculator { return lastAverage; } + @Override + public double[] computeLocalOfPreviousObservations() throws Exception { + double[] localEntropies = new double[observations.length]; + double entropy = 0.0; + for (int t = 0; t < observations.length; t++) { + double prob = svke.getProbability(observations[t]); + double cont = -Math.log(prob); + localEntropies[t] = cont / Math.log(2.0); + entropy += cont; + if (debug) { + System.out.println(t + ": p(" + observations[t] + ")= " + + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + + (entropy/Math.log(2.0))); + } + } + lastAverage = entropy / (double) totalObservations / Math.log(2.0); + return localEntropies; + } + @Override public void setDebug(boolean debug) { this.debug = debug; diff --git a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java index e039d4f..bbe7b29 100755 --- a/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java +++ b/java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java @@ -19,6 +19,7 @@ package infodynamics.measures.continuous.kernel; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; +import infodynamics.utils.MatrixUtils; /** *Computes the differential entropy of a given set of observations @@ -141,6 +142,18 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul this.observations = observations; } + /** + * @throws Exception where the observations do not match the expected number of + * dimensions + */ + @Override + public void setObservations(double[] observations) throws Exception { + if (dimensions != 1) { + throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions)); + } + setObservations(MatrixUtils.reshape(observations, observations.length, 1)); + } + @Override public double computeAverageLocalOfObservations() { double entropy = 0.0; @@ -183,13 +196,13 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul for (int b = 0; b < states.length; b++) { double prob = mvke.getProbability(states[b]); double cont = -Math.log(prob); - localEntropy[b] = cont; + localEntropy[b] = cont / Math.log(2.0); entropy += cont; if (debug) { System.out.println(b + ": " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0))); } } - entropy /= (double) totalObservations / Math.log(2.0); + entropy = entropy / (double) totalObservations / Math.log(2.0); // Don't use /= as I'm not sure of the order of operations it applies. if (isPreviousObservations) { lastEntropy = entropy; } diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index ba67fc1..2586fa1 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -19,10 +19,8 @@ package infodynamics.measures.continuous.kozachenko; import java.util.Random; -import java.util.Iterator; import java.util.Vector; -import infodynamics.measures.continuous.EntropyCalculator; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; import infodynamics.utils.EuclideanUtils; import infodynamics.utils.MathsUtils; @@ -62,7 +60,7 @@ import infodynamics.utils.MatrixUtils; * www) */ public class EntropyCalculatorMultiVariateKozachenko - implements EntropyCalculator, EntropyCalculatorMultiVariate { + implements EntropyCalculatorMultiVariate { /** * Total number of observations supplied. diff --git a/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java b/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java index f9d67f9..a1355d7 100755 --- a/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java +++ b/java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java @@ -24,7 +24,7 @@ import junit.framework.TestCase; public class EntropyCalculatorGaussianTest extends TestCase { - public void testVarianceSetting() { + public void testVarianceSetting() throws Exception { EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); entcalc.initialise(); double variance = 3.45567; @@ -43,7 +43,7 @@ public class EntropyCalculatorGaussianTest extends TestCase { assertEquals(variance, entcalc.variance); } - public void testEntropyCalculation() { + public void testEntropyCalculation() throws Exception { EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); entcalc.initialise(); double variance = 3.45567; @@ -51,4 +51,19 @@ public class EntropyCalculatorGaussianTest extends TestCase { double expectedEntropy = 0.5 * Math.log(2.0*Math.PI*Math.E*variance); assertEquals(expectedEntropy, entcalc.computeAverageLocalOfObservations()); } + + public void testLocalEntropiesAverage() throws Exception { + RandomGenerator rg = new RandomGenerator(); + double[] data = rg.generateNormalData(100, 0, 1); + EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian(); + entcalc.initialise(); + entcalc.setObservations(data); + double entropy = entcalc.computeAverageLocalOfObservations(); + double[] localEntropies = entcalc.computeLocalOfPreviousObservations(); + double avgLocal = MatrixUtils.mean(localEntropies); + // There are many sources of numerical noise in the combination of so many + // local entropy calculations here (in comparison to the average calculation + // which only comes from the variance), so we need to leave a wide tolerance + assertEquals(entropy, avgLocal, 0.02); + } } diff --git a/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java b/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java new file mode 100644 index 0000000..a328ca8 --- /dev/null +++ b/java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java @@ -0,0 +1,22 @@ +package infodynamics.measures.continuous.kernel; + +import infodynamics.utils.MatrixUtils; +import infodynamics.utils.RandomGenerator; +import junit.framework.TestCase; + +public class EntropyCalculatorKernelTest extends TestCase { + + public void testLocalEntropiesAverage() throws Exception { + RandomGenerator rg = new RandomGenerator(); + double[] data = rg.generateNormalData(100, 0, 1); + EntropyCalculatorKernel entcalc = new EntropyCalculatorKernel(); + entcalc.initialise(0.5); + entcalc.setObservations(data); + double entropy = entcalc.computeAverageLocalOfObservations(); + double[] localEntropies = entcalc.computeLocalOfPreviousObservations(); + double avgLocal = MatrixUtils.mean(localEntropies); + // There are many sources of numerical noise in the combination of so many + // local entropy calculations here, so we need to leave a wide tolerance + assertEquals(entropy, avgLocal, 0.000001); + } +}