natas_hw
/
jidt
mirror of https://github.com/jlizier/jidt
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding computeLocalOfPreviousObservations() to continuous Entropy estimator interface and all underlying implementations. Added unit tests that locals should average back ok. Also fixing multivariate Entropy estimator interface to implement the univariate interface also (and fixing underlying implementations to comply)

This commit is contained in:
jlizier 2018-04-27 11:20:25 +10:00
parent bf0062b902
commit d1bba0e4c1
9 changed files with 160 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
java/source/infodynamics/measures/continuous/EntropyCalculator.java
View File

@ -56,7 +56,20 @@ public interface EntropyCalculator extends InfoMeasureCalculatorContinuous {
* observations that are used (they are not accumulated).
*
* @param observations array of (univariate) samples
* @throws Exception
*/
public void setObservations(double observations[]);
public void setObservations(double observations[]) throws Exception;
/**
* Compute the local entropy values for each of the
* previously-supplied samples.
*
* <p>PDFs are computed using all of the previously supplied
* observations.</p>
*
* @return the array of local entropy values corresponding to each
* previously supplied observation.
*/
public double[] computeLocalOfPreviousObservations() throws Exception;
}

15
java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java
View File

@ -55,7 +55,7 @@ Theory' (John Wiley & Sons, New York, 1991).</li>
* <a href="http://lizier.me/joseph/">www</a>)
*/
public interface EntropyCalculatorMultiVariate
extends InfoMeasureCalculatorContinuous {
extends EntropyCalculator {
/**
* Property name for the number of dimensions
@ -82,6 +82,7 @@ public interface EntropyCalculatorMultiVariate
* @param propertyValue value of the property
* @throws Exception for invalid property values
*/
@Override
public void setProperty(String propertyName, String propertyValue) throws Exception;
/**
@ -122,16 +123,4 @@ public interface EntropyCalculatorMultiVariate
*/
public double[] computeLocalUsingPreviousObservations(double newObservations[][]) throws Exception;
/**
* Compute the local entropy values for each of the
* previously-supplied samples.
*
* <p>PDFs are computed using all of the previously supplied
* observations.</p>
*
* @return the array of local entropy values corresponding to each
* previously supplied observation.
*/
public double[] computeLocalOfPreviousObservations() throws Exception;
}

58
java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussian.java
View File

@ -60,9 +60,10 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
protected boolean debug;
/**
* Total number of observations supplied.
* The set of observations, retained in case the user wants to retrieve the local
* entropy values of these
*/
protected int totalObservations;
protected double[] observations;
/**
* Store the last computed average Entropy
@ -78,13 +79,14 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
@Override
public void initialise() {
// Nothing to do
observations = null;
variance = 0;
}
public void setObservations(double[] observations) {
variance = MatrixUtils.stdDev(observations);
variance *= variance;
totalObservations = observations.length;
this.observations = observations;
}
/**
@ -94,8 +96,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
*
* @param variance the variance of the univariate distribution.
*/
public void setVariance(double variance) {
public void setVariance(double variance) throws Exception {
this.variance = variance;
if (variance < 0) {
throw new Exception("Cannot have negative variance");
}
observations = null;
}
/**
@ -118,6 +124,41 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
return lastAverage;
}
/**
* @throws Exception if {@link #setVariance(double)} was used previously instead
* of {@link #setObservations(double[][])}
*/
public double[] computeLocalOfPreviousObservations() throws Exception {
if (observations == null) {
throw new Exception("Cannot compute local values since no observations were supplied");
}
// Check that the variance was non-zero:
if (variance == 0) {
throw new Exception("variance is not positive - cannot compute local entropies");
}
// Now we are clear to take the variance inverse
double invVariance = 1.0 / variance;
double mean = MatrixUtils.mean(observations);
double[] localValues = new double[observations.length];
for (int t = 0; t < observations.length; t++) {
double deviationFromMean = observations[t] - mean;
// Computing PDF
// (see the PDF defined at the wikipedia page referenced in the method header)
double jointExpArg = deviationFromMean * deviationFromMean * invVariance;
double pJoint = Math.exp(-0.5 * jointExpArg) /
Math.sqrt(2.0 * Math.PI * variance);
localValues[t] = - Math.log(pJoint);
}
// Don't set average if this was the previously supplied observations,
// since it won't be the same as what would have been computed
// analytically.
return localValues;
}
@Override
public void setDebug(boolean debug) {
this.debug = debug;
@ -144,7 +185,12 @@ public class EntropyCalculatorGaussian implements EntropyCalculator {
@Override
public int getNumObservations() throws Exception {
return totalObservations;
if (observations == null) {
throw new Exception("Cannot return number of observations because either " +
"this calculator has not had observations supplied or " +
"the user supplied the variance instead of observations");
}
return observations.length;
}
@Override

17
java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java
View File

@ -130,6 +130,19 @@ public class EntropyCalculatorMultiVariateGaussian
this.observations = observations;
}
/**
* @throws Exception where the observations do not match the expected number of
* dimensions, or covariance matrix is not positive definite (reflecting
* redundant variables in the observations)
*/
@Override
public void setObservations(double[] observations) throws Exception {
if (dimensions != 1) {
throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions));
}
setObservations(MatrixUtils.reshape(observations, observations.length, 1));
}
/**
* <p>Set the covariance of the distribution for which we will compute the
* entropy.</p>
@ -242,8 +255,10 @@ public class EntropyCalculatorMultiVariateGaussian
}
/**
* @see <a href="http://en.wikipedia.org/wiki/Multivariate_normal_distribution">Multivariate normal distribution on Wikipedia</a>
* @return an array of local values in nats (NOT bits).
*/
@Override
public double[] computeLocalUsingPreviousObservations(double[][] states)
throws Exception {
if (means == null) {
@ -268,7 +283,6 @@ public class EntropyCalculatorMultiVariateGaussian
double[][] invCovariance = MatrixUtils.solveViaCholeskyResult(L,
MatrixUtils.identityMatrix(L.length));
// If we have a time delay, slide the local values
double[] localValues = new double[states.length];
for (int t = 0; t < states.length; t++) {
double[] deviationsFromMean =
@ -297,6 +311,7 @@ public class EntropyCalculatorMultiVariateGaussian
* {@link #setCovarianceAndMeans(double[][], double[])} were used previously instead
* of {@link #setObservations(double[][])}
*/
@Override
public double[] computeLocalOfPreviousObservations() throws Exception {
if (observations == null) {
throw new Exception("Cannot compute local values since no observations were supplied");

23
java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorKernel.java
View File

@ -132,8 +132,8 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
double entropy = 0.0;
for (int t = 0; t < observations.length; t++) {
double prob = svke.getProbability(observations[t]);
double cont = Math.log(prob);
entropy -= cont;
double cont = -Math.log(prob);
entropy += cont;
if (debug) {
System.out.println(t + ": p(" + observations[t] + ")= " +
prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " +
@ -144,6 +144,25 @@ public class EntropyCalculatorKernel implements EntropyCalculator {
return lastAverage;
}
@Override
public double[] computeLocalOfPreviousObservations() throws Exception {
double[] localEntropies = new double[observations.length];
double entropy = 0.0;
for (int t = 0; t < observations.length; t++) {
double prob = svke.getProbability(observations[t]);
double cont = -Math.log(prob);
localEntropies[t] = cont / Math.log(2.0);
entropy += cont;
if (debug) {
System.out.println(t + ": p(" + observations[t] + ")= " +
prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " +
(entropy/Math.log(2.0)));
}
}
lastAverage = entropy / (double) totalObservations / Math.log(2.0);
return localEntropies;
}
@Override
public void setDebug(boolean debug) {
this.debug = debug;

17
java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java
View File

@ -19,6 +19,7 @@
package infodynamics.measures.continuous.kernel;
import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
import infodynamics.utils.MatrixUtils;
/**
* <p>Computes the differential entropy of a given set of observations
@ -141,6 +142,18 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
this.observations = observations;
}
/**
* @throws Exception where the observations do not match the expected number of
* dimensions
*/
@Override
public void setObservations(double[] observations) throws Exception {
if (dimensions != 1) {
throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions));
}
setObservations(MatrixUtils.reshape(observations, observations.length, 1));
}
@Override
public double computeAverageLocalOfObservations() {
double entropy = 0.0;
@ -183,13 +196,13 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
for (int b = 0; b < states.length; b++) {
double prob = mvke.getProbability(states[b]);
double cont = -Math.log(prob);
localEntropy[b] = cont;
localEntropy[b] = cont / Math.log(2.0);
entropy += cont;
if (debug) {
System.out.println(b + ": " + prob + " -> " + (cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0)));
}
}
entropy /= (double) totalObservations / Math.log(2.0);
entropy = entropy / (double) totalObservations / Math.log(2.0); // Don't use /= as I'm not sure of the order of operations it applies.
if (isPreviousObservations) {
lastEntropy = entropy;
}

4
java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java
View File

@ -19,10 +19,8 @@
package infodynamics.measures.continuous.kozachenko;
import java.util.Random;
import java.util.Iterator;
import java.util.Vector;
import infodynamics.measures.continuous.EntropyCalculator;
import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
import infodynamics.utils.EuclideanUtils;
import infodynamics.utils.MathsUtils;
@ -62,7 +60,7 @@ import infodynamics.utils.MatrixUtils;
* <a href="http://lizier.me/joseph/">www</a>)
*/
public class EntropyCalculatorMultiVariateKozachenko
implements EntropyCalculator, EntropyCalculatorMultiVariate {
implements EntropyCalculatorMultiVariate {
/**
* Total number of observations supplied.

19
java/unittests/infodynamics/measures/continuous/gaussian/EntropyCalculatorGaussianTest.java
View File

@ -24,7 +24,7 @@ import junit.framework.TestCase;
public class EntropyCalculatorGaussianTest extends TestCase {
public void testVarianceSetting() {
public void testVarianceSetting() throws Exception {
EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian();
entcalc.initialise();
double variance = 3.45567;
@ -43,7 +43,7 @@ public class EntropyCalculatorGaussianTest extends TestCase {
assertEquals(variance, entcalc.variance);
}
public void testEntropyCalculation() {
public void testEntropyCalculation() throws Exception {
EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian();
entcalc.initialise();
double variance = 3.45567;
@ -51,4 +51,19 @@ public class EntropyCalculatorGaussianTest extends TestCase {
double expectedEntropy = 0.5 * Math.log(2.0*Math.PI*Math.E*variance);
assertEquals(expectedEntropy, entcalc.computeAverageLocalOfObservations());
}
public void testLocalEntropiesAverage() throws Exception {
RandomGenerator rg = new RandomGenerator();
double[] data = rg.generateNormalData(100, 0, 1);
EntropyCalculatorGaussian entcalc = new EntropyCalculatorGaussian();
entcalc.initialise();
entcalc.setObservations(data);
double entropy = entcalc.computeAverageLocalOfObservations();
double[] localEntropies = entcalc.computeLocalOfPreviousObservations();
double avgLocal = MatrixUtils.mean(localEntropies);
// There are many sources of numerical noise in the combination of so many
// local entropy calculations here (in comparison to the average calculation
// which only comes from the variance), so we need to leave a wide tolerance
assertEquals(entropy, avgLocal, 0.02);
}
}

22
java/unittests/infodynamics/measures/continuous/kernel/EntropyCalculatorKernelTest.java Normal file
View File

@ -0,0 +1,22 @@
package infodynamics.measures.continuous.kernel;
import infodynamics.utils.MatrixUtils;
import infodynamics.utils.RandomGenerator;
import junit.framework.TestCase;
public class EntropyCalculatorKernelTest extends TestCase {
public void testLocalEntropiesAverage() throws Exception {
RandomGenerator rg = new RandomGenerator();
double[] data = rg.generateNormalData(100, 0, 1);
EntropyCalculatorKernel entcalc = new EntropyCalculatorKernel();
entcalc.initialise(0.5);
entcalc.setObservations(data);
double entropy = entcalc.computeAverageLocalOfObservations();
double[] localEntropies = entcalc.computeLocalOfPreviousObservations();
double avgLocal = MatrixUtils.mean(localEntropies);
// There are many sources of numerical noise in the combination of so many
// local entropy calculations here, so we need to leave a wide tolerance
assertEquals(entropy, avgLocal, 0.000001);
}
}