From 35df6ca3daa85a88d0efe78354f9ae342132d402 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Mediano Date: Tue, 28 Nov 2017 16:04:09 +0000 Subject: [PATCH 1/4] Kozachenko entropy estimator now adds noise to data by default. --- ...tropyCalculatorMultiVariateKozachenko.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index 9007c8b..06e3c08 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -18,6 +18,8 @@ package infodynamics.measures.continuous.kozachenko; +import java.util.Random; + import infodynamics.measures.continuous.EntropyCalculator; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; import infodynamics.utils.EuclideanUtils; @@ -67,6 +69,26 @@ public class EntropyCalculatorMultiVariateKozachenko private double lastAverage = 0.0; private double[] lastLocalEntropy; private boolean isComputed; + + /** + * Property name for whether to normalise the incoming data to + * mean 0, standard deviation 1 (default true) + */ + public static final String PROP_NORMALISE = "NORMALISE"; + /** + * Property name for an amount of random Gaussian noise to be + * added to the data (default is 1e-8, matching the MILCA toolkit). + */ + public static final String PROP_ADD_NOISE = "NOISE_LEVEL_TO_ADD"; + + /** + * Whether to add an amount of random noise to the incoming data + */ + protected boolean addNoise = true; + /** + * Amount of random Gaussian noise to add to the incoming data + */ + protected double noiseLevel = (double) 1e-8; public static final double EULER_MASCHERONI_CONSTANT = 0.5772156; @@ -98,6 +120,15 @@ public class EntropyCalculatorMultiVariateKozachenko boolean propertySet = true; if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) { dimensions = Integer.parseInt(propertyValue); + } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) { + if (propertyValue.equals("0") || + propertyValue.equalsIgnoreCase("false")) { + addNoise = false; + noiseLevel = 0; + } else { + addNoise = true; + noiseLevel = Double.parseDouble(propertyValue); + } } else { // No property was set propertySet = false; @@ -112,6 +143,8 @@ public class EntropyCalculatorMultiVariateKozachenko public String getProperty(String propertyName) throws Exception { if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) { return Integer.toString(dimensions); + } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) { + return Double.toString(noiseLevel); } else { // No property was set, and no superclass to call: return null; @@ -124,6 +157,16 @@ public class EntropyCalculatorMultiVariateKozachenko totalObservations = observations.length; isComputed = false; lastLocalEntropy = null; + + if (addNoise) { + Random random = new Random(); + // Add Gaussian noise of std dev noiseLevel to the data + for (int r = 0; r < totalObservations; r++) { + for (int c = 0; c < dimensions; c++) { + rawData[r][c] += random.nextGaussian()*noiseLevel; + } + } + } } /* From e290040d8d1a05e716a03aeb5ede757d98bc224f Mon Sep 17 00:00:00 2001 From: Pedro Martinez Mediano Date: Tue, 28 Nov 2017 16:26:46 +0000 Subject: [PATCH 2/4] Added docs to Kozachenko entropy calculator. --- ...tropyCalculatorMultiVariateKozachenko.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index 06e3c08..33f5cd8 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -62,13 +62,36 @@ import infodynamics.utils.MatrixUtils; public class EntropyCalculatorMultiVariateKozachenko implements EntropyCalculator, EntropyCalculatorMultiVariate { - protected boolean debug = false; - private int totalObservations; + /** + * Total number of observations supplied. + */ + private int totalObservations = 0; + /** + * Number of dimensions of our multivariate data set + */ private int dimensions = 1; + /** + * The set of observations, retained in case the user wants to retrieve the local + * entropy values of these. + */ protected double[][] rawData; + /** + * Store the last computed average H + */ private double lastAverage = 0.0; + /** + * Store the last computed local H + */ private double[] lastLocalEntropy; + /** + * Track whether we've computed the average for the supplied + * observations yet + */ private boolean isComputed; + /** + * Whether to report debug messages or not + */ + protected boolean debug = false; /** * Property name for whether to normalise the incoming data to @@ -89,7 +112,9 @@ public class EntropyCalculatorMultiVariateKozachenko * Amount of random Gaussian noise to add to the incoming data */ protected double noiseLevel = (double) 1e-8; - + /** + * Stored pre-computed value of the Euler-Mascheroni constant + */ public static final double EULER_MASCHERONI_CONSTANT = 0.5772156; /** @@ -130,7 +155,7 @@ public class EntropyCalculatorMultiVariateKozachenko noiseLevel = Double.parseDouble(propertyValue); } } else { - // No property was set + // No property was set, and no superclass to call. propertySet = false; } if (debug && propertySet) { @@ -146,7 +171,7 @@ public class EntropyCalculatorMultiVariateKozachenko } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) { return Double.toString(noiseLevel); } else { - // No property was set, and no superclass to call: + // No property was set, and no superclass to call. return null; } } From 444fb708013c70a548e28a3bc85da9633f634a13 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Mediano Date: Tue, 28 Nov 2017 16:56:51 +0000 Subject: [PATCH 3/4] Added addObservations-type functions to Kozachenko entropy calculator. --- ...tropyCalculatorMultiVariateKozachenko.java | 75 ++++++++++++++++--- 1 file changed, 65 insertions(+), 10 deletions(-) diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index 33f5cd8..55cbd79 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -19,6 +19,8 @@ package infodynamics.measures.continuous.kozachenko; import java.util.Random; +import java.util.Iterator; +import java.util.Vector; import infodynamics.measures.continuous.EntropyCalculator; import infodynamics.measures.continuous.EntropyCalculatorMultiVariate; @@ -88,6 +90,11 @@ public class EntropyCalculatorMultiVariateKozachenko * observations yet */ private boolean isComputed; + /** + * Storage for observations supplied via {@link #addObservations(double[][])} + * type calls + */ + protected Vector vectorOfObservations; /** * Whether to report debug messages or not */ @@ -138,7 +145,7 @@ public class EntropyCalculatorMultiVariateKozachenko isComputed = false; lastLocalEntropy = null; } - + @Override public void setProperty(String propertyName, String propertyValue) throws Exception { @@ -176,12 +183,31 @@ public class EntropyCalculatorMultiVariateKozachenko } } - @Override - public void setObservations(double[][] observations) { - rawData = observations; - totalObservations = observations.length; + public void startAddObservations() { isComputed = false; + totalObservations = 0; lastLocalEntropy = null; + rawData = null; + vectorOfObservations = new Vector(); + } + + public void finaliseAddObservations() { + + rawData = new double[totalObservations][dimensions]; + + // Construct the joint vectors from the given observations + // (removing redundant data which is outside any timeDiff) + int startObservation = 0; + for (double[][] obs : vectorOfObservations) { + // Copy the data from these given observations into our master array + MatrixUtils.arrayCopy(obs, 0, 0, + rawData, startObservation, 0, + obs.length, dimensions); + startObservation += obs.length; + } + + // We don't need to keep the vector of observation sets anymore: + vectorOfObservations = null; if (addNoise) { Random random = new Random(); @@ -194,6 +220,20 @@ public class EntropyCalculatorMultiVariateKozachenko } } + @Override + public void setObservations(double[][] observations) { + startAddObservations(); + addObservations(observations); + finaliseAddObservations(); + } + + public void setObservations(double[][] observations1, double[][] observations2) + throws Exception { + startAddObservations(); + addObservations(observations1, observations2); + finaliseAddObservations(); + } + /* * (non-Javadoc) * @see infodynamics.measures.continuous.EntropyCalculator#setObservations(double[]) @@ -202,9 +242,24 @@ public class EntropyCalculatorMultiVariateKozachenko * EntropyCalculator interface. */ @Override - public void setObservations(double[] observations) { + public void setObservations(double[] observations) { + startAddObservations(); + addObservations(observations); + finaliseAddObservations(); + } + + public void addObservations(double[][] observations) { + if (vectorOfObservations == null) { + // startAddObservations was not called first + throw new RuntimeException("User did not call startAddObservations before addObservations"); + } + vectorOfObservations.add(observations); + totalObservations += observations.length; + } + + public void addObservations(double[] observations) { rawData = MatrixUtils.reshape(observations, observations.length, 1); - setObservations(rawData); + addObservations(rawData); } /** @@ -217,9 +272,9 @@ public class EntropyCalculatorMultiVariateKozachenko * @param data1 first few variables in the joint data * @param data2 the other variables in the joint data * @throws Exception When the length of the two arrays of observations do not match. - * @see #setObservations(double[][]) + * @see #addObservations(double[][]) */ - public void setObservations(double[][] data1, + public void addObservations(double[][] data1, double[][] data2) throws Exception { int timeSteps = data1.length; if ((data1 == null) || (data2 == null)) { @@ -237,7 +292,7 @@ public class EntropyCalculatorMultiVariateKozachenko System.arraycopy(data2[t], 0, data[t], data1Variables, data2Variables); } // Now defer to the normal setObservations method - setObservations(data); + addObservations(data); } /** From eabd7f370165fcc23de1967c02e8eeb13b32ed61 Mon Sep 17 00:00:00 2001 From: Pedro Martinez Mediano Date: Tue, 28 Nov 2017 16:58:13 +0000 Subject: [PATCH 4/4] Fixed typo in Kozachenko entropy calculator. --- .../kozachenko/EntropyCalculatorMultiVariateKozachenko.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java index 55cbd79..ba67fc1 100755 --- a/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java +++ b/java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java @@ -304,7 +304,7 @@ public class EntropyCalculatorMultiVariateKozachenko return lastAverage; } double sdTermHere = sdTerm(totalObservations, dimensions); - double emConstHere = eulerMacheroniTerm(totalObservations); + double emConstHere = eulerMascheroniTerm(totalObservations); double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData); double entropy = 0.0; if (debug) { @@ -344,7 +344,7 @@ public class EntropyCalculatorMultiVariateKozachenko } double sdTermHere = sdTerm(totalObservations, dimensions); - double emConstHere = eulerMacheroniTerm(totalObservations); + double emConstHere = eulerMascheroniTerm(totalObservations); double constantToAddIn = sdTermHere + emConstHere; double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData); @@ -394,7 +394,7 @@ public class EntropyCalculatorMultiVariateKozachenko * * @return */ - public double eulerMacheroniTerm(int N) { + public double eulerMascheroniTerm(int N) { // Using natural units // return EULER_MASCHERONI_CONSTANT / Math.log(2); try {