Adding property to set random seed for noise addition to EntropyMultiVariate estimators (mimicing that for MI/CMI estimators), extending the fix for issue #99 to Entropy estimators

java/source/infodynamics/measures/continuous/EntropyCalculator.java

@@ -48,8 +48,11 @@ Theory' (John Wiley & Sons, New York, 1991).</li>
  */
 public interface EntropyCalculator extends InfoMeasureCalculatorContinuous {
 
-	// TODO Add addObservations() methods for entropy calculator
-	
+	// TODO Add addObservations() methods for entropy calculator.
+	//  It's currently in EntropyCalculatorMultivariate; bring in
+	//  here when we make all univariate entropy calculators use their 
+	//  underlying multivariate form.
+
 	/**
 	 * Sets the samples from which to compute the PDF for the entropy.
 	 * Should only be called once, the last call contains the

java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariate.java

@@ -61,6 +61,26 @@ public interface EntropyCalculatorMultiVariate
 	 * Property name for the number of dimensions
 	 */
 	public static final String NUM_DIMENSIONS_PROP_NAME = "NUM_DIMENSIONS";
+	/**
+	 * Property for whether we normalise the incoming observations to mean 0,
+	 * standard deviation 1.
+	 */
+	public static final String NORMALISE_PROP_NAME = "NORMALISE";
+	/**
+	 * Property name for an amount of random Gaussian noise to be
+	 *  added to the data (default is 1e-8, matching the MILCA toolkit).
+	 */
+	public static final String PROP_ADD_NOISE = "NOISE_LEVEL_TO_ADD";
+	/**
+	 * Property name for the seed for the random number generator for noise to be
+	 *  added to the data (default is no seed)
+	 */
+	public static final String PROP_NOISE_SEED = "NOISE_SEED";
+	/**
+	 * Property value to indicate no seed for the random number generator for noise to be
+	 *  added to the data
+	 */
+	public static final String NOISE_NO_SEED_VALUE = "NONE";
 	
 	/**
 	 * Set properties for the underlying calculator implementation.
@@ -72,6 +92,20 @@ public interface EntropyCalculatorMultiVariate
 	 * <ul>
 	 *  <li>{@link #NUM_DIMENSIONS_PROP_NAME} -- number of dimensions in the joint
 	 * 		variable that we are computing the entropy of.</li>
+	 * 	<li>{@link #NORMALISE_PROP_NAME} -- whether to normalise the incoming variable values
+	 * 			to mean 0, standard deviation 1, or not (default false). Sets {@link #normalise}.</li>
+	 *  <li>{@link #PROP_ADD_NOISE} -- a standard deviation for an amount of
+	 *      random Gaussian noise to add to
+	 *      each variable, to avoid having neighbourhoods with artificially
+	 *      large counts. (We also accept "false" to indicate "0".)
+	 *      The amount is added in after any normalisation,
+	 *      so can be considered as a number of standard deviations of the data.
+	 *      Default is 0 for most estimators; this is strongly recommended by
+	 *      by Kraskov for the KSG method though, so for the Kozachenko estimator we 
+	 *      use 1e-8 to match the MILCA toolkit (though note it adds in
+	 *      a random amount of noise in [0,noiseLevel) ).</li>
+	 *  <li>{@link #PROP_NOISE_SEED} -- a long value seed for the random noise generator or
+	 *      the string {@link MutualInfoCalculatorMultiVariate#NOISE_NO_SEED_VALUE} for no seed (default)</li>
 	 * </ul>
 	 *  
 	 * <p>Unknown property values are ignored.</p>
@@ -94,6 +128,46 @@ public interface EntropyCalculatorMultiVariate
 	 */
 	public void initialise(int dimensions);
 	
+	/**
+	 * Signal that we will add in the samples for computing the PDF 
+	 * from several disjoint time-series or trials via calls to
+	 * "addObservations" rather than "setObservations" type methods
+	 * (defined by the child interfaces and classes).
+	 */
+	public void startAddObservations();
+	
+	/**
+	 * Add more observations for which to compute the PDFs for the entropy.
+	 * May be called multiple times between {@link #startAddObservations()} and
+	 * {@link #finaliseAddObservations()}.
+	 * 
+	 * @param observations multivariate time series of observations; first index
+	 *  is time step, second index is variable number (total should match dimensions
+	 *  supplied to {@link #initialise(int)}
+	 * @throws Exception if the dimensions of the observations do not match 
+	 *  the expected value supplied in {@link #initialise(int)}; implementations
+	 *  may throw other more specific exceptions also.
+	 */
+	public void addObservations(double[][] observations) throws Exception;
+
+	/**
+	 * Add more samples from which to compute the PDF for the entropy.
+	 * Only allowed to be called when set up for dimension == 1.
+	 * May be called multiple times between {@link #startAddObservations()} and
+	 * {@link #finaliseAddObservations()}.
+	 * 
+	 * @param observations array of (univariate) samples
+	 * @throws Exception if the expected dimensions were not 1.
+	 */
+	public void addObservations(double[] observations) throws Exception;
+
+	/**
+	 * Signal that the observations are now all added, PDFs can now be constructed.
+	 * 
+	 * @throws Exception 
+	 */
+	public void finaliseAddObservations() throws Exception;
+	
 	/**
 	 * Set the observations for which to compute the PDFs for the entropy
 	 * Should only be called once, the last call contains the

java/source/infodynamics/measures/continuous/EntropyCalculatorMultiVariateCommon.java

@@ -0,0 +1,342 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2024, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package infodynamics.measures.continuous;
+
+import java.util.Random;
+import java.util.Vector;
+
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * Implements {@link EntropyCalculatorMultiVariate} to provide a base
+ * class with common functionality for child class implementations of
+ * {@link EntropyCalculatorMultiVariate}
+ * via various estimators. 
+ * 
+ * <p>These various estimators include: e.g. box-kernel estimation, Kozachenko, etc
+ * (see the child classes linked above).
+ * </p>
+ * 
+ * <p>Usage is as outlined in {@link EntropyCalculatorMultiVariate}.</p>
+ * 
+  * @author Joseph Lizier (<a href="joseph.lizier at gmail.com">email</a>,
+ * <a href="http://lizier.me/joseph/">www</a>)
+ */
+public abstract class EntropyCalculatorMultiVariateCommon implements EntropyCalculatorMultiVariate {
+	/**
+	 * Whether we're in debug mode
+	 */
+	protected boolean debug = false;
+	/**
+	 * Number of observations supplied
+	 */
+	protected int totalObservations = 0;
+	/**
+	 * Number of joint variables/dimensions
+	 */
+	protected int dimensions = 1;
+	/**
+	 * Track whether we've computed the average for the supplied
+	 *  observations yet
+	 */
+	protected boolean isComputed = false;
+	/**
+	 * Last computed average entropy
+	 */
+	protected double lastAverage = 0.0;
+	/**
+	 * Whether we normalise the incoming observations to mean 0,
+	 * standard deviation 1.
+	 */
+	protected boolean normalise = true;
+	/**
+	 * Whether to add an amount of random noise to the incoming data
+	 */
+	protected boolean addNoise = false;
+	/**
+	 * Amount of random Gaussian noise to add to the incoming data
+	 */
+	protected double noiseLevel = (double) 0.0;
+	/**
+	 * Has the user set a seed for the random noise
+	 */
+	protected boolean noiseSeedSet = false;
+	/**
+	 * Seed that the user set for the random noise
+	 */
+	protected long noiseSeed = 0;
+	/**
+	 * Storage for observations supplied via {@link #addObservations(double[][])}
+	 * type calls
+	 */
+	protected Vector<double[][]> vectorOfObservations;
+	/**
+	 * The set of observations, retained in case the user wants to retrieve the local
+	 *  entropy values of these.
+	 */
+	protected double[][] observations;
+
+	/**
+	 * Default constructor
+	 */
+	public EntropyCalculatorMultiVariateCommon() {
+		// Nothing to do
+	}
+	
+	@Override
+	public void initialise() throws Exception {
+		initialise(dimensions);
+	}
+
+	@Override
+	public void initialise(int dimensions) {
+		this.dimensions = dimensions;
+		observations = null;
+		totalObservations = 0;
+		isComputed = false;
+		lastAverage = 0;
+	}
+
+	@Override
+	public void setProperty(String propertyName, String propertyValue) throws Exception {
+
+		boolean propertySet = true;
+
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			dimensions = Integer.parseInt(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			normalise = Boolean.parseBoolean(propertyValue);
+		} else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
+			if (propertyValue.equals("0") ||
+					propertyValue.equalsIgnoreCase("false")) {
+				addNoise = false;
+				noiseLevel = 0;
+			} else {
+				addNoise = true;
+				noiseLevel = Double.parseDouble(propertyValue);
+			}
+	    } else if (propertyName.equalsIgnoreCase(PROP_NOISE_SEED)) {
+	    	if (propertyValue.equals(NOISE_NO_SEED_VALUE)) {
+	    		noiseSeedSet = false;
+	    	} else {
+	    		noiseSeedSet = true;
+	    		noiseSeed = Long.parseLong(propertyValue);
+	    	}
+		} else {
+			// No property was set
+			propertySet = false;
+		}
+		if (debug && propertySet) {
+			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
+					" to " + propertyValue);
+		}
+	}
+	
+	@Override
+	public String getProperty(String propertyName) throws Exception {
+		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
+			return Integer.toString(dimensions);
+		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+			return Boolean.toString(normalise);
+	    } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
+	        return Double.toString(noiseLevel);
+	    } else if (propertyName.equalsIgnoreCase(PROP_NOISE_SEED)) {
+	    	if (noiseSeedSet) {
+	    		return Long.toString(noiseSeed);
+	    	} else {
+	    		return NOISE_NO_SEED_VALUE;
+	    	}
+		} else {
+			// No property was set, and no superclass to call:
+			return null;
+		}
+	}
+
+	@Override
+	public void startAddObservations() {
+		isComputed = false;
+		totalObservations = 0;
+		observations = null;
+		vectorOfObservations = new Vector<double[][]>();
+	}
+
+	@Override
+	public void finaliseAddObservations() throws Exception {
+
+		observations = new double[totalObservations][dimensions];
+		
+		// Construct the joint vectors from the given observations
+		int startObservation = 0;
+		for (double[][] obs : vectorOfObservations) {
+			if ((obs == null) || (obs.length < 1)) {
+				continue;
+			}
+			// Dimension was checked earlier by addObservations.
+			// Copy the data from these given observations into our master array
+			MatrixUtils.arrayCopy(obs, 0, 0,
+					observations, startObservation, 0,
+					obs.length, dimensions);
+			startObservation += obs.length;
+		}
+
+		// We don't need to keep the vector of observation sets anymore:
+		vectorOfObservations = null;
+
+		if (addNoise) {
+			Random random = new Random();
+			if (noiseSeedSet) {
+				random.setSeed(noiseSeed);
+			}
+			// Add Gaussian noise of std dev noiseLevel to the data
+			for (int r = 0; r < totalObservations; r++) {
+				for (int c = 0; c < dimensions; c++) {
+					observations[r][c] += random.nextGaussian()*noiseLevel;
+				}
+			}
+		}
+	}
+
+	@Override
+	public void setObservations(double[][] observations) throws Exception {
+		startAddObservations();
+		addObservations(observations);
+		finaliseAddObservations();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see infodynamics.measures.continuous.EntropyCalculator#setObservations(double[])
+	 * 
+	 * This method here to ensure we make compatibility with the 
+	 *  EntropyCalculator interface; only valid if dimension > 1
+	 */
+	@Override
+	public void setObservations(double[] observations) throws Exception {
+		startAddObservations();
+		addObservations(observations);
+		finaliseAddObservations();
+	}
+
+	/**
+	 * Shortcut method to join two sets of samples into a joint multivariate 
+	 * 
+	 * Each row of the data is an observation; each column of
+	 *  the row is a new variable in the multivariate observation.
+	 * This method signature allows the user to call setObservations for
+	 *  joint time series without combining them into a single joint time
+	 *  series (we do the combining for them).
+	 * 
+	 * @param data1 observations1 few variables in the joint data
+	 * @param observations2 the other variables in the joint data
+	 * @throws Exception When the length of the two arrays of observations do not match.
+	 * @see #setObservations(double[][])
+	 */
+	@Deprecated
+	public void setObservations(double[][] observations1, double[][] observations2)
+			throws Exception {
+		startAddObservations();
+		addObservations(observations1, observations2);
+		finaliseAddObservations();
+	}
+
+	@Override
+	public void addObservations(double[][] observations) throws Exception {
+		if (vectorOfObservations == null) {
+		// startAddObservations was not called first
+			throw new RuntimeException("User did not call startAddObservations before addObservations");
+		}
+		if ((observations != null) && (observations.length > 0)) {
+			// Check the dimension:
+			if (observations[0].length != dimensions) {
+				throw new Exception(String.format("Cannot supply observations with dimension %d when expected dimension = %d",
+						observations[0].length, dimensions));
+			}
+			totalObservations += observations.length;
+		}
+		// Add the observations whether empty or else valid:
+		vectorOfObservations.add(observations);
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * @see infodynamics.measures.continuous.EntropyCalculator#addObservations(double[])
+	 * 
+	 * This method here to ensure we make compatibility with the 
+	 *  EntropyCalculator interface; only valid if dimension > 1
+	 */
+	@Override
+	public void addObservations(double[] observations) throws Exception {
+		if (dimensions != 1) {
+			throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions));
+		}
+		double[][] observations2D = MatrixUtils.reshape(observations, observations.length, 1);
+		addObservations(observations2D);
+	}
+	
+	/**
+	 * Shortcut method to join two sets of samples into a joint multivariate. 
+	 * 
+	 * Each row of the data is an observation; each column of
+	 *  the row is a new variable in the multivariate observation.
+	 * This method signature allows the user to call addObservations for
+	 *  joint time series without combining them into a single joint time
+	 *  series (we do the combining for them).
+	 * 
+	 * @param data1 first few variables in the joint data
+	 * @param data2 the other variables in the joint data
+	 * @throws Exception When the length of the two arrays of observations do not match.
+	 * @see #addObservations(double[][])
+	 */
+	@Deprecated
+	public void addObservations(double[][] data1,
+			double[][] data2) throws Exception {
+		int timeSteps = data1.length;
+		if ((data1 == null) || (data2 == null)) {
+			throw new Exception("Cannot have null data arguments");
+		}
+		if (data1.length != data2.length) {
+			throw new Exception("Length of data1 (" + data1.length + ") is not equal to the length of data2 (" +
+					data2.length + ")");
+		}
+		int data1Variables = data1[0].length;
+		int data2Variables = data2[0].length;
+		double[][] data = new double[timeSteps][data1Variables + data2Variables];
+		for (int t = 0; t < timeSteps; t++) {
+			System.arraycopy(data1[t], 0, data[t], 0, data1Variables);
+			System.arraycopy(data2[t], 0, data[t], data1Variables, data2Variables);
+		}
+		// Now defer to the normal setObservations method
+		addObservations(data);
+	}
+
+	@Override
+	public int getNumObservations() throws Exception {
+		return totalObservations;
+	}
+
+	@Override
+	public double getLastAverage() {
+		return lastAverage;
+	}
+	
+	@Override
+	public void setDebug(boolean debug) {
+		this.debug = debug;
+	}
+
+}

java/source/infodynamics/measures/continuous/gaussian/EntropyCalculatorMultiVariateGaussian.java

@@ -19,6 +19,7 @@
 package infodynamics.measures.continuous.gaussian;
 
 import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
+import infodynamics.measures.continuous.EntropyCalculatorMultiVariateCommon;
 import infodynamics.utils.MatrixUtils;
 
 /**
@@ -50,6 +51,7 @@ Theory' (John Wiley & Sons, New York, 1991).</li>
  * <a href="http://lizier.me/joseph/">www</a>)
  */
 public class EntropyCalculatorMultiVariateGaussian
+	extends EntropyCalculatorMultiVariateCommon 
 	implements EntropyCalculatorMultiVariate, Cloneable {
 
 	/**
@@ -63,84 +65,34 @@ public class EntropyCalculatorMultiVariateGaussian
 	 */
 	protected double[] means;
 	
-	/**
-	 * The set of observations, retained in case the user wants to retrieve the local
-	 *  entropy values of these
-	 */
-	protected double[][] observations;
-	
-	/**
-	 * Number of dimensions for our multivariate data
-	 */
-	protected int dimensions = 1;
-	
 	/**
 	 * Determinant of the covariance matrix; stored to save computation time
 	 */
 	protected double detCovariance = 0.0;
-
-	/**
-	 * Last average entropy we computed
-	 */
-	protected double lastAverage = 0;
-	
-	/**
-	 * Whether we are in debug mode
-	 */
-	protected boolean debug = false;
 	
 	/**
 	 * Construct an instance
 	 */
 	public EntropyCalculatorMultiVariateGaussian() {
-		// Nothing to do
+		super();
 	}
 	
-	@Override
-	public void initialise() throws Exception {
-		initialise(dimensions);
-	}
-
 	public void initialise(int dimensions) {
+		super.initialise(dimensions);
 		means = null;
 		L = null;
-		observations = null;
-		this.dimensions = dimensions;
 		detCovariance = 0;
-		lastAverage = 0.0;
 	}
 
-	/**
-	 * @throws Exception where the observations do not match the expected number of 
-	 *  dimensions, or covariance matrix is not positive definite (reflecting 
-	 *  redundant variables in the observations)
-	 */
 	@Override
-	public void setObservations(double[][] observations) throws Exception {
-		// Check that the observations was of the correct number of dimensions:
-		if (observations[0].length != dimensions) {
-			means = null;
-			L = null;
-			throw new Exception("Supplied observations does not match initialised number of dimensions");
-		}
+	public void finaliseAddObservations() throws Exception {
+		super.finaliseAddObservations();
 		means = MatrixUtils.means(observations);
+		double[][] originalObservations = observations; // Keep a reference as below
 		setCovariance(MatrixUtils.covarianceMatrix(observations, means));
 		// And keep a reference to the observations used here (must set this
 		//  *after* setCovariance, since setCovariance sets the observations to null
-		this.observations = observations;
-	}
-
-	/**
-	 * @throws Exception where the observations do not match the expected number of 
-	 *  dimensions, or covariance matrix is not positive definite (reflecting 
-	 *  redundant variables in the observations)
-	 */
-	@Override
-	public void setObservations(double[] observations) throws Exception {
-		if (dimensions != 1) {
-			throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions));
-		}
-		setObservations(MatrixUtils.reshape(observations, observations.length, 1));
+		observations = originalObservations;
 	}
 
 	/**
@@ -209,49 +161,45 @@ public class EntropyCalculatorMultiVariateGaussian
 	 */
 	@Override
 	public double computeAverageLocalOfObservations() {
+		if (isComputed) {
+			return lastAverage;
+		}
 		// Simple way:
 		// detCovariance = MatrixUtils.determinantSymmPosDefMatrix(covariance);
 		// Using cached Cholesky decomposition:
 		detCovariance = MatrixUtils.determinantViaCholeskyResult(L);
 		lastAverage =  0.5 * (dimensions* (1 + Math.log(2.0*Math.PI)) +
 			Math.log(detCovariance));
+		isComputed = true;
 		return lastAverage;
 	}
 
-	@Override
-	public void setDebug(boolean debug) {
-		this.debug = debug;
-	}
-
+	/**
+	 * <p>Set properties for this calculator.
+	 *  New property values are not guaranteed to take effect until the next call
+	 *  to an initialise method.
+	 * 
+	 * <p>Valid property names, and what their
+	 * values should represent, include:</p>
+	 * <ul>
+	 * 		<li>{@link #NORMALISE_PROP_NAME} as per {@link EntropyCalculatorMultiVariate#setProperty()}
+	 *         however note that for this Gaussian estimator setting this to true would fix the entropy
+	 *         values.</li>
+	 *  	<li>any valid properties for {@link EntropyCalculatorMultiVariate#setProperty(String, String)}.</li>
+	 * </ul> 
+	 * 
+	 * <p>Unknown property values are ignored.</p>
+	 * 
+	 * @param propertyName name of the property
+	 * @param propertyValue value of the property
+	 * @throws Exception for invalid property values
+	 */
 	@Override
 	public void setProperty(String propertyName, String propertyValue)
 			throws Exception {
-		boolean propertySet = true;
-		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			dimensions = Integer.parseInt(propertyValue);
-		} else {
-			// No property was set
-			propertySet = false;
-		}
-		if (debug && propertySet) {
-			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
-					" to " + propertyValue);
-		}
-	}
-
-	@Override
-	public String getProperty(String propertyName) throws Exception {
-		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			return Integer.toString(dimensions);
-		} else {
-			// No property was set, and no superclass to call:
-			return null;
-		}
-	}
-	
-	@Override
-	public double getLastAverage() {
-		return lastAverage;
+		// Actually don't need to implement this method, but want 
+		//  the javadocs to get generated to warn user about the normalise property.
+		super.setProperty(propertyName, propertyValue);
 	}
 
 	/**
@@ -316,9 +264,13 @@ public class EntropyCalculatorMultiVariateGaussian
 		if (observations == null) {
 			throw new Exception("Cannot compute local values since no observations were supplied");
 		}
-		return computeLocalUsingPreviousObservations(observations);
+		double[] localValues = computeLocalUsingPreviousObservations(observations);
+		lastAverage = MatrixUtils.mean(localValues);
+		isComputed = true;
+		return localValues;
 	}
 
+	@Override
 	public int getNumObservations() throws Exception {
 		if (observations == null) {
 			throw new Exception("Cannot return number of observations because either " +

java/source/infodynamics/measures/continuous/kernel/EntropyCalculatorMultiVariateKernel.java

@@ -19,7 +19,7 @@
 package infodynamics.measures.continuous.kernel;
 
 import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
-import infodynamics.utils.MatrixUtils;
+import infodynamics.measures.continuous.EntropyCalculatorMultiVariateCommon;
 
 /**
  * <p>Computes the differential entropy of a given set of observations
@@ -44,42 +44,13 @@ import infodynamics.utils.MatrixUtils;
  * @author Joseph Lizier (<a href="joseph.lizier at gmail.com">email</a>,
  * <a href="http://lizier.me/joseph/">www</a>)
  */
-public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMultiVariate {
+public class EntropyCalculatorMultiVariateKernel
+	extends EntropyCalculatorMultiVariateCommon 
+	implements EntropyCalculatorMultiVariate
+	{
 
 	private KernelEstimatorMultiVariate mvke = null;
-	/**
-	 * Number of observations supplied
-	 */
-	private int totalObservations = 0;
-	// private int dimensions = 0;
-	/**
-	 * Whether we're in debug mode
-	 */
-	private boolean debug = false;
-	/**
-	 * The supplied observations
-	 */
-	private double[][] observations = null;
-	/**
-	 * Last computed average entropy
-	 */
-	private double lastEntropy;
-	/**
-	 * Whether we normalise the incoming observations to mean 0,
-	 * standard deviation 1.
-	 */
-	private boolean normalise = true;
-	/**
-	 * Property for whether we normalise the incoming observations to mean 0,
-	 * standard deviation 1.
-	 */
-	public static final String NORMALISE_PROP_NAME = "NORMALISE";
 	
-	/**
-	 * Number of joint variables/dimensions
-	 */
-	protected int dimensions = 1;
-
 	/**
 	 * Default value for kernel width
 	 */
@@ -101,15 +72,10 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 * Construct an instance
 	 */
 	public EntropyCalculatorMultiVariateKernel() {
+		super();
 		mvke = new KernelEstimatorMultiVariate();
 		mvke.setDebug(debug);
 		mvke.setNormalise(normalise);
-		lastEntropy = 0.0;
-	}
-
-	@Override
-	public void initialise() throws Exception {
-		initialise(dimensions);
 	}
 
 	public void initialise(int dimensions) {
@@ -128,34 +94,22 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 *  standard deviations from the mean (otherwise it is an absolute value)
 	 */
 	public void initialise(int dimensions, double kernelWidth) {
+		super.initialise(dimensions);
 		this.kernelWidth = kernelWidth;
-		this.dimensions = dimensions;
 		mvke.initialise(dimensions, kernelWidth);
-		// this.dimensions = dimensions;
-		lastEntropy = 0.0;
 	}
 
 	@Override
-	public void setObservations(double observations[][]) {
+	public void finaliseAddObservations() throws Exception {
+		super.finaliseAddObservations();
 		mvke.setObservations(observations);
-		totalObservations = observations.length;
-		this.observations = observations;
-	}
-
-	/**
-	 * @throws Exception where the observations do not match the expected number of 
-	 *  dimensions
-	 */
-	@Override
-	public void setObservations(double[] observations) throws Exception {
-		if (dimensions != 1) {
-			throw new Exception(String.format("Cannot set univariate observations when expected dimension = %d", dimensions));
-		}
-		setObservations(MatrixUtils.reshape(observations, observations.length, 1));
 	}
 
 	@Override
 	public double computeAverageLocalOfObservations() {
+		if (isComputed) {
+			return lastAverage;
+		}
 		double entropy = 0.0;
 		for (int b = 0; b < totalObservations; b++) {
 			double prob = mvke.getProbability(observations[b]);
@@ -165,8 +119,9 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 				System.out.println(b + ": " + prob + " -> " + (-cont/Math.log(2.0)) + " -> sum: " + (entropy/Math.log(2.0)));
 			}
 		}
-		lastEntropy = entropy / (double) totalObservations / Math.log(2.0);
-		return lastEntropy;
+		lastAverage = entropy / (double) totalObservations / Math.log(2.0);
+		isComputed = true;
+		return lastAverage;
 	}
 
 	@Override
@@ -204,22 +159,18 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		}
 		entropy = entropy / (double) totalObservations / Math.log(2.0); // Don't use /= as I'm not sure of the order of operations it applies.
 		if (isPreviousObservations) {
-			lastEntropy = entropy; 
+			lastAverage = entropy;
+			isComputed = true;
 		}
 		return localEntropy;
 	}
 
 	@Override
 	public void setDebug(boolean debug) {
-		this.debug = debug;
+		super.setDebug(debug);
 		mvke.setDebug(debug);
 	}
 
-	@Override
-	public double getLastAverage() {
-		return lastEntropy;
-	}
-	
 	/**
 	 * <p>Set properties for the kernel entropy calculator.
 	 *  New property values are not guaranteed to take effect until the next call
@@ -232,8 +183,6 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 	 * 			kernel width to be used in the calculation. If {@link #normalise} is set,
 	 * 		    then this is a number of standard deviations; otherwise it
 	 * 			is an absolute value. Default is {@link #DEFAULT_KERNEL_WIDTH}.</li>
-	 * 		<li>{@link #NORMALISE_PROP_NAME} -- whether to normalise the incoming variable values
-	 * 			to mean 0, standard deviation 1, or not (default false). Sets {@link #normalise}.</li>
 	 *  	<li>any valid properties for {@link EntropyCalculatorMultiVariate#setProperty(String, String)}.</li>
 	 * </ul> 
 	 * 
@@ -254,14 +203,15 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		if (propertyName.equalsIgnoreCase(KERNEL_WIDTH_PROP_NAME) ||
 				propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
 			kernelWidth = Double.parseDouble(propertyValue);
-		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
-			normalise = Boolean.parseBoolean(propertyValue);
-			mvke.setNormalise(normalise);
-		} else if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			dimensions = Integer.parseInt(propertyValue);
 		} else {
 			// No property was set
 			propertySet = false;
+			// try the superclass:
+			super.setProperty(propertyName, propertyValue);
+			if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
+				// Handle an additional step for this one:
+				mvke.setNormalise(normalise);
+			}
 		}
 		if (debug && propertySet) {
 			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
@@ -274,19 +224,10 @@ public class EntropyCalculatorMultiVariateKernel implements EntropyCalculatorMul
 		if (propertyName.equalsIgnoreCase(KERNEL_WIDTH_PROP_NAME) ||
 				propertyName.equalsIgnoreCase(EPSILON_PROP_NAME)) {
 			return Double.toString(kernelWidth);
-		} else if (propertyName.equalsIgnoreCase(NORMALISE_PROP_NAME)) {
-			return Boolean.toString(normalise);
-		} else if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			return Integer.toString(dimensions);
 		} else {
-			// No property was set, and no superclass to call:
-			return null;
+		      // try the superclass:
+		      return super.getProperty(propertyName);
 		}
 	}
 
-	@Override
-	public int getNumObservations() throws Exception {
-		return totalObservations;
-	}
-
 }

java/source/infodynamics/measures/continuous/kozachenko/EntropyCalculatorMultiVariateKozachenko.java

@@ -18,13 +18,10 @@
 
 package infodynamics.measures.continuous.kozachenko;
 
-import java.util.Random;
-import java.util.Vector;
-
 import infodynamics.measures.continuous.EntropyCalculatorMultiVariate;
+import infodynamics.measures.continuous.EntropyCalculatorMultiVariateCommon;
 import infodynamics.utils.EuclideanUtils;
 import infodynamics.utils.MathsUtils;
-import infodynamics.utils.MatrixUtils;
 
 /**
  * <p>Computes the differential entropy of a given set of observations
@@ -59,59 +56,11 @@ import infodynamics.utils.MatrixUtils;
  * @author Joseph Lizier (<a href="joseph.lizier at gmail.com">email</a>,
  * <a href="http://lizier.me/joseph/">www</a>)
  */
-public class EntropyCalculatorMultiVariateKozachenko  
+public class EntropyCalculatorMultiVariateKozachenko
+	extends EntropyCalculatorMultiVariateCommon 
 	implements EntropyCalculatorMultiVariate {
 
-	/**
-	 * Total number of observations supplied.
-	 */
-	private int totalObservations = 0;
-	/**
-	 * Number of dimensions of our multivariate data set
-	 */
-	private int dimensions = 1;
-	/**
-	 * The set of observations, retained in case the user wants to retrieve the local
-	 *  entropy values of these.
-	 */
-	protected double[][] rawData;
-	/**
-	 * Store the last computed average H
-	 */
-	private double lastAverage = 0.0;
-	/**
-	 * Store the last computed local H
-	 */
-	private double[] lastLocalEntropy;
-	/**
-	 * Track whether we've computed the average for the supplied
-	 *  observations yet
-	 */
-	private boolean isComputed;
-  /**
-	 * Storage for observations supplied via {@link #addObservations(double[][])}
-	 * type calls
-	 */
-	protected Vector<double[][]> vectorOfObservations;
-	/**
-	 * Whether to report debug messages or not
-	 */
-	protected boolean debug = false;
 
-  /**
-   * Property name for an amount of random Gaussian noise to be
-   *  added to the data (default is 1e-8, matching the MILCA toolkit).
-   */
-  public static final String PROP_ADD_NOISE = "NOISE_LEVEL_TO_ADD";
-
-  /**
-   * Whether to add an amount of random noise to the incoming data
-   */
-  protected boolean addNoise = true;
-  /**
-   * Amount of random Gaussian noise to add to the incoming data
-   */
-  protected double noiseLevel = (double) 1e-8;
   /**
    * Stored pre-computed value of the Euler-Mascheroni constant
    */
@@ -121,171 +70,9 @@ public class EntropyCalculatorMultiVariateKozachenko
 	 * Construct an instance
 	 */
 	public EntropyCalculatorMultiVariateKozachenko() {
-		totalObservations = 0;
-		isComputed = false;
-		lastLocalEntropy = null;
-	}
-
-	@Override
-	public void initialise() throws Exception {
-		initialise(dimensions);
-	}
-
-	public void initialise(int dimensions) {
-		this.dimensions = dimensions;
-		rawData = null;
-		totalObservations = 0;
-		isComputed = false;
-		lastLocalEntropy = null;
-	}
-
-	@Override
-	public void setProperty(String propertyName, String propertyValue)
-			throws Exception {
-		boolean propertySet = true;
-		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			dimensions = Integer.parseInt(propertyValue);
-    } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
-      if (propertyValue.equals("0") ||
-          propertyValue.equalsIgnoreCase("false")) {
-        addNoise = false;
-        noiseLevel = 0;
-      } else {
-        addNoise = true;
-        noiseLevel = Double.parseDouble(propertyValue);
-      }
-		} else {
-			// No property was set, and no superclass to call.
-			propertySet = false;
-		}
-		if (debug && propertySet) {
-			System.out.println(this.getClass().getSimpleName() + ": Set property " + propertyName +
-					" to " + propertyValue);
-		}
-	}
-
-	@Override
-	public String getProperty(String propertyName) throws Exception {
-		if (propertyName.equalsIgnoreCase(NUM_DIMENSIONS_PROP_NAME)) {
-			return Integer.toString(dimensions);
-    } else if (propertyName.equalsIgnoreCase(PROP_ADD_NOISE)) {
-      return Double.toString(noiseLevel);
-		} else {
-			// No property was set, and no superclass to call.
-			return null;
-		}
-	}
-
-	public void startAddObservations() {
-		isComputed = false;
-    totalObservations = 0;
-		lastLocalEntropy = null;
-    rawData = null;
-		vectorOfObservations = new Vector<double[][]>();
-	}
-
-	public void finaliseAddObservations() {
-
-		rawData = new double[totalObservations][dimensions];
-		
-		// Construct the joint vectors from the given observations
-		//  (removing redundant data which is outside any timeDiff)
-		int startObservation = 0;
-		for (double[][] obs : vectorOfObservations) {
-			// Copy the data from these given observations into our master array
-			MatrixUtils.arrayCopy(obs, 0, 0,
-					rawData, startObservation, 0,
-					obs.length, dimensions);
-			startObservation += obs.length;
-		}
-
-		// We don't need to keep the vector of observation sets anymore:
-		vectorOfObservations = null;
-
-    if (addNoise) {
-      Random random = new Random();
-      // Add Gaussian noise of std dev noiseLevel to the data
-      for (int r = 0; r < totalObservations; r++) {
-        for (int c = 0; c < dimensions; c++) {
-          rawData[r][c] += random.nextGaussian()*noiseLevel;
-        }
-      }
-    }
-	}
-
-	@Override
-  public void setObservations(double[][] observations) {
-    startAddObservations();
-    addObservations(observations);
-    finaliseAddObservations();
-  }
-
-  public void setObservations(double[][] observations1, double[][] observations2)
-      throws Exception {
-    startAddObservations();
-    addObservations(observations1, observations2);
-    finaliseAddObservations();
-  }
-
-	/*
-	 * (non-Javadoc)
-	 * @see infodynamics.measures.continuous.EntropyCalculator#setObservations(double[])
-	 * 
-	 * This method here to ensure we make compatibility with the 
-	 *  EntropyCalculator interface.
-	 */
-	@Override
-  public void setObservations(double[] observations) {
-    startAddObservations();
-    addObservations(observations);
-    finaliseAddObservations();
-  }
-
-  public void addObservations(double[][] observations) {
-		if (vectorOfObservations == null) {
-			// startAddObservations was not called first
-			throw new RuntimeException("User did not call startAddObservations before addObservations");
-		}
-    vectorOfObservations.add(observations);
-    totalObservations += observations.length;
-  }
-
-	public void addObservations(double[] observations) {
-		rawData = MatrixUtils.reshape(observations, observations.length, 1);
-		addObservations(rawData);
-	}
-	
-	/**
-	 * Each row of the data is an observation; each column of
-	 *  the row is a new variable in the multivariate observation.
-	 * This method signature allows the user to call setObservations for
-	 *  joint time series without combining them into a single joint time
-	 *  series (we do the combining for them).
-	 * 
-	 * @param data1 first few variables in the joint data
-	 * @param data2 the other variables in the joint data
-	 * @throws Exception When the length of the two arrays of observations do not match.
-	 * @see #addObservations(double[][])
-	 */
-	public void addObservations(double[][] data1,
-			double[][] data2) throws Exception {
-		int timeSteps = data1.length;
-		if ((data1 == null) || (data2 == null)) {
-			throw new Exception("Cannot have null data arguments");
-		}
-		if (data1.length != data2.length) {
-			throw new Exception("Length of data1 (" + data1.length + ") is not equal to the length of data2 (" +
-					data2.length + ")");
-		}
-		int data1Variables = data1[0].length;
-		int data2Variables = data2[0].length;
-		double[][] data = new double[timeSteps][data1Variables + data2Variables];
-		for (int t = 0; t < timeSteps; t++) {
-			System.arraycopy(data1[t], 0, data[t], 0, data1Variables);
-			System.arraycopy(data2[t], 0, data[t], data1Variables, data2Variables);
-		}
-		// Now defer to the normal setObservations method
-		addObservations(data);
+		super();
+		noiseLevel = (double) 1e-8; // Default to align with KSG estimators
+	    addNoise = true;
 	}
 
 	/**
@@ -298,12 +85,12 @@ public class EntropyCalculatorMultiVariateKozachenko
 		}
 		double sdTermHere = sdTerm(totalObservations, dimensions);
 		double emConstHere = eulerMascheroniTerm(totalObservations);
-		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData);
+		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(observations);
 		double entropy = 0.0;
 		if (debug) {
 			System.out.println("t,\tminDist,\tlogMinDist,\tsum");
 		}
-		for (int t = 0; t < rawData.length; t++) {
+		for (int t = 0; t < observations.length; t++) {
 			entropy += Math.log(2.0 * minDistance[t]);
 			if (debug) {
 				System.out.println(t + ",\t" + 
@@ -332,21 +119,18 @@ public class EntropyCalculatorMultiVariateKozachenko
 	 */
 	@Override
 	public double[] computeLocalOfPreviousObservations() {
-		if (lastLocalEntropy != null) {
-			return lastLocalEntropy;
-		}
 
 		double sdTermHere = sdTerm(totalObservations, dimensions);
 		double emConstHere = eulerMascheroniTerm(totalObservations);
 		double constantToAddIn = sdTermHere + emConstHere;
 		
-		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData);
+		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(observations);
 		double entropy = 0.0;
-		double[] localEntropy = new double[rawData.length];
+		double[] localEntropy = new double[observations.length];
 		if (debug) {
 			System.out.println("t,\tminDist,\tlogMinDist,\tlocal,\tsum");
 		}
-		for (int t = 0; t < rawData.length; t++) {
+		for (int t = 0; t < observations.length; t++) {
 			localEntropy[t] = Math.log(2.0 * minDistance[t]) * (double) dimensions;
 			// using natural units
 			// localEntropy[t] /= Math.log(2);
@@ -362,7 +146,7 @@ public class EntropyCalculatorMultiVariateKozachenko
 		}
 		entropy /= (double) totalObservations;
 		lastAverage = entropy;
-		lastLocalEntropy = localEntropy;
+		isComputed = true;
 		return localEntropy;
 	}
 
@@ -375,13 +159,13 @@ public class EntropyCalculatorMultiVariateKozachenko
 		double emConstHere = eulerMascheroniTerm(totalObservations);
 		double constantToAddIn = sdTermHere + emConstHere;
 		
-		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(rawData);
+		double[] minDistance = EuclideanUtils.computeMinEuclideanDistances(observations);
 		double entropy = 0.0;
-		double[] localEntropy = new double[rawData.length];
+		double[] localEntropy = new double[observations.length];
 		if (debug) {
 			System.out.println("t,\tminDist,\tlogMinDist,\tlocal,\tsum");
 		}
-		for (int t = 0; t < rawData.length; t++) {
+		for (int t = 0; t < observations.length; t++) {
 			localEntropy[t] = Math.log(2.0 * minDistance[t]) * (double) dimensions;
 			// using natural units
 			// localEntropy[t] /= Math.log(2);
@@ -395,9 +179,6 @@ public class EntropyCalculatorMultiVariateKozachenko
 						entropy);
 			}
 		}
-		entropy /= (double) totalObservations;
-		lastAverage = entropy;
-		lastLocalEntropy = localEntropy;
 		return localEntropy;
 	}
 
@@ -473,18 +254,4 @@ public class EntropyCalculatorMultiVariateKozachenko
 		return result;
 	}
 	
-	@Override
-	public void setDebug(boolean debug) {
-		this.debug = debug;
-	}
-
-	@Override
-	public double getLastAverage() {
-		return lastAverage;
-	}
-
-	@Override
-	public int getNumObservations() {
-		return totalObservations;
-	}
 }