Altering TE Kraskov to do auto-embedding properly when we have added multiple observations with validity vectors. Also taking the opportunity to re-engineer the internals to lay groundwork for auto-embedding function to be shared with the Gaussian estimator at a later stage. Unit tests added here as well for the above.

java/source/infodynamics/measures/continuous/TransferEntropyCalculatorMultiVariateViaCondMutualInfo.java

@@ -335,7 +335,8 @@ public class TransferEntropyCalculatorMultiVariateViaCondMutualInfo
 			boolean[] sourceValid, boolean[] destValid) throws Exception {
 		
 		// Compute the start and end time pairs using our embedding parameters:
-		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(
+				k, k_tau, l, l_tau, delay, sourceValid, destValid);
 		
 		int totalObservationsAdded = 0;
 		for (int[] timePair : startAndEndTimePairs) {
@@ -433,7 +434,8 @@ public class TransferEntropyCalculatorMultiVariateViaCondMutualInfo
  			return;
  		}
 
- 		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+ 		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(
+ 				k, k_tau, l, l_tau, delay, sourceValid, destValid);
  		
  		// We've found the set of start and end times for this pair
  		startAddObservations();

java/source/infodynamics/measures/continuous/TransferEntropyCalculatorViaCondMutualInfo.java

@@ -229,7 +229,8 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 		this.l_tau = l_tau;
 		this.delay = delay;
 		
-		setStartTimeForFirstDestEmbedding();
+		startTimeForFirstDestEmbedding =
+				computeStartTimeForFirstDestEmbedding(k, k_tau, l, l_tau, delay);
 
 		vectorOfSourceTimeSeries = null;
 		vectorOfDestinationTimeSeries = null;
@@ -242,16 +243,22 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 	 * Protected internal method to 
 	 * set the point at which we can start taking observations from in any
 	 * addObservations call.
+	 * 
+	 * User supplied parameters for k etc, so this can be used not only to
+	 *  set the definitive startTimeForFirstDestEmbedding but also when
+	 *  we're searching the parameter space in auto-embedding.
+	 * 
 	 */
-	protected void setStartTimeForFirstDestEmbedding() {
+	protected static int computeStartTimeForFirstDestEmbedding(
+			int k_in_use, int k_tau_in_use, int l_in_use, int l_tau_in_use, int delay_in_use) {
 		// These two integers represent the last
 		// point of the destination embedding, in the cases where the destination
 		// embedding itself determines where we can start taking observations, or
 		// the case where the source embedding plus delay is longer and so determines
 		// where we can start taking observations.
-		int startTimeBasedOnDestPast = (k-1)*k_tau;
-		int startTimeBasedOnSourcePast = (l-1)*l_tau + delay - 1;
-		startTimeForFirstDestEmbedding = Math.max(startTimeBasedOnDestPast, startTimeBasedOnSourcePast);
+		int startTimeBasedOnDestPast = (k_in_use-1)*k_tau_in_use;
+		int startTimeBasedOnSourcePast = (l_in_use-1)*l_tau_in_use + delay_in_use - 1;
+		return Math.max(startTimeBasedOnDestPast, startTimeBasedOnSourcePast);
 	}
 	
 	/**
@@ -363,50 +370,75 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 
 	/**
 	 * Protected method to internally parse and submit observations through
-	 *  to the underlying conditional MI calculator once any internal parameter settings
-	 *  have been finalised (in the case of automatically determining the embedding
-	 *  parameters) 
+	 *  to the given conditional MI calculator with specific embedding parameter settings
+	 *  supplied.
+	 * This may be used in the final calculation, or by the auto-embedding 
+	 *  procedures, hence the use of static method and arguments rather than
+	 *  using any member variables directly.
 	 * 
+	 * @param condMiCalc_in_use conditional MI calculator to use 
+	 * @param k_in_use k embedding dimension to use for target
+	 * @param k_tau_in_use target tau embedding delay to use
+	 * @param l_in_use l embedding dimension to use for source
+	 * @param l_tau_in_use source tau embedding delay to use
+	 * @param delay_in_use source-target delay to use
 	 * @param source time series of source observations
 	 * @param destination time series of destination observations
 	 * @return the number of observations added
 	 * @throws Exception
 	 */
-	protected int addObservationsAfterParamsDetermined(double[] source, double[] destination) throws Exception {
+	protected static int addObservationsWithGivenParams(
+			ConditionalMutualInfoCalculatorMultiVariate condMiCalc_in_use,
+			int k_in_use, int k_tau_in_use, int l_in_use, int l_tau_in_use,
+			int delay_in_use,
+			double[] source, double[] destination) throws Exception {
 		if (source.length != destination.length) {
 			throw new Exception(String.format("Source and destination lengths (%d and %d) must match!",
 					source.length, destination.length));
 		}
-		if (source.length < startTimeForFirstDestEmbedding + 2) {
+		int startTimeForFirstDestEmbedding_in_use =
+				computeStartTimeForFirstDestEmbedding(k_in_use, k_tau_in_use,
+						l_in_use, l_tau_in_use, delay_in_use);
+		if (source.length < startTimeForFirstDestEmbedding_in_use + 2) {
 			// There are no observations to add here, the time series is too short
 			// Don't throw an exception, do nothing since more observations
 			//  can be added later.
 			return 0;
 		}
 		double[][] currentDestPastVectors = 
-				MatrixUtils.makeDelayEmbeddingVector(destination, k, k_tau,
-						startTimeForFirstDestEmbedding,
-						destination.length - startTimeForFirstDestEmbedding - 1);
+				MatrixUtils.makeDelayEmbeddingVector(destination, k_in_use, k_tau_in_use,
+						startTimeForFirstDestEmbedding_in_use,
+						destination.length - startTimeForFirstDestEmbedding_in_use - 1);
 		double[][] currentDestNextVectors =
 				MatrixUtils.makeDelayEmbeddingVector(destination, 1,
-						startTimeForFirstDestEmbedding + 1,
-						destination.length - startTimeForFirstDestEmbedding - 1);
+						startTimeForFirstDestEmbedding_in_use + 1,
+						destination.length - startTimeForFirstDestEmbedding_in_use - 1);
 		double[][] currentSourcePastVectors = 
-				MatrixUtils.makeDelayEmbeddingVector(source, l, l_tau,
-						startTimeForFirstDestEmbedding + 1 - delay,
-						source.length - startTimeForFirstDestEmbedding - 1);
-		condMiCalc.addObservations(currentSourcePastVectors, currentDestNextVectors, currentDestPastVectors);
-		return destination.length - startTimeForFirstDestEmbedding - 1;
+				MatrixUtils.makeDelayEmbeddingVector(source, l_in_use, l_tau_in_use,
+						startTimeForFirstDestEmbedding_in_use + 1 - delay_in_use,
+						source.length - startTimeForFirstDestEmbedding_in_use - 1);
+		condMiCalc_in_use.addObservations(currentSourcePastVectors, currentDestNextVectors, currentDestPastVectors);
+		return destination.length - startTimeForFirstDestEmbedding_in_use - 1;
 	}
 
 	/**
 	 * Protected method to internally parse and submit observations through
-	 *  to the underlying conditional MI calculator once any internal parameter settings
-	 *  have been finalised (in the case of automatically determining the embedding
-	 *  parameters) 
+	 *  to the given conditional MI calculator with specific embedding parameter settings
+	 *  supplied.
 	 * This is done given time-series of booleans indicating whether each entry
 	 *  is valid
+	 * This may be used in the final calculation, or by the auto-embedding 
+	 *  procedures, hence the use of static method and arguments rather than
+	 *  using any member variables directly.
 	 * 
+	 * @param condMiCalc_in_use conditional MI calculator to use
+	 * @param k_in_use k embedding dimension to use for target
+	 * @param k_tau_in_use target tau embedding delay to use
+	 * @param l_in_use l embedding dimension to use for source
+	 * @param l_tau_in_use source tau embedding delay to use
+	 * @param delay_in_use source-target delay to use
+	 * @param source time series of source observations
+	 * @param destination time series of destination observations
 	 * @param source time series of source observations
 	 * @param destination time series of destination observations
 	 * @param sourceValid array (with indices the same as source) indicating whether
@@ -416,17 +448,25 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 	 * @return total number of observations added 
 	 * @throws Exception
 	 */
-	protected int addObservationsAfterParamsDetermined(double[] source, double[] destination,
+	protected static int addObservationsWithGivenParams(
+			ConditionalMutualInfoCalculatorMultiVariate condMiCalc_in_use,
+			int k_in_use, int k_tau_in_use, int l_in_use, int l_tau_in_use,
+			int delay_in_use,
+			double[] source, double[] destination,
 			boolean[] sourceValid, boolean[] destValid) throws Exception {
 		
 		// Compute the start and end time pairs using our embedding parameters:
-		Vector<int[]> startAndEndTimePairs = computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> startAndEndTimePairs =
+				computeStartAndEndTimePairs(k_in_use, k_tau_in_use, l_in_use,
+						l_tau_in_use, delay_in_use, sourceValid, destValid);
 		
 		int totalObservationsAdded = 0;
 		for (int[] timePair : startAndEndTimePairs) {
 			int startTime = timePair[0];
 			int endTime = timePair[1];
-			totalObservationsAdded += addObservationsAfterParamsDetermined(
+			totalObservationsAdded += addObservationsWithGivenParams(
+					condMiCalc_in_use, k_in_use, k_tau_in_use, l_in_use,
+					l_tau_in_use, delay_in_use,
 					MatrixUtils.select(source, startTime, endTime - startTime + 1),
 					MatrixUtils.select(destination, startTime, endTime - startTime + 1));
 		}
@@ -466,14 +506,44 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 		// Auto embed if required
 		preFinaliseAddObservations();
 
+		separateNumObservations = prepareCMICalculator(condMiCalc, k, k_tau, l, l_tau, delay);
+		
+		vectorOfSourceTimeSeries = null; // No longer required
+		vectorOfDestinationTimeSeries = null; // No longer required
+		vectorOfValidityOfSource = null;
+		vectorOfValidityOfDestination = null;
+	}
+
+	/**
+	 * Prepare the given pre-instantiated (and properties supplied)
+	 *  Conditional mutual information calculator with this data set,
+	 *  using the embedding parameters supplied.
+	 * This may be used in the final calculation, or by the auto-embedding 
+	 *  procedures, hence the use of method arguments rather than
+	 *  using the member variables directly.
+	 * 
+	 * @param condMiCalc_in_use conditional MI calculator to use
+	 * @param k_in_use k embedding dimension to use for target
+	 * @param k_tau_in_use target tau embedding delay to use
+	 * @param l_in_use l embedding dimension to use for source
+	 * @param l_tau_in_use source tau embedding delay to use
+	 * @param delay_in_use source-target delay to use
+	 * @return integer array of number of samples added for each time series pair in the sample set.
+	 * @throws Exception
+	 */
+	protected int[] prepareCMICalculator(
+			ConditionalMutualInfoCalculatorMultiVariate condMiCalc_in_use,
+			int k_in_use, int k_tau_in_use, int l_in_use, int l_tau_in_use,
+			int delay_in_use) throws Exception {
+
 		// Initialise the conditional MI calculator, including any auto-embedding length
-		condMiCalc.initialise(l, 1, k);
-		condMiCalc.startAddObservations();
+		condMiCalc_in_use.initialise(l_in_use, 1, k_in_use);
+		condMiCalc_in_use.startAddObservations();
 		// Send all of the observations through:
 		Iterator<double[]> destIterator = vectorOfDestinationTimeSeries.iterator();
 		Iterator<boolean[]> sourceValidityIterator = vectorOfValidityOfSource.iterator();
 		Iterator<boolean[]> destValidityIterator = vectorOfValidityOfDestination.iterator();
-		separateNumObservations = new int[vectorOfDestinationTimeSeries.size()];
+		int[] separateNumObservationsArray = new int[vectorOfDestinationTimeSeries.size()];
 		int setNum = 0;
 		for (double[] source : vectorOfSourceTimeSeries) {
 			double[] destination = destIterator.next();
@@ -482,22 +552,24 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 			int observationsAddedThisTime = 0;
 			if (sourceValidity == null) {
 				// Add the whole time-series
-				observationsAddedThisTime = addObservationsAfterParamsDetermined(source, destination);
+				observationsAddedThisTime = addObservationsWithGivenParams(
+						condMiCalc_in_use, k_in_use, k_tau_in_use, l_in_use,
+						l_tau_in_use, delay_in_use, source, destination);
 			} else {
-				observationsAddedThisTime = addObservationsAfterParamsDetermined(source, destination,
+				observationsAddedThisTime = addObservationsWithGivenParams(
+						condMiCalc_in_use, k_in_use, k_tau_in_use, l_in_use,
+						l_tau_in_use, delay_in_use, source, destination,
 						sourceValidity, destValidity);
 			}
-			separateNumObservations[setNum++] = observationsAddedThisTime;
+			separateNumObservationsArray[setNum++] = observationsAddedThisTime;
 		}
-		vectorOfSourceTimeSeries = null; // No longer required
-		vectorOfDestinationTimeSeries = null; // No longer required
-		vectorOfValidityOfSource = null;
-		vectorOfValidityOfDestination = null;
 		
 		// TODO do we need to throw an exception if there are no observations to add?
-		condMiCalc.finaliseAddObservations();
+		condMiCalc_in_use.finaliseAddObservations();
+		
+		return separateNumObservationsArray;
 	}
-
+	
 	@Override
 	public void setObservations(double[] source, double[] destination,
 			boolean[] sourceValid, boolean[] destValid) throws Exception {
@@ -521,21 +593,29 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 	 * <p>Made public so it can be used if one wants to compute the number of
 	 *  observations prior to setting the observations.</p>
 	 * 
+	 * @param k_in_use k embedding dimension to use for target
+	 * @param k_tau_in_use target tau embedding delay to use
+	 * @param l_in_use l embedding dimension to use for source
+	 * @param l_tau_in_use source tau embedding delay to use
+	 * @param delay_in_use source-target delay to use
 	 * @param sourceValid a time series (with indices the same as observations)
 	 *  indicating whether the entry in observations at that index is valid for the source; 
 	 * @param destValid as described for <code>sourceValid</code>
 	 * @return a vector for start and end time pairs of valid series
 	 *  of observations.
 	 */
-	public Vector<int[]> computeStartAndEndTimePairs(boolean[] sourceValid, boolean[] destValid) throws Exception {
+	public static Vector<int[]> computeStartAndEndTimePairs(
+			int k_in_use, int k_tau_in_use, int l_in_use, int l_tau_in_use,
+			int delay_in_use,
+			boolean[] sourceValid, boolean[] destValid) throws Exception {
 		
 		if (sourceValid.length != destValid.length) {
 			throw new Exception("Validity arrays must be of same length");
 		}
 		
-		int lengthOfDestPastRequired = (k-1)*k_tau + 1;
-		int lengthOfSourcePastRequired = (l-1)*l_tau + 1;
-		// int numSourcePointsBeforeDestStart = delay - 1 + lengthOfSourcePastRequired
+		int lengthOfDestPastRequired = (k_in_use-1)*k_tau_in_use + 1;
+		int lengthOfSourcePastRequired = (l_in_use-1)*l_tau_in_use + 1;
+		// int numSourcePointsBeforeDestStart = delay_in_use - 1 + lengthOfSourcePastRequired
 		//									- lengthOfDestPastRequired;
 
 		// Scan along the data avoiding invalid values
@@ -545,12 +625,15 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 		// Simple solution -- this takes more complexity in time, but is 
 		//  much faster to code:
 		boolean previousWasOk = false;
-		for (int t = startTimeForFirstDestEmbedding; t < destValid.length - 1; t++) {
+		int startTimeForFirstDestEmbedding_in_use =
+				computeStartTimeForFirstDestEmbedding(k_in_use, k_tau_in_use, l_in_use,
+						l_tau_in_use, delay_in_use);
+		for (int t = startTimeForFirstDestEmbedding_in_use; t < destValid.length - 1; t++) {
 			// Check the tuple with the history vector starting from
 			//  t and running backwards
 			if (previousWasOk) {
 				// Just check the very next values of each:
-				if (destValid[t + 1] && sourceValid[t + 1 - delay]) {
+				if (destValid[t + 1] && sourceValid[t + 1 - delay_in_use]) {
 					// We can continue adding to this sequence
 					continue;
 				} else {
@@ -578,7 +661,7 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 				continue;
 			}
 			allOk = true;
-			for (int tBack = delay - 1; tBack < delay - 1 + lengthOfSourcePastRequired; tBack++) {
+			for (int tBack = delay_in_use - 1; tBack < delay_in_use - 1 + lengthOfSourcePastRequired; tBack++) {
 				if (!sourceValid[t - tBack]) {
 					allOk = false;
 					break;
@@ -588,7 +671,7 @@ public class TransferEntropyCalculatorViaCondMutualInfo implements
 				continue;
 			}
 			// Postcondition: We've got a first valid tuple:
-			startTime = t - startTimeForFirstDestEmbedding;
+			startTime = t - startTimeForFirstDestEmbedding_in_use;
 			previousWasOk = true;
 		}
 		// Now check if we were running a sequence and terminate it:

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorKraskov.java

@@ -20,6 +20,7 @@ package infodynamics.measures.continuous.kraskov;
 
 import java.util.Hashtable;
 import java.util.Iterator;
+import java.util.Vector;
 
 import infodynamics.measures.continuous.ActiveInfoStorageCalculator;
 import infodynamics.measures.continuous.ConditionalMutualInfoCalculatorMultiVariate;
@@ -451,12 +452,7 @@ public class TransferEntropyCalculatorKraskov
 			if (debug) {
 				System.out.println("Starting embedding of destination:");
 			}
-			aisCalc.initialise();
-			aisCalc.startAddObservations();
-			for (double[] destination : vectorOfDestinationTimeSeries) {
-				aisCalc.addObservations(destination);
-			}
-			aisCalc.finaliseAddObservations();
+			prepareAISCalculator(aisCalc, vectorOfDestinationTimeSeries, vectorOfValidityOfDestination);
 			// Set the auto-embedding parameters for the destination:
 			k = Integer.parseInt(aisCalc.getProperty(ActiveInfoStorageCalculator.K_PROP_NAME));
 			k_tau = Integer.parseInt(aisCalc.getProperty(ActiveInfoStorageCalculator.TAU_PROP_NAME));
@@ -471,12 +467,7 @@ public class TransferEntropyCalculatorKraskov
 				if (debug) {
 					System.out.println("Starting embedding of source:");
 				}
-				aisCalc.initialise();
-				aisCalc.startAddObservations();
-				for (double[] source : vectorOfSourceTimeSeries) {
-					aisCalc.addObservations(source);
-				}
-				aisCalc.finaliseAddObservations();
+				prepareAISCalculator(aisCalc, vectorOfSourceTimeSeries, vectorOfValidityOfSource);
 				// Set the auto-embedding parameters for the source:
 				l = Integer.parseInt(aisCalc.getProperty(ActiveInfoStorageCalculator.K_PROP_NAME));
 				l_tau = Integer.parseInt(aisCalc.getProperty(ActiveInfoStorageCalculator.TAU_PROP_NAME));
@@ -490,16 +481,6 @@ public class TransferEntropyCalculatorKraskov
 					System.out.println("Starting embedding of source:");
 				}
 
-				// Instantiate a new calculator to optimize the embedding parameters
-				TransferEntropyCalculatorKraskov teEmbeddingCalc =
-						new TransferEntropyCalculatorKraskov();
-
-				// Set all properties of the current calculator except embedding method
-				for (String key : props.keySet()) {
-					teEmbeddingCalc.setProperty(key, props.get(key));
-				}
-				teEmbeddingCalc.setProperty(PROP_AUTO_EMBED_METHOD, AUTO_EMBED_METHOD_NONE);
-
 				double bestTE = Double.NEGATIVE_INFINITY;
 				int l_candidate_best = 1;
 				int l_tau_candidate_best = 1;
@@ -508,16 +489,10 @@ public class TransferEntropyCalculatorKraskov
 				for (int l_candidate = 1; l_candidate <= k_search_max; l_candidate++) {
 					for (int l_tau_candidate = 1; l_tau_candidate <= tau_search_max; l_tau_candidate++) {
 
-						teEmbeddingCalc.initialise(k, k_tau, l_candidate, l_tau_candidate, delay);
-						teEmbeddingCalc.startAddObservations();
-
-						Iterator<double[]> destIterator = vectorOfDestinationTimeSeries.iterator();
-						for (double[] source : vectorOfSourceTimeSeries) {
-							double[] dest = destIterator.next();
-							teEmbeddingCalc.addObservations(source, dest);
-						}
-						teEmbeddingCalc.finaliseAddObservations();
-						double thisTE = teEmbeddingCalc.computeAverageLocalOfObservations();
+						// Use our internal CMI calculator in case it has any particular 
+						//  properties we need to have been set already
+						prepareCMICalculator(condMiCalc, k, k_tau, l_candidate, l_tau_candidate, delay);
+						double thisTE = condMiCalc.computeAverageLocalOfObservations();
 
 						if (debug) {
 							System.out.printf("TE for l=%d, l_tau=%d is %.3f\n",
@@ -546,7 +521,36 @@ public class TransferEntropyCalculatorKraskov
 			}
 	
 			// Now that embedding parameters are finalised:
-			setStartTimeForFirstDestEmbedding();
+			startTimeForFirstDestEmbedding =
+					computeStartTimeForFirstDestEmbedding(k, k_tau, l, l_tau, delay);
 		}
 	}
+	
+	/**
+	 * Prepare the given pre-instantiated (and properties supplied)
+	 *  Active information storage calculator with the given data set,
+	 *  for a calculation of auto-embedding parameters.
+	 * 
+	 * @param aisCalc_in_use AIS calculator to supply
+	 * @param setOfTimeSeriesSamples set of times series samples for the calculation
+	 * @param setOfValidities set of time series of validity indications. Each can be a null array if all are valid
+	 * @throws Exception
+	 */
+	protected static void prepareAISCalculator(ActiveInfoStorageCalculator aisCalc,
+			Vector<double[]> setOfTimeSeriesSamples, Vector<boolean[]> setOfValidities) 
+					throws Exception {
+		
+		aisCalc.initialise();
+		aisCalc.startAddObservations();
+		Iterator<boolean[]> validityIterator = setOfValidities.iterator();
+		for (double[] timeSeries : setOfTimeSeriesSamples) {
+			boolean[] validity = validityIterator.next();
+			if (validity == null) {
+				aisCalc.addObservations(timeSeries);
+			} else {
+				aisCalc.addObservations(timeSeries, validity);
+			}
+		}
+		aisCalc.finaliseAddObservations();
+	}
 }

java/source/infodynamics/measures/continuous/kraskov/TransferEntropyCalculatorMultiVariateKraskov.java

@@ -516,7 +516,8 @@ public class TransferEntropyCalculatorMultiVariateKraskov
       }
 	
 			// Now that embedding parameters are finalised:
-			setStartTimeForFirstDestEmbedding();
+			startTimeForFirstDestEmbedding =
+					computeStartTimeForFirstDestEmbedding(k, k_tau, l, l_tau, delay);
 		}
 	}
 }

java/source/infodynamics/networkinference/interregional/InterregionalTransferEntropy.java

@@ -19,6 +19,7 @@
 package infodynamics.networkinference.interregional;
 
 import infodynamics.measures.continuous.TransferEntropyCalculatorMultiVariate;
+import infodynamics.measures.continuous.TransferEntropyCalculatorViaCondMutualInfo;
 import infodynamics.measures.continuous.kraskov.TransferEntropyCalculatorMultiVariateKraskov;
 import infodynamics.utils.ParsedProperties;
 
@@ -77,8 +78,11 @@ public class InterregionalTransferEntropy extends InterregionalChannelMeasure {
 			TransferEntropyCalculatorMultiVariateKraskov tecmvKras =
 				new TransferEntropyCalculatorMultiVariateKraskov();
 			tecmvKras.initialise(k);
-			Vector<int[]> startAndEndTimePairs = tecmvKras.computeStartAndEndTimePairs(
-					jointValidity1, jointValidity2);
+			Vector<int[]> startAndEndTimePairs =
+					TransferEntropyCalculatorViaCondMutualInfo.
+						computeStartAndEndTimePairs(
+						k, 1, 1, 1, 1,
+						jointValidity1, jointValidity2);
 			// We've found the set of start and end times for this pair
 			int obsNum = 0;
 			for (int[] timePair : startAndEndTimePairs) {
@@ -131,8 +135,11 @@ public class InterregionalTransferEntropy extends InterregionalChannelMeasure {
 			new TransferEntropyCalculatorMultiVariateKraskov();
 		
 		tecmvKras.initialise(k);
-		Vector<int[]> startAndEndTimePairs = tecmvKras.computeStartAndEndTimePairs(
-				sourceValid, destValid);
+		Vector<int[]> startAndEndTimePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs(
+					k, 1, 1, 1, 1,
+					sourceValid, destValid);
 		
 		// We've found the set of start and end times for this pair
 		int numObservations = 0;

java/unittests/infodynamics/measures/continuous/gaussian/TransferEntropyGaussianTester.java

@@ -21,6 +21,7 @@ package infodynamics.measures.continuous.gaussian;
 import java.util.Vector;
 
 import infodynamics.measures.continuous.TransferEntropyAbstractTester;
+import infodynamics.measures.continuous.TransferEntropyCalculatorViaCondMutualInfo;
 import infodynamics.utils.ArrayFileReader;
 import infodynamics.utils.MatrixUtils;
 import infodynamics.utils.RandomGenerator;
@@ -155,7 +156,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 		TransferEntropyCalculatorGaussian teCalc =
 				new TransferEntropyCalculatorGaussian();
 		teCalc.initialise(k, 1, 1, 1, 1);
-		Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> timePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs
+						(k, 1, 1, 1, 1, sourceValid, destValid);
 		// And compare to the existing 
 		Vector<int[]> expectedTimePairs = computeStartAndEndTimePairs(sourceValid, destValid, k);
 		
@@ -185,7 +189,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 		TransferEntropyCalculatorGaussian teCalc =
 				new TransferEntropyCalculatorGaussian();
 		teCalc.initialise(k, 1, 1, 1, 1);
-		Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> timePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs(
+						k, 1, 1, 1, 1, sourceValid, destValid);
 		
 		assertEquals(expected.size(), timePairs.size());				
 		for (int i = 0; i < timePairs.size(); i++) {
@@ -208,7 +215,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 		TransferEntropyCalculatorGaussian teCalc =
 				new TransferEntropyCalculatorGaussian();
 		teCalc.initialise(k, 1, 1, 1, 2);
-		Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> timePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs(
+						k, 1, 1, 1, 2, sourceValid, destValid);
 		
 		assertEquals(expected.size(), timePairs.size());				
 		for (int i = 0; i < timePairs.size(); i++) {
@@ -232,7 +242,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 		TransferEntropyCalculatorGaussian teCalc =
 				new TransferEntropyCalculatorGaussian();
 		teCalc.initialise(k, 1, 1, 1, 3);
-		Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> timePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs(
+						k, 1, 1, 1, 3, sourceValid, destValid);
 		
 		assertEquals(expected.size(), timePairs.size());				
 		for (int i = 0; i < timePairs.size(); i++) {
@@ -255,7 +268,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 		TransferEntropyCalculatorGaussian teCalc =
 				new TransferEntropyCalculatorGaussian();
 		teCalc.initialise(2, 2, 2, 2, 2);
-		Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+		Vector<int[]> timePairs =
+				TransferEntropyCalculatorViaCondMutualInfo.
+					computeStartAndEndTimePairs(
+						2, 2, 2, 2, 2, sourceValid, destValid);
 		
 		assertEquals(expected.size(), timePairs.size());				
 		for (int i = 0; i < timePairs.size(); i++) {
@@ -288,7 +304,10 @@ public class TransferEntropyGaussianTester extends TransferEntropyAbstractTester
 			TransferEntropyCalculatorGaussian teCalc =
 					new TransferEntropyCalculatorGaussian();
 			teCalc.initialise(k, 1, 1, 1, 1);
-			Vector<int[]> timePairs = teCalc.computeStartAndEndTimePairs(sourceValid, destValid);
+			Vector<int[]> timePairs =
+					TransferEntropyCalculatorViaCondMutualInfo.
+						computeStartAndEndTimePairs(
+							k, 1, 1, 1, 1, sourceValid, destValid);
 			// And compare to the existing 
 			Vector<int[]> expectedTimePairs = computeStartAndEndTimePairs(sourceValid, destValid, k);
 			

java/unittests/infodynamics/measures/continuous/kraskov/TransferEntropyTester.java

@@ -19,6 +19,7 @@
 package infodynamics.measures.continuous.kraskov;
 
 import java.util.Arrays;
+import java.util.Random;
 
 import infodynamics.utils.ArrayFileReader;
 import infodynamics.utils.MatrixUtils;
@@ -676,4 +677,50 @@ public class TransferEntropyTester
 		assertEquals(correctL, optimisedL);
   }
 
+  public void testAutoEmbeddingTEWithValidity() throws Exception {
+	    System.out.println("Start AIS+TE autoembedding test.");
+
+	    // Generate multivariate data (note that source time series has no memory)
+			RandomGenerator rg = new RandomGenerator();
+			double[] source = rg.generateNormalData(10000, 0, 1);
+			double[] target = rg.generateNormalData(10000, 0, 1);
+		boolean[] validity = new boolean[target.length];
+		Random random = new Random();
+		for (int t = 0; t < target.length; t++) {
+			if (random.nextDouble() < 0.05) {
+				validity[t] = false;
+			} else {
+				validity[t] = true;
+			}
+		}
+			
+
+	    for (int i=3; i < source.length; i++) {
+	      target[i] = 0.2*source[i-2] + 0.2*source[i-1] + 
+	                     0.2*target[i-2] + 0.2*target[i-1] + target[i];
+	    }
+
+	    int correctK = 2;
+	    int correctL = 2;
+
+	    // Instantiate calculator and set search bounds
+			TransferEntropyCalculatorKraskov teCalc = 
+					new TransferEntropyCalculatorKraskov();
+			teCalc.setProperty("k", "4");
+			teCalc.setProperty(teCalc.PROP_K_SEARCH_MAX, "2");
+			teCalc.setProperty(teCalc.PROP_TAU_SEARCH_MAX, "1");
+			teCalc.setProperty(teCalc.PROP_AUTO_EMBED_METHOD, teCalc.AUTO_EMBED_METHOD_MAX_CORR_AIS_AND_TE);
+	    teCalc.setDebug(true);
+
+	    // Run optimisation
+			teCalc.initialise();
+			teCalc.setObservations(source, target, validity, validity);
+			int optimisedK = Integer.parseInt(teCalc.getProperty(TransferEntropyCalculatorKraskov.K_PROP_NAME));
+			int optimisedL = Integer.parseInt(teCalc.getProperty(TransferEntropyCalculatorKraskov.L_PROP_NAME));
+
+	    // Test that answer was correct
+			assertEquals(correctK, optimisedK);
+			assertEquals(correctL, optimisedL);
+	  }
+
 }