Fixed analytic expressions for transfer entropies in the simple demos for coupled Gaussians (should have computed directly from the empirical correlations all along, but didn't notice because errors were very small)

demos/clojure/examples/example3TeContinuousDataKernel.clj

@@ -45,6 +45,7 @@
 
 (.setObservations teCalc sourceArray destArray)
 ; For copied source, should give something close to expected value for correlated Gaussians:
+; TODO The analytic result quoted here isn't quite right, see e.g. octave demos (can't be bothered fixing here...)
 (println "TE result " (.computeAverageLocalOfObservations teCalc)
 	" expected to be close to " (/ (Math/log (/ 1 (- 1 (* covariance covariance)))) (Math/log 2))
 	" for these correlated Gaussians but biased upward")

demos/clojure/examples/example4TeContinuousDataKraskov.clj

@@ -49,6 +49,7 @@
 ;  data is a set of random variables) - the result will be of the order
 ;  of what we expect, but not exactly equal to it; in fact, there will
 ;  be a large variance around it.
+; TODO The analytic result quoted here isn't quite right, see e.g. octave demos (can't be bothered fixing here...)
 (println "TE result " (.computeAverageLocalOfObservations teCalc)
 	" nats expected to be close to " (Math/log (/ 1 (- 1 (* covariance covariance))))
 	" nats for these correlated Gaussians")

demos/java/infodynamics/demos/Example3TeContinuousDataKernel.java

@@ -58,11 +58,16 @@ public class Example3TeContinuousDataKernel {
 		teCalc.setProperty("NORMALISE", "true"); // Normalise the individual variables (default)
 		teCalc.initialise(1, 0.5); // Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units
 		teCalc.setObservations(sourceArray, destArray);
-		// For copied source, should give something close to expected value for correlated Gaussians:
 		double result = teCalc.computeAverageLocalOfObservations();
+		// For copied source, should give something close to expected value for correlated Gaussians:.
+		// Expected correlation is expected covariance / product of expected standard deviations:
+		//  (where square of destArray standard dev is sum of squares of std devs of
+		//  underlying distributions)
+		double corr_expected = covariance /
+				(1.0 * Math.sqrt(Math.pow(covariance,2) + Math.pow(1-covariance,2)));
 		System.out.printf("TE result %.4f bits; expected to be close to " +
 				"%.4f bits for these correlated Gaussians but biased upwards\n",
-				result, Math.log(1.0/(1-Math.pow(covariance,2)))/Math.log(2));
+				result, -0.5 * Math.log(1-Math.pow(corr_expected,2))/Math.log(2));
 
 		teCalc.initialise(); // Initialise leaving the parameters the same
 		teCalc.setObservations(sourceArray2, destArray);

demos/java/infodynamics/demos/Example4TeContinuousDataKraskov.java

@@ -65,9 +65,14 @@ public class Example4TeContinuousDataKraskov {
 		//  data is a set of random variables) - the result will be of the order
 		//  of what we expect, but not exactly equal to it; in fact, there will
 		//  be a large variance around it.
+		// Expected correlation is expected covariance / product of expected standard deviations:
+		//  (where square of destArray standard dev is sum of squares of std devs of
+		//  underlying distributions)
+		double corr_expected = covariance /
+				(1.0 * Math.sqrt(Math.pow(covariance,2) + Math.pow(1-covariance,2)));
 		System.out.printf("TE result %.4f nats; expected to be close to " +
 				"%.4f nats for these correlated Gaussians\n",
-				result, Math.log(1.0/(1-Math.pow(covariance,2))));
+				result, -0.5 * Math.log(1-Math.pow(corr_expected,2)));
 
 		//  Perform calculation with uncorrelated source:
 		teCalc.initialise(); // Initialise leaving the parameters the same

demos/julia/example3TeContinuousDataKernel.jl

@@ -42,9 +42,13 @@ jcall(teCalc, "setProperty", Void, (JString, JString), "NORMALISE", "true"); # N
 jcall(teCalc, "initialise", Void, (jint, jdouble), 1, 0.5); # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units
 jcall(teCalc, "setObservations", Void, (Array{jdouble,1}, Array{jdouble,1}),
 				     sourceArray, destArray);
-# For copied source, should give something close to expected value for correlated Gaussians:
 result = jcall(teCalc, "computeAverageLocalOfObservations", jdouble, ());
-@printf("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n", result, log(1/(1-covariance^2))/log(2));
+# For copied source, should give something close to expected value for correlated Gaussians:
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected = covariance / (1 * sqrt(covariance^2 + (1-covariance)^2));
+@printf("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n", result, -0.5*log(1-corr_expected^2)/log(2));
 
 jcall(teCalc, "initialise", Void, ()); # Initialise leaving the parameters the same
 jcall(teCalc, "setObservations", Void, (Array{jdouble,1}, Array{jdouble,1}),

demos/julia/example4TeContinuousDataKraskov.jl

@@ -49,8 +49,12 @@ result = jcall(teCalc, "computeAverageLocalOfObservations", jdouble, ());
 #  data is a set of random variables) - the result will be of the order
 #  of what we expect, but not exactly equal to it; in fact, there will
 #  be a large variance around it.
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected = covariance / (1 * sqrt(covariance^2 + (1-covariance)^2));
 @printf("TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians\n",
-    result, log(1/(1-covariance^2)));
+    result, -0.5*log(1-corr_expected^2));
 
 # Perform calculation with uncorrelated source:
 jcall(teCalc, "initialise", Void, ()); # Initialise leaving the parameters the same

demos/octave/example3TeContinuousDataKernel.m

@@ -34,10 +34,14 @@ teCalc=javaObject('infodynamics.measures.continuous.kernel.TransferEntropyCalcul
 teCalc.setProperty('NORMALISE', 'true'); % Normalise the individual variables
 teCalc.initialise(1, 0.5); % Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units
 teCalc.setObservations(sourceArray, destArray);
-% For copied source, should give something close to expected value for correlated Gaussians:
 result = teCalc.computeAverageLocalOfObservations();
+% For copied source, should give something close to expected value for correlated Gaussians.
+% Expected correlation is expected covariance / product of expected standard deviations:
+%  (where square of destArray standard dev is sum of squares of std devs of
+%  underlying distributions)
+corr_expected = covariance ./ (1 * sqrt(covariance^2 + (1-covariance)^2));
 fprintf('TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards\n', ...
-    result, log(1/(1-covariance^2))/log(2));
+    result, -0.5*log(1-corr_expected^2)/log(2));
 teCalc.initialise(); % Initialise leaving the parameters the same
 teCalc.setObservations(sourceArray2, destArray);
 % For random source, it should give something close to 0 bits

demos/octave/example4TeContinuousDataKraskov.m

@@ -40,8 +40,12 @@ result = teCalc.computeAverageLocalOfObservations();
 %  data is a set of random variables) - the result will be of the order
 %  of what we expect, but not exactly equal to it; in fact, there will
 %  be a large variance around it.
+% Expected correlation is expected covariance / product of expected standard deviations:
+%  (where square of destArray standard dev is sum of squares of std devs of
+%  underlying distributions)
+corr_expected = covariance ./ (1 * sqrt(covariance^2 + (1-covariance)^2));
 fprintf('TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians\n', ...
-    result, log(1/(1-covariance^2)));
+    result, - 0.5 * log(1-corr_expected^2));
 % Perform calculation with uncorrelated source:
 teCalc.initialise(); % Initialise leaving the parameters the same
 teCalc.setObservations(sourceArray2, destArray);

demos/python/example3TeContinuousDataKernel.py

@@ -45,10 +45,14 @@ teCalc = teCalcClass()
 teCalc.setProperty("NORMALISE", "true") # Normalise the individual variables
 teCalc.initialise(1, 0.5) # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units
 teCalc.setObservations(JArray(JDouble, 1)(sourceArray), JArray(JDouble, 1)(destArray))
-# For copied source, should give something close to 1 bit:
 result = teCalc.computeAverageLocalOfObservations()
+# For copied source, should give something close to 1 bit:
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected = covariance / (1 * math.sqrt(covariance**2 + (1-covariance)**2));
 print("TE result %.4f bits; expected to be close to %.4f bits for these correlated Gaussians but biased upwards" % \
-    (result, math.log(1/(1-math.pow(covariance,2)))/math.log(2)))
+    (result, -0.5*math.log(1-math.pow(corr_expected,2))/math.log(2)))
 teCalc.initialise() # Initialise leaving the parameters the same
 teCalc.setObservations(JArray(JDouble, 1)(sourceArray2), JArray(JDouble, 1)(destArray))
 # For random source, it should give something close to 0 bits

demos/python/example4TeContinuousDataKraskov.py

@@ -52,8 +52,12 @@ result = teCalc.computeAverageLocalOfObservations()
 #  data is a set of random variables) - the result will be of the order
 #  of what we expect, but not exactly equal to it; in fact, there will
 #  be a large variance around it.
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected = covariance / (1 * math.sqrt(covariance**2 + (1-covariance)**2));
 print("TE result %.4f nats; expected to be close to %.4f nats for these correlated Gaussians" % \
-    (result, math.log(1/(1-math.pow(covariance,2)))))
+    (result, -0.5 * math.log(1-corr_expected**2)))
 # Perform calculation with uncorrelated source:
 teCalc.initialise() # Initialise leaving the parameters the same
 teCalc.setObservations(JArray(JDouble, 1)(sourceArray2), JArray(JDouble, 1)(destArray))

demos/r/example3TeContinuousDataKernel.r

@@ -41,9 +41,13 @@ teCalc<-.jnew("infodynamics/measures/continuous/kernel/TransferEntropyCalculator
 .jcall(teCalc,"V","setProperty", "NORMALISE", "true") # Normalise the individual variables
 .jcall(teCalc,"V","initialise", 1L, 0.5) # Use history length 1 (Schreiber k=1), kernel width of 0.5 normalised units
 .jcall(teCalc,"V","setObservations", sourceArray, destArray)
-# For copied source, should give something close to expected value for correlated Gaussians:
 result <- .jcall(teCalc,"D","computeAverageLocalOfObservations")
-cat("TE result ",  result, "bits; expected to be close to ", log(1/(1-covariance^2))/log(2), " bits for these correlated Gaussians but biased upwards\n")
+# For copied source, should give something close to expected value for correlated Gaussians.
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected <- covariance / (1 * sqrt(covariance^2 + (1-covariance)^2));
+cat("TE result ",  result, "bits; expected to be close to ", -0.5*log(1-corr_expected^2)/log(2), " bits for these correlated Gaussians but biased upwards\n")
 
 .jcall(teCalc,"V","initialise") # Initialise leaving the parameters the same
 .jcall(teCalc,"V","setObservations", sourceArray2, destArray)

demos/r/example4TeContinuousDataKraskov.r

@@ -48,7 +48,11 @@ result <- .jcall(teCalc,"D","computeAverageLocalOfObservations")
 #  data is a set of random variables) - the result will be of the order
 #  of what we expect, but not exactly equal to it; in fact, there will
 #  be a large variance around it.
-cat("TE result ",  result, "nats; expected to be close to ", log(1/(1-covariance^2)), " nats for these correlated Gaussians\n")
+# Expected correlation is expected covariance / product of expected standard deviations:
+#  (where square of destArray standard dev is sum of squares of std devs of
+#  underlying distributions)
+corr_expected <- covariance / (1 * sqrt(covariance^2 + (1-covariance)^2));
+cat("TE result ",  result, "nats; expected to be close to ", -0.5*log(1-corr_expected^2), " nats for these correlated Gaussians\n")
 
 # Perform calculation with uncorrelated source:
 .jcall(teCalc,"V","initialise") # Initialise leaving the parameters the same