Added discrete implementation of various multivariate IT measures and unit tests.

java/source/infodynamics/measures/discrete/DualTotalCorrelationCalculatorDiscrete.java

@@ -0,0 +1,84 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Computes the dual total correlation (DTC) of a given multivariate
+ *  <code>int[][]</code> set of
+ *  observations (extending {@link MultiVariateInfoMeasureCalculatorDiscrete}).</p>
+ *
+ * <p>Usage is as per the paradigm outlined for {@link MultiVariateInfoMeasureCalculatorDiscrete}.
+ * </p>
+ * 
+ * <p><b>References:</b><br/>
+ * <ul>
+ *  <li>Rosas, F., Mediano, P., Gastpar, M, Jensen, H.,
+ *   <a href="http://dx.doi.org/10.1103/PhysRevE.100.032305">"Quantifying high-order
+ *   interdependencies via multivariate extensions of the mutual information"</a>,
+ *   Physical Review E 100, (2019) 032305.</li>
+ * </ul>
+ *
+ * @author Pedro A.M. Mediano (<a href="pmediano at pm.me">email</a>,
+ * <a href="http://www.doc.ic.ac.uk/~pam213">www</a>)
+ */
+public class DualTotalCorrelationCalculatorDiscrete
+    extends MultiVariateInfoMeasureCalculatorDiscrete {
+  
+  /**
+   * Construct an instance.
+   * 
+   * @param base number of symbols for each variable.
+   *        E.g. binary variables are in base-2.
+   * @param numVars numbers of joint variables that DTC
+   *     will be computed over.
+   */
+  public DualTotalCorrelationCalculatorDiscrete(int base, int numVars) {
+    super(base, numVars);
+  }
+
+  protected double computeLocalValueForTuple(int[] tuple, int jointValue) {
+
+    if (jointCount[jointValue] == 0) {
+      // This joint state does not occur, so it makes no contribution here
+      return 0;
+    }
+
+    double jointProb = (double) jointCount[jointValue] / (double) observations;
+    double logValue = (numVars - 1) * Math.log(jointProb);
+
+    for (int i = 0; i < numVars; i++) {
+      int marginalState = computeBigMarginalState(jointValue, i, tuple[i]);
+      double marginalProb = (double) bigMarginalCounts[i][marginalState] / (double) observations;
+      logValue -= Math.log(marginalProb);
+    }
+
+    double localValue = logValue / log_2;
+
+    if (jointProb > 0.0) {
+      checkLocals(localValue);
+    }
+
+    return localValue;
+  }
+
+}
+

java/source/infodynamics/measures/discrete/OInfoCalculatorDiscrete.java

@@ -0,0 +1,85 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Computes the O-information of a given multivariate
+ *  <code>int[][]</code> set of
+ *  observations (extending {@link MultiVariateInfoMeasureCalculatorDiscrete}).</p>
+ *
+ * <p>Usage is as per the paradigm outlined for {@link MultiVariateInfoMeasureCalculatorDiscrete}.
+ * </p>
+ * 
+ * <p><b>References:</b><br/>
+ * <ul>
+ *  <li>Rosas, F., Mediano, P., Gastpar, M, Jensen, H.,
+ *   <a href="http://dx.doi.org/10.1103/PhysRevE.100.032305">"Quantifying high-order
+ *   interdependencies via multivariate extensions of the mutual information"</a>,
+ *   Physical Review E 100, (2019) 032305.</li>
+ * </ul>
+ *
+ * @author Pedro A.M. Mediano (<a href="pmediano at pm.me">email</a>,
+ * <a href="http://www.doc.ic.ac.uk/~pam213">www</a>)
+ */
+public class OInfoCalculatorDiscrete
+    extends MultiVariateInfoMeasureCalculatorDiscrete {
+  
+  /**
+   * Construct an instance.
+   * 
+   * @param base number of symbols for each variable.
+   *        E.g. binary variables are in base-2.
+   * @param numVars numbers of joint variables that DTC
+   *     will be computed over.
+   */
+  public OInfoCalculatorDiscrete(int base, int numVars) {
+    super(base, numVars);
+  }
+
+  protected double computeLocalValueForTuple(int[] tuple, int jointValue) {
+
+    if (jointCount[jointValue] == 0) {
+      // This joint state does not occur, so it makes no contribution here
+      return 0;
+    }
+
+    double jointProb = (double) jointCount[jointValue] / (double) observations;
+    double logValue = (2 - numVars) * Math.log(jointProb);
+
+    for (int i = 0; i < numVars; i++) {
+      int bigMarginalState = computeBigMarginalState(jointValue, i, tuple[i]);
+      double bigMarginalProb = (double) bigMarginalCounts[i][bigMarginalState] / (double) observations;
+      double smallMarginalProb = (double) smallMarginalCounts[i][tuple[i]] / (double) observations;
+      logValue += Math.log(bigMarginalProb) - Math.log(smallMarginalProb);
+    }
+
+    double localValue = logValue / log_2;
+
+    if (jointProb > 0.0) {
+      checkLocals(localValue);
+    }
+
+    return localValue;
+  }
+
+}
+

java/source/infodynamics/measures/discrete/SInfoCalculatorDiscrete.java

@@ -0,0 +1,95 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.MathsUtils;
+import infodynamics.utils.MatrixUtils;
+
+/**
+ * <p>Computes the S-information of a given multivariate
+ *  <code>int[][]</code> set of
+ *  observations (extending {@link MultiVariateInfoMeasureCalculatorDiscrete}).</p>
+ *
+ * <p>Usage is as per the paradigm outlined for {@link MultiVariateInfoMeasureCalculatorDiscrete}.
+ * </p>
+ * 
+ * <p><b>References:</b><br/>
+ * <ul>
+ *  <li>Rosas, F., Mediano, P., Gastpar, M, Jensen, H.,
+ *   <a href="http://dx.doi.org/10.1103/PhysRevE.100.032305">"Quantifying high-order
+ *   interdependencies via multivariate extensions of the mutual information"</a>,
+ *   Physical Review E 100, (2019) 032305.</li>
+ * </ul>
+ *
+ * @author Pedro A.M. Mediano (<a href="pmediano at pm.me">email</a>,
+ * <a href="http://www.doc.ic.ac.uk/~pam213">www</a>)
+ */
+public class SInfoCalculatorDiscrete
+    extends MultiVariateInfoMeasureCalculatorDiscrete {
+  
+  /**
+   * Construct an instance.
+   * 
+   * @param base number of symbols for each variable.
+   *        E.g. binary variables are in base-2.
+   * @param numVars numbers of joint variables that DTC
+   *     will be computed over.
+   */
+  public SInfoCalculatorDiscrete(int base, int numVars) {
+    super(base, numVars);
+  }
+
+  protected double computeLocalValueForTuple(int[] tuple, int jointValue) {
+
+    if (jointCount[jointValue] == 0) {
+      // This joint state does not occur, so it makes no contribution here
+      return 0;
+    }
+
+    double jointProb = (double) jointCount[jointValue] / (double) observations;
+
+    // Local TC value
+    double localTC = Math.log(jointProb);
+    for (int i = 0; i < numVars; i++) {
+      int marginalState = tuple[i];
+      double marginalProb = (double) smallMarginalCounts[i][marginalState] / (double) observations;
+      localTC -= Math.log(marginalProb);
+    }
+
+    // Local DTC value
+    double localDTC = (numVars - 1) * Math.log(jointProb);
+    for (int i = 0; i < numVars; i++) {
+      int marginalState = computeBigMarginalState(jointValue, i, tuple[i]);
+      double marginalProb = (double) bigMarginalCounts[i][marginalState] / (double) observations;
+      localDTC -= Math.log(marginalProb);
+    }
+
+    // Combine local TC and DTC into S-info
+    double logValue = localTC + localDTC;
+    double localValue = logValue / log_2;
+
+    if (jointProb > 0.0) {
+      checkLocals(localValue);
+    }
+
+    return localValue;
+  }
+
+}
+

java/unittests/infodynamics/measures/discrete/DualTotalCorrelationTester.java

@@ -0,0 +1,108 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.RandomGenerator;
+import infodynamics.utils.MatrixUtils;
+import infodynamics.utils.MathsUtils;
+
+import java.util.Arrays;
+import junit.framework.TestCase;
+
+
+public class DualTotalCorrelationTester extends TestCase {
+
+  public void testIndependent() throws Exception {
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 3);
+    double dtc = dtcCalc.compute(new int[][] {{0,0,0},{0,0,1},{0,1,0},{0,1,1},{1,0,0},{1,0,1},{1,1,0},{1,1,1}});
+    assertEquals(0.0, dtc, 0.000001);
+  }
+
+  public void testXor() throws Exception {
+    // 3 variables
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 3);
+    double dtc = dtcCalc.compute(new int[][] {{0,0,1},{0,1,0},{1,0,0},{1,1,1}});
+    assertEquals(2.0, dtc, 0.000001);
+
+    // 4 variables
+    dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 4);
+    dtc = dtcCalc.compute(new int[][] {{0,0,0,1},{0,0,1,0},{0,1,0,0},{1,0,0,0},{1,1,1,0},{1,1,0,1},{1,0,1,1},{0,1,1,1}});
+    assertEquals(3.0, dtc, 0.000001);
+  }
+
+  public void testCopy() throws Exception {
+    // 3 variables
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 3);
+    double dtc = dtcCalc.compute(new int[][] {{0,0,0},{1,1,1}});
+    assertEquals(1.0, dtc, 0.000001);
+
+    // 4 variables
+    dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 4);
+    dtc = dtcCalc.compute(new int[][] {{0,0,0,0},{1,1,1,1}});
+    assertEquals(1.0, dtc, 0.000001);
+  }
+
+  public void testCompareEntropy() throws Exception {
+    // Generate random data and check that it matches the explicit computation
+    // using entropy calculators
+    RandomGenerator rg = new RandomGenerator();
+    int D = 4;
+    int[][] data = rg.generateRandomInts(10, D, 2);
+
+    // DTC calculator
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, D);
+    double dtc_direct = dtcCalc.compute(data);
+
+    // Entropy calculators
+    EntropyCalculatorDiscrete hCalc = new EntropyCalculatorDiscrete(MathsUtils.power(2, D));
+    hCalc.initialise();
+    hCalc.addObservations(MatrixUtils.computeCombinedValues(data, 2));
+    double dtc_test = (1 - D) * hCalc.computeAverageLocalOfObservations();
+
+    hCalc = new EntropyCalculatorDiscrete(MathsUtils.power(2, D-1));
+    for (int i = 0; i < 4; i++) {
+      hCalc.initialise();
+      hCalc.addObservations(MatrixUtils.computeCombinedValues(MatrixUtils.selectColumns(data, allExcept(i, D)), 2));
+      dtc_test += hCalc.computeAverageLocalOfObservations();
+    }
+
+    assertEquals(dtc_direct, dtc_test, 0.000001);
+
+  }
+
+  protected int[] allExcept(int idx, int N) {
+    boolean[] v = new boolean[N];
+    Arrays.fill(v, true);
+    v[idx] = false;
+
+    int[] v2 = new int[N - 1];
+    int counter = 0;
+    for (int i = 0; i < N; i++) {
+      if (v[i]) {
+        v2[counter] = i;
+        counter++;
+      }
+    }
+
+    return v2;
+  }
+
+
+}
+

java/unittests/infodynamics/measures/discrete/OInfoTester.java

@@ -0,0 +1,87 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.RandomGenerator;
+
+import junit.framework.TestCase;
+
+public class OInfoTester extends TestCase {
+
+  public void testIndependent() throws Exception {
+    OInfoCalculatorDiscrete oCalc = new OInfoCalculatorDiscrete(2, 3);
+    double oinfo = oCalc.compute(new int[][] {{0,0,0},{0,0,1},{0,1,0},{0,1,1},{1,0,0},{1,0,1},{1,1,0},{1,1,1}});
+    assertEquals(0.0, oinfo, 0.000001);
+  }
+
+  public void testXor() throws Exception {
+    // 3 variables
+    OInfoCalculatorDiscrete oCalc = new OInfoCalculatorDiscrete(2, 3);
+    double oinfo = oCalc.compute(new int[][] {{0,0,1},{0,1,0},{1,0,0},{1,1,1}});
+    assertEquals(-1.0, oinfo, 0.000001);
+
+    // 4 variables
+    oCalc = new OInfoCalculatorDiscrete(2, 4);
+    oinfo = oCalc.compute(new int[][] {{0,0,0,1},{0,0,1,0},{0,1,0,0},{1,0,0,0},{1,1,1,0},{1,1,0,1},{1,0,1,1},{0,1,1,1}});
+    assertEquals(-2.0, oinfo, 0.000001);
+  }
+
+  public void testCopy() throws Exception {
+    // 3 variables
+    OInfoCalculatorDiscrete oCalc = new OInfoCalculatorDiscrete(2, 3);
+    double oinfo = oCalc.compute(new int[][] {{0,0,0},{1,1,1}});
+    assertEquals(1.0, oinfo, 0.000001);
+
+    // 4 variables
+    oCalc = new OInfoCalculatorDiscrete(2, 4);
+    oinfo = oCalc.compute(new int[][] {{0,0,0,0},{1,1,1,1}});
+    assertEquals(2.0, oinfo, 0.000001);
+  }
+
+  public void testPairwise() throws Exception {
+    // Variables 0 and 1 are correlated and independent from 2 and 3, that are
+    // also correlated
+    OInfoCalculatorDiscrete oCalc = new OInfoCalculatorDiscrete(2, 4);
+    double oinfo = oCalc.compute(new int[][] {{0,0,0,0},{0,0,1,1},{1,1,0,0},{1,1,1,1}});
+    assertEquals(0.0, oinfo, 0.000001);
+  }
+
+  public void testCompareTCAndDTC() throws Exception {
+    // Generate random data and check that it matches the explicit computation
+    // using TC and DTC calculators
+    RandomGenerator rg = new RandomGenerator();
+    int[][] data = rg.generateRandomInts(10, 4, 2);
+
+    // O-info calculator
+    OInfoCalculatorDiscrete oCalc = new OInfoCalculatorDiscrete(2, 4);
+    double oinfo_direct = oCalc.compute(data);
+
+    // TC and DTC calculators
+    MultiInformationCalculatorDiscrete tcCalc = new MultiInformationCalculatorDiscrete(2, 4);
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 4);
+    tcCalc.initialise();
+    tcCalc.addObservations(data);
+    double oinfo_test = tcCalc.computeAverageLocalOfObservations() - dtcCalc.compute(data);
+
+    assertEquals(oinfo_direct, oinfo_test, 0.000001);
+
+  }
+
+}
+

java/unittests/infodynamics/measures/discrete/SInfoTester.java

@@ -0,0 +1,79 @@
+/*
+ *  Java Information Dynamics Toolkit (JIDT)
+ *  Copyright (C) 2012, Joseph T. Lizier
+ *  
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *  
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *  
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package infodynamics.measures.discrete;
+
+import infodynamics.utils.RandomGenerator;
+
+import junit.framework.TestCase;
+
+public class SInfoTester extends TestCase {
+
+  public void testIndependent() throws Exception {
+    SInfoCalculatorDiscrete sCalc = new SInfoCalculatorDiscrete(2, 3);
+    double sinfo = sCalc.compute(new int[][] {{0,0,0},{0,0,1},{0,1,0},{0,1,1},{1,0,0},{1,0,1},{1,1,0},{1,1,1}});
+    assertEquals(0.0, sinfo, 0.000001);
+  }
+
+  public void testXor() throws Exception {
+    // 3 variables
+    SInfoCalculatorDiscrete sCalc = new SInfoCalculatorDiscrete(2, 3);
+    double sinfo = sCalc.compute(new int[][] {{0,0,1},{0,1,0},{1,0,0},{1,1,1}});
+    assertEquals(3.0, sinfo, 0.000001);
+
+    // 4 variables
+    sCalc = new SInfoCalculatorDiscrete(2, 4);
+    sinfo = sCalc.compute(new int[][] {{0,0,0,1},{0,0,1,0},{0,1,0,0},{1,0,0,0},{1,1,1,0},{1,1,0,1},{1,0,1,1},{0,1,1,1}});
+    assertEquals(4.0, sinfo, 0.000001);
+  }
+
+  public void testCopy() throws Exception {
+    // 3 variables
+    SInfoCalculatorDiscrete sCalc = new SInfoCalculatorDiscrete(2, 3);
+    double sinfo = sCalc.compute(new int[][] {{0,0,0},{1,1,1}});
+    assertEquals(3.0, sinfo, 0.000001);
+
+    // 4 variables
+    sCalc = new SInfoCalculatorDiscrete(2, 4);
+    sinfo = sCalc.compute(new int[][] {{0,0,0,0},{1,1,1,1}});
+    assertEquals(4.0, sinfo, 0.000001);
+  }
+
+  public void testCompareTCAndDTC() throws Exception {
+    // Generate random data and check that it matches the explicit computation
+    // using TC and DTC calculators
+    RandomGenerator rg = new RandomGenerator();
+    int[][] data = rg.generateRandomInts(10, 4, 2);
+
+    // O-info calculator
+    SInfoCalculatorDiscrete sCalc = new SInfoCalculatorDiscrete(2, 4);
+    double sinfo_direct = sCalc.compute(data);
+
+    // TC and DTC calculators
+    MultiInformationCalculatorDiscrete tcCalc = new MultiInformationCalculatorDiscrete(2, 4);
+    DualTotalCorrelationCalculatorDiscrete dtcCalc = new DualTotalCorrelationCalculatorDiscrete(2, 4);
+    tcCalc.initialise();
+    tcCalc.addObservations(data);
+    double sinfo_test = tcCalc.computeAverageLocalOfObservations() + dtcCalc.compute(data);
+
+    assertEquals(sinfo_direct, sinfo_test, 0.000001);
+
+  }
+
+}
+