From 52a1bb94b65ab57726034e9a64695e7004253cde Mon Sep 17 00:00:00 2001 From: Joseph Lizier Date: Wed, 21 Feb 2024 16:58:30 +1100 Subject: [PATCH] Adding text analysis exercise in Python and Matlab. Need to adjust to download and initially parse the text --- .../conditionalMIAsFunctionOfLag.ipynb | 147 ++++++++ .../conditionalMIAsFunctionOfLag.m | 29 ++ ...onditionalMIAsFunctionOfLag_Solution.ipynb | 229 +++++++++++ .../TextAnalysis/entropyOfCharacters.ipynb | 196 ++++++++++ .../TextAnalysis/entropyOfCharacters.m | 34 ++ .../entropyOfCharacters_Solution.ipynb | 280 ++++++++++++++ .../TextAnalysis/miAsFunctionOfLag.m | 17 + .../miAsFunctionOfLagAndPointwise.ipynb | 260 +++++++++++++ ...AsFunctionOfLagAndPointwise_Solution.ipynb | 355 ++++++++++++++++++ .../TextAnalysis/pointwiseMIofText.m | 37 ++ 10 files changed, 1584 insertions(+) create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.m create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag_Solution.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.m create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters_Solution.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLag.m create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise_Solution.ipynb create mode 100644 course/Module02-JointAndConditionalEntropy/TextAnalysis/pointwiseMIofText.m diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.ipynb new file mode 100644 index 0000000..83ce561 --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.ipynb @@ -0,0 +1,147 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Conditional mutual information between successive letters in written English\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` etc. we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 7. (Optional Extension) Conditional mutual information between successive letters in written English\n", + "\n", + "In this extension activity, we will continue our analysis of written English extracted from the [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld) scripts as begun in the previous modules.\n", + "\n", + "1. Download the scripts from the links on Module 2 on Canvas, load into Python and preprocess as per steps 1-4 of the activity from module 2, such that we have the characters stored in the numpy array `processedStr`:
\n", + "_Note:_ you may need to alter the filename/path to match your own --" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [], + "source": [ + "filename = './Seinfeld-scripts-textOnly.txt'\n", + "with open(filename, 'rt') as f:\n", + " str = f.read()\n", + "p = re.compile('[!\"#\\$%&\\'\\(\\)\\*\\+\\,-\\.\\/:;<=>\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "2. We previously computed the mutual information between characters over several lags between these characters. In particular, we examined the mutual information between characters separated by a lag of 2, and posed the question of whether the information carried by the earlier character about the later one is also included in the character in between them. Think about how you could investigate this question using conditional mutual information?\n", + "3. Calculate the mutual information between characters separated by another character, conditioned on the character in the middle. Use our function `simpleinfotheory.conditionalmutualinformationempirical()`.
\n", + " _Hint_: You should create vectors of samples for each of: the earlier character, the middle character, and the later character; and pass these through to the script. Recall that to select all but the last two items in a numpy array `x`, you can refer to `x[:-2]`, and similarly to select all but the first and last items in an array you can refer to `x[1:-1]`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1225213-86e2-49f3-9ff9-dfe0f6d34876", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the mutual information between successive characters conditioned on the character in between:\n" + ] + }, + { + "cell_type": "markdown", + "id": "ab60c5ec-592a-4e15-97c7-280f7ce91253", + "metadata": {}, + "source": [ + "4. Compare this conditional mutual information to the mutual information between the two characters separated by another character as computed in the previous module (see the result for a lag of 2 on the sample plot in that activity). Is the conditional mutual information here larger or smaller than that? What does this tell us about the structure of the relationships in sequences of characters in English text?\n", + "5. _Challenge_: Can you compute such conditional mutual information over lags of up to 5 characters, conditioning on all intervening characters, and then plot these? Alternatively, you could compute the joint mutual information from sets of consecutive characters (up to 5 of them) to the next character. (How are these two quantities related?).
\n", + " Note that the calculations here will take _significantly_ longer than previous ones since we are dealing with higher and higher order multivariate spaces (not so long for a lag of 2, but ~1 minute for lag 5). What is the size of the probability space we are estimating once we are looking at a lag of 5 (i.e. 4 characters in between the previous and next)? Do you think we can properly estimate the joint probabilities here from the amount of data that we have?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9a980f0-ad8a-4aea-9fbe-5d0a78282db8", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the mutual information between successive characters conditioned on up to 5 characters in between:\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.m b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.m new file mode 100644 index 0000000..af08a0c --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag.m @@ -0,0 +1,29 @@ +% assumes processedStr holds the text as previously processed - you can run +% the previous solution code entropyOfCharacters.m to pull this up + +% Compute conditional MI as a function of lag: + +maxLag = 5; + +condMisVsLag = zeros(1, maxLag); +conditionalCharacters = []; + +nextChar = processedStr(maxLag+1:end); +for lag=1:maxLag + sourceChar = processedStr(maxLag+1-lag:end-lag); + if (lag == 1) + % Nothing to condition on, just compute MI + condMisVsLag(lag) = mutualinformationempirical(sourceChar, nextChar); + else + condMisVsLag(lag) = conditionalmutualinformationempirical(sourceChar, nextChar, conditionalCharacters); + end + fprintf('cond MI over lag %d is %.4f\n', lag, condMisVsLag(lag)); + conditionalCharacters = [conditionalCharacters, sourceChar']; +end + +figure(); +plot(1:maxLag, condMisVsLag, 'rx'); +xlabel('Lag') +ylabel('MI (bits)'); +title('Average MI between characters separated by the given lag conditioned on intervening chars'); + diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag_Solution.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag_Solution.ipynb new file mode 100644 index 0000000..537c54d --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/conditionalMIAsFunctionOfLag_Solution.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Conditional mutual information between successive letters in written English\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` etc. we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 7. (Optional Extension) Conditional mutual information between successive letters in written English\n", + "\n", + "In this extension activity, we will continue our analysis of written English extracted from the [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld) scripts as begun in the previous modules.\n", + "\n", + "1. Download the scripts from the links on Module 2 on Canvas, load into Python and preprocess as per steps 1-4 of the activity from module 2, such that we have the characters stored in the numpy array `processedStr`:
\n", + "_Note:_ you may need to alter the filename/path to match your own --" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([' ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',\n", + " 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',\n", + " 'z'], dtype='\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "2. We previously computed the mutual information between characters over several lags between these characters. In particular, we examined the mutual information between characters separated by a lag of 2, and posed the question of whether the information carried by the earlier character about the later one is also included in the character in between them. Think about how you could investigate this question using conditional mutual information?\n", + "3. Calculate the mutual information between characters separated by another character, conditioned on the character in the middle. Use our function `simpleinfotheory.conditionalmutualinformationempirical()`.
\n", + " _Hint_: You should create vectors of samples for each of: the earlier character, the middle character, and the later character; and pass these through to the script. Recall that to select all but the last two items in a numpy array `x`, you can refer to `x[:-2]`, and similarly to select all but the first and last items in an array you can refer to `x[1:-1]`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f1225213-86e2-49f3-9ff9-dfe0f6d34876", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.739076942492698" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Compute the mutual information between successive characters conditioned on the character in between:\n", + "simpleinfotheory.conditionalmutualinformationempirical(processedStr[:-2],processedStr[2:],processedStr[1:-1])" + ] + }, + { + "cell_type": "markdown", + "id": "ab60c5ec-592a-4e15-97c7-280f7ce91253", + "metadata": {}, + "source": [ + "4. Compare this conditional mutual information to the mutual information between the two characters separated by another character as computed in the previous module (see the result for a lag of 2 on the sample plot in that activity). Is the conditional mutual information here larger or smaller than that? What does this tell us about the structure of the relationships in sequences of characters in English text?\n", + "5. _Challenge_: Can you compute such conditional mutual information over lags of up to 5 characters, conditioning on all intervening characters, and then plot these? Alternatively, you could compute the joint mutual information from sets of consecutive characters (up to 5 of them) to the next character. (How are these two quantities related?).
\n", + " Note that the calculations here will take _significantly_ longer than previous ones since we are dealing with higher and higher order multivariate spaces (not so long for a lag of 2, but ~1 minute for lag 5). What is the size of the probability space we are estimating once we are looking at a lag of 5 (i.e. 4 characters in between the previous and next)? Do you think we can properly estimate the joint probabilities here from the amount of data that we have?" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c9a980f0-ad8a-4aea-9fbe-5d0a78282db8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cond MI over lag 1 is 0.7196\n", + "\n", + "cond MI over lag 2 is 0.7391\n", + "\n", + "cond MI over lag 3 is 0.5251\n", + "\n", + "cond MI over lag 4 is 0.3197\n", + "\n", + "cond MI over lag 5 is 0.2474\n", + "\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHWCAYAAACL2KgUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABXE0lEQVR4nO3deXhMZ/8/8Pdk30eCBElkUY01SlSEWkMI0UdbtbWx94uWltCWelpLl1Dkoa1QrZ1Wai2KipJYgtqq9lJbVCLWJLaQ5PP7I7+ZmkyWmUhM5ni/rmuuS+65zzmfM/csb2dViYiAiIiISCEsTF0AERERUWliuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4MSNfffUVVCoV6tWrZ+pSyp3WrVtDpVLB398fBV10e8eOHVCpVFCpVFi4cKG2feHChVCpVDhw4ECR8ze0nzGSkpIwYcIE3L59u9TmaY58fX0RERFh6jJKJDY2Vuf9RE/HvXv3MGHCBCQkJJT6vBMSEqBSqYqd94QJE6BSqXD9+vVSr+FxP/zwA2bMmKHXfuHCBahUKkybNq1Ml2+uGG7MyPz58wEAx48fx759+0xcTfnj7OyM8+fPY9u2bXrPzZ8/Hy4uLiaoqnBJSUmYOHHiMx9uzBnDjWncu3cPEydOLJNwU94UFm6oaAw3ZuLAgQM4cuQIOnfuDACYN2/eU69BRHD//v2nvlxDVa9eHU2bNtWGQI3MzEysWLECPXr0MFFlZGo5OTnIysoydRkGKe+fs7Lw6NEjZGdnm7oMUhCGGzOhCTOTJ09Gs2bNsHz5cty7dw9A3heDu7s7IiMj9aa7ffs27O3tERUVpW3LyMjA6NGj4efnBxsbG3h6emLEiBG4e/euzrQqlQrDhg3DnDlzULt2bdja2mLRokUAgIkTJyI4OBhubm5wcXFBo0aNMG/ePL1dQllZWRg1ahSqVKkCBwcHtGzZEgcPHoSvry/69eun0zc1NRWDBw+Gl5cXbGxs4Ofnh4kTJxr1pTdgwACsXr1aZ2vI8uXLAQA9e/Y0eD6FuXXrFvr37w83Nzc4OjqiS5cuOHfunF6/rVu3IjQ0FC4uLnBwcEDz5s3x22+/aZ+fMGEC3n//fQCAn5+fdpdZQkIC3n//fajVauTk5Gj7Dx8+HCqVClOnTtW23bhxAxYWFvj666+1bYaOrYggNjYWL7zwAuzt7eHq6opu3brprUvr1q1Rr1497N+/Hy1atICDgwP8/f0xefJk5ObmFvt65ebm4uuvv9Yup0KFCmjatCnWrVun13fz5s1o1KgR7O3tUatWLb2Qeu3aNbz99tuoU6cOnJyc4O7ujrZt22Lnzp06/TSb67/88kt89tln8PPzg62tLbZv344HDx5g1KhReOGFF6BWq+Hm5oaQkBD8/PPPRtfu6+uL48ePIzExUTt+vr6+Ro9FUZ+z2bNno0GDBnBycoKzszNq1aqFjz76qNjX3ZDpDPm8Pf5afv7556hevTrs7OzQuHFjnfczAJw9exb9+/dHzZo14eDgAE9PT3Tp0gVHjx7V6afZ7bNkyRKMGjUKnp6esLW1xdmzZw0a4wsXLqBy5coA8r6HNK/9498nZ86cQe/eveHu7g5bW1vUrl0bs2bN0nudTp06hY4dO8LBwQGVKlXCkCFDkJmZWezr+7jk5GS8+uqrcHFxgVqtxptvvolr165pnx84cCDc3Ny039ePa9u2LerWrVvovFu3bo1ffvkFFy9e1K6nSqXS6xcTEwM/Pz84OTkhJCQEe/fu1etz4MABvPzyy3Bzc4OdnR0aNmyIn376yah1NStC5d69e/dErVbLiy++KCIi33//vQCQhQsXavuMHDlS7O3tJT09XWfa2NhYASB//vmniIjcvXtXXnjhBalUqZLExMTI1q1bZebMmaJWq6Vt27aSm5urnRaAeHp6SmBgoPzwww+ybds2OXbsmIiI9OvXT+bNmyfx8fESHx8vn376qdjb28vEiRN1lt+rVy+xsLCQMWPGyJYtW2TGjBni7e0tarVa+vbtq+2XkpIi3t7e4uPjI99++61s3bpVPv30U7G1tZV+/foV+xq1atVK6tatKxkZGeLo6CixsbHa54KDg6VPnz6yf/9+ASALFizQPrdgwQIBIPv37y9y/pp+3t7eMmDAANm0aZPMnTtX3N3dxdvbW27duqXtu2TJElGpVNK1a1dZvXq1rF+/XiIiIsTS0lK2bt0qIiLJyckyfPhwASCrV6+WPXv2yJ49eyQ9PV02b94sACQpKUk7z1q1aom9vb20b99e2xYXFycA5MSJEyJi3Ni+9dZbYm1tLaNGjZLNmzfLDz/8ILVq1RIPDw9JTU3VeV0rVqwoNWvWlDlz5kh8fLy8/fbbAkAWLVpU7LhERkaKSqWSQYMGyc8//yybNm2Szz//XGbOnKnt4+PjI15eXlKnTh1ZvHix/Prrr/L6668LAElMTNT2O3XqlAwdOlSWL18uCQkJsmHDBhk4cKBYWFjI9u3btf3Onz+vfe+2adNGVq5cKVu2bJHz58/L7du3pV+/frJkyRLZtm2bbN68WUaPHi0WFhZ661Nc7YcOHRJ/f39p2LChdvwOHTpk9FgU9jn78ccfBYAMHz5ctmzZIlu3bpU5c+bIu+++W+Rrbsh0hn7eNK+lt7e3vPTSS7Jq1SpZsWKFvPjii2Jtba3zHk1MTJRRo0bJypUrJTExUdasWSNdu3YVe3t7OXXqlLbf9u3btevcrVs3WbdunWzYsEFu3Lhh0Bg/ePBA+xkZOHCg9rU/e/asiIgcP35c1Gq11K9fXxYvXixbtmyRUaNGiYWFhUyYMEFbR2pqqri7u4unp6csWLBANm7cKG+88YZUr15dAOi8pwoyfvx4ASA+Pj7y/vvvy6+//ioxMTHi6OgoDRs2lIcPH4qIyJEjRwSAfPfddzrTHz9+XADIrFmzCl3G8ePHpXnz5lKlShXteu7Zs0dnbHx9faVjx46ydu1aWbt2rdSvX19cXV3l9u3b2vls27ZNbGxspEWLFhIXFyebN2+Wfv366X0fKgnDjRlYvHixAJA5c+aIiEhmZqY4OTlJixYttH3+/PNPASBz587VmbZJkyYSFBSk/Ts6OlosLCz0fsxXrlwpAGTjxo3aNgCiVqvl5s2bRdaXk5Mjjx49kkmTJknFihW1X9yaD++HH36o01/z5ft4uBk8eLA4OTnJxYsXdfpOmzZNAMjx48eLrEETbkRE+vbtK40bN9apISEhoVTCzSuvvKLTvnv3bgEgn332mYjk/ai5ublJly5d9F6jBg0aSJMmTbRtU6dOFQBy/vx5nb53794VGxsbmTRpkoiIXL58Wfs62tvby4MHD0QkL6BUq1ZNO52hY7tnzx4BINOnT9fpl5ycLPb29vLBBx9o21q1aiUAZN++fTp969SpIx06dCjyNduxY4cAkHHjxhXZz8fHR+zs7HTG/v79++Lm5iaDBw8udLrs7Gx59OiRhIaG6oyL5ku/Ro0a2h+Y4uYxcOBAadiwodG1161bV1q1aqXXXhqfs2HDhkmFChWKXH5BDJnO0M+b5rWsVq2a3L9/X9svIyND3NzcpF27doUuIzs7Wx4+fCg1a9aUkSNHats14aZly5bFrkthY3zt2jUBIOPHj9ebpkOHDuLl5aX3H71hw4aJnZ2d9nX+8MMPRaVSyR9//KHTr3379kaFm8fXTURk2bJlAkCWLl2qbWvVqpW88MILOv2GDh0qLi4ukpmZWeRyOnfuLD4+PnrtmrGpX7++ZGdna9t///13ASA//vijtq1WrVrSsGFDefTokc48IiIipGrVqpKTk1NkDeaIu6XMwLx582Bvb6/dreLk5ITXX38dO3fuxJkzZwAA9evXR1BQEBYsWKCd7uTJk/j9998xYMAAbduGDRtQr149vPDCC8jOztY+OnToUOAZAm3btoWrq6teTdu2bUO7du2gVqthaWkJa2trfPLJJ7hx4wbS0tIAAImJiQCA7t2760zbrVs3WFlZ6bRt2LABbdq0QbVq1XTqCg8P15mXIQYMGIADBw7g6NGjmDdvHmrUqIGWLVsaPH1R3njjDZ2/mzVrBh8fH2zfvh1A3kHCN2/eRN++fXXWIzc3Fx07dsT+/fv1dkvk5+DggJCQEGzduhUAEB8fjwoVKuD999/Hw4cPsWvXLgB5u77atWunnc7Qsd2wYQNUKhXefPNNnX5VqlRBgwYN9N4DVapUQZMmTXTaAgMDcfHixSLXY9OmTQCAd955p8h+APDCCy+gevXq2r/t7Ozw/PPP6y1jzpw5aNSoEezs7GBlZQVra2v89ttvOHnypN48X375ZVhbW+u1r1ixAs2bN4eTk5N2HvPmzdOZhzG1F6Q0PmdNmjTB7du30atXL/z8888Gn5VjyHTGft5effVV2NnZaf92dnZGly5dsGPHDu3u0+zsbHzxxReoU6cObGxsYGVlBRsbG5w5c6bA8XnttdcKrN+YMc7vwYMH+O233/DKK6/AwcFBZ906deqEBw8eaHfZbN++HXXr1kWDBg105tG7d+9il/O4/N8J3bt3h5WVlfY7AQDee+89/PHHH9i9ezeAvF2WS5YsQd++feHk5GTU8vLr3LkzLC0ttX8HBgYCgPazc/bsWZw6dUpbZ/7XJCUlBadPn36iGsojhpty7uzZs9ixYwc6d+4MEcHt27dx+/ZtdOvWDQB0jksYMGAA9uzZg1OnTgEAFixYAFtbW/Tq1Uvb5+rVq/jzzz9hbW2t83B2doaI6H0RVq1aVa+m33//HWFhYQCA7777Drt378b+/fsxbtw4ANAeDHnjxg0AgIeHh870VlZWqFixok7b1atXsX79er26NPujjTndsmXLlqhZsya+/fZbLFmyBAMGDChwP3VJVKlSpcA2zbpevXoVQF6Ay78uU6ZMgYjg5s2bxS6nXbt22Lt3L+7evYutW7eibdu2qFixIoKCgrB161acP38e58+f1wk3ho7t1atXISLw8PDQ67t371691zr/WAGAra1tsQe9Xrt2DZaWlgW+ZvkZsoyYmBgMHToUwcHBWLVqFfbu3Yv9+/ejY8eOBdZS0Ht39erV6N69Ozw9PbF06VLs2bMH+/fvx4ABA/DgwYMS1V6Q0vicRUZGYv78+bh48SJee+01uLu7Izg4GPHx8UUu25DpjP28Ffa+f/jwIe7cuQMAiIqKwscff4yuXbti/fr12LdvH/bv348GDRoYPD7GjnF+N27cQHZ2Nr7++mu9devUqZPOut24caPQ9TJG/v6a7zfNdwIA/Oc//4Gvr6/2uJ+FCxfi7t27JQ7Pj8v/2bG1tQXw7/ew5jtp9OjReq/J22+/DcC471dzYVV8FzKl+fPnQ0SwcuVKrFy5Uu/5RYsW4bPPPoOlpSV69eqFqKgoLFy4EJ9//jmWLFmCrl276vyPsFKlSrC3t9c7WPPx5x9XUChYvnw5rK2tsWHDBp3/za1du1ann+ZDd/XqVXh6emrbs7OzdT74muUGBgbi888/L7CuatWqFdhemP79++O///0vVCoV+vbta9S0RUlNTS2w7bnnngPw7+v39ddfo2nTpgXOI3/YK0hoaCg+/vhj7NixA7/99hvGjx+vbd+yZQv8/Py0f2sYOraVKlWCSqXCzp07tV+EjyuorSQqV66MnJwcpKamFvhDZqylS5eidevWmD17tk57YQeAFvTeXbp0Kfz8/BAXF6fzfP4zqZ609tL4nAF57+P+/fvj7t272LFjB8aPH4+IiAj89ddf8PHxKXT5xU1n7OetsPe9jY2NdsvD0qVL0adPH3zxxRc6/a5fv44KFSroTV/Y+Bgzxvm5urrC0tISkZGRhQYHzWenYsWKha6XMVJTUwv8fns8dFhYWOCdd97BRx99hOnTpyM2NhahoaEICAgwalkloXmvjR07Fq+++mqBfZ5GHU8bw005lpOTg0WLFqFGjRr4/vvv9Z7fsGEDpk+fjk2bNiEiIgKurq7o2rUrFi9ejJCQEKSmpurskgKAiIgIfPHFF6hYsaL2Q24slUoFKysrnU2h9+/fx5IlS3T6aXYFxcXFoVGjRtr2lStX6p0BFRERgY0bN6JGjRoF7gYzVt++fbFv3z7Url1b54vnSS1btkxnc3pSUhIuXryIQYMGAQCaN2+OChUq4MSJExg2bFiR88r/P6zHNWnSBC4uLpgxYwZSU1PRvn17AHlbdKZMmYKffvoJderU0fkRMnRsIyIiMHnyZPzzzz96uwxLU3h4OKKjozF79mxMmjTpieenUqn0gteff/6JPXv2wNvb2+B52NjY6Pywpqam6p0tZWjthW3BKo3P2eMcHR0RHh6Ohw8fomvXrjh+/HiR4aa46Yz9vK1evRpTp07V/mcmMzMT69evR4sWLbTfAwWNzy+//IJ//vlHG/6LY+gYF/bZcXBwQJs2bXD48GEEBgbCxsam0GW1adMGX375JY4cOaKza+qHH34wqFaNZcuWISgoSPv3Tz/9hOzsbLRu3Vqn36BBgzBhwgS88cYbOH36NKZMmWLQ/A3ZSlqUgIAA1KxZE0eOHNELnkrGcFOObdq0CVeuXMGUKVP0PigAUK9ePXzzzTeYN2+e9gqvAwYMQFxcHIYNGwYvLy+d3RYAMGLECKxatQotW7bEyJEjERgYiNzcXFy6dAlbtmzBqFGjEBwcXGRdnTt3RkxMDHr37o3/+7//w40bNzBt2jS9L6W6deuiV69emD59OiwtLdG2bVscP34c06dPh1qthoXFv3tFJ02ahPj4eDRr1gzvvvsuAgIC8ODBA1y4cAEbN27EnDlz4OXlZfBrV61aNb0tSaXhwIEDGDRoEF5//XUkJydj3Lhx8PT01G7edXJywtdff42+ffvi5s2b6NatG9zd3XHt2jUcOXIE165d0/6vtH79+gCAmTNnom/fvrC2tkZAQACcnZ1haWmJVq1aYf369fDz80ONGjUA5IUnW1tb/Pbbb3j33Xd1ajN0bJs3b47/+7//Q//+/XHgwAG0bNkSjo6OSElJwa5du1C/fn0MHTr0iV+rFi1aIDIyEp999hmuXr2KiIgI2Nra4vDhw3BwcMDw4cONml9ERAQ+/fRTjB8/Hq1atcLp06cxadIk+Pn5GXy5gIiICKxevRpvv/02unXrhuTkZHz66aeoWrWq9vg1Y2qvX78+li9fjri4OPj7+8POzg7169cvlc/ZW2+9BXt7ezRv3hxVq1ZFamoqoqOjoVar8eKLLz7RdMZ+3iwtLdG+fXtERUUhNzcXU6ZMQUZGBiZOnKjz2i5cuBC1atVCYGAgDh48iKlTpxr1uTV0jJ2dneHj44Off/4ZoaGhcHNzQ6VKleDr64uZM2fipZdeQosWLTB06FD4+voiMzMTZ8+exfr167UX+RwxYgTmz5+Pzp0747PPPoOHhweWLVum3a1vqNWrV8PKygrt27fH8ePH8fHHH6NBgwZ6/3GoUKEC+vTpg9mzZ8PHxwddunQxaP7169fH6tWrMXv2bAQFBcHCwgKNGzc2qsZvv/0W4eHh6NChA/r16wdPT0/cvHkTJ0+exKFDh7BixQqj5mcWTHgwMxWja9euYmNjI2lpaYX26dmzp1hZWWlP383JyRFvb+8iz/S4c+eO/Pe//5WAgACxsbHRnjY5cuRIndOAAcg777xT4Dzmz58vAQEBYmtrK/7+/hIdHS3z5s3TO/vnwYMHEhUVJe7u7mJnZydNmzaVPXv2iFqt1jvL4Nq1a/Luu++Kn5+fWFtbi5ubmwQFBcm4cePkzp07Rb5Wj58tVZjSOFtqy5YtEhkZKRUqVBB7e3vp1KmTnDlzRq9/YmKidO7cWdzc3MTa2lo8PT2lc+fOsmLFCp1+Y8eOlWrVqomFhYXeGRozZ84UAPLWW2/pTKM5m2PdunV6yzV0bEXyxjA4OFgcHR3F3t5eatSoIX369JEDBw5o+xT2uvbt27fAMzjyy8nJkf/9739Sr149bT0hISGyfv16bR8fHx/p3Lmz3rStWrXSORMpKytLRo8eLZ6enmJnZyeNGjWStWvX6tWiOYtk6tSpBdY0efJk8fX1FVtbW6ldu7Z899132jNfjK39woULEhYWJs7OztrTgjWe9HO2aNEiadOmjXh4eIiNjY1Uq1ZNunfvrr2sQ2EMnc6Qz5vmtZwyZYpMnDhRvLy8xMbGRho2bCi//vqrzvxu3bolAwcOFHd3d3FwcJCXXnpJdu7cqTeOmrOl8n8WRAwfYxGRrVu3SsOGDcXW1lbv7Mvz58/LgAEDxNPTU6ytraVy5crSrFkz7VmNGidOnJD27duLnZ2duLm5ycCBA+Xnn3826mypgwcPSpcuXcTJyUmcnZ2lV69ecvXq1QKnSUhIEAAyefLkIuf9uJs3b0q3bt2kQoUKolKptO/Tot7nKOBMsiNHjkj37t3F3d1drK2tpUqVKtK2bVvtWbhKoxIp4EY8RGUoKSkJzZs3x7Jly4w+M4GInp4LFy7Az88PU6dOxejRo01djtkbNWoUZs+ejeTk5AIPoqfSw91SVKbi4+OxZ88eBAUFwd7eHkeOHMHkyZNRs2bNQg9uIyJSkr179+Kvv/5CbGwsBg8ezGDzFDDcUJlycXHBli1bMGPGDGRmZqJSpUragzUfP9OKiEipQkJC4ODggIiICHz22WemLueZwN1SREREpCi8iB8REREpCsMNlVhBd/YuTzR3H85/qftnUf6xKui12bhxIyZMmGDQ9OXRwoULoVKpcOHCBVOXAgDo16+fzl3CjZGUlIQJEybo3N1eKVQqVaHvs/JAc5d2Mm885oZKbM2aNXBxcTF1GVQCjRo1wp49e1CnTh1t28aNGzFr1qwCf3g41sb7+OOP8d5775Vo2qSkJEycOBH9+vUr8Oq+5mzPnj1GXfuGqCQYbqjEGjZsaOoSqIRcXFwKvT1EQTjWxtNceLE8uXfvHhwcHExagzHvO6V69OiR9krvVDa4W4r0/PzzzwgMDIStrS38/f0xc+ZMTJgwQe9eMI/vqrh27RpsbGzw8ccf683v1KlTUKlU+Oqrr7RtqampGDx4MLy8vGBjYwM/Pz9MnDhR5yqkFy5cgEqlwrRp0xATEwM/Pz84OTkhJCREe2dfYx04cAA9e/aEr68v7O3t4evri169ehV4h+tdu3YhJCQEdnZ28PT0xMcff4zvv//e4F0f+/btQ5cuXVCxYkXY2dmhRo0aGDFihN4yQkND4ezsDAcHBzRr1gy//PKLTh/N7pbt27dj6NChqFSpEipWrIhXX30VV65c0en76NEjfPDBB6hSpQocHBzw0ksv4ffff9erLf9uqX79+mlv6qdSqbQPzXoWtFvq0qVLePPNN+Hu7g5bW1vUrl0b06dPR25urraPsWN44MABvPzyy3Bzc4OdnR0aNmyIn376Sa/f3r170bx5c9jZ2aFatWoYO3YsHj16VOA4FGTdunXaM1icnZ3Rvn177NmzR6eP5j1//Phx9OrVC2q1Gh4eHhgwYADS09OLXUZBu6U0uzyWLFmC2rVrw8HBAQ0aNMCGDRt0lvv+++8DyLsPkmYsHt+FGBcXh5CQEDg6OsLJyQkdOnTA4cOH9Zbv5OSEo0ePIiwsDM7OzggNDcWIESPg6OiIjIwMvZp79OgBDw8PndfSmGWdPXsWnTp1gpOTE7y9vTFq1Ci9+3bl3y1lzPs7KysLo0aN0r6/W7ZsiYMHDxq82zQrKwuTJk1C7dq1YWdnh4oVK6JNmzZISkrS61vUGAF5NzXu378/atasCQcHB3h6eqJLly44evSoTj/NZ23JkiUYNWoUPD09YWtri7Nnz+LevXsYPXo0/Pz8YGdnBzc3NzRu3Bg//vhjsetCxTDtNQSpvNm0aZNYWFhI69atZc2aNbJixQoJDg4WX19fvSu4+vj46FwV9JVXXhFvb2/JycnR6ffBBx+IjY2NXL9+XUREUlJSxNvbW3x8fOTbb7+VrVu3yqeffiq2trbSr18/7XSaK3D6+vpKx44dZe3atbJ27VqpX7++uLq6yu3bt4tcF82VUB+/0uiKFSvkk08+kTVr1khiYqIsX75cWrVqJZUrV5Zr165p+x05ckTs7OwkMDBQli9fLuvWrZNOnTppX4fHr8JckM2bN4u1tbUEBgbKwoULZdu2bTJ//nzp2bOntk9CQoJYW1tLUFCQxMXFydq1ayUsLExUKpUsX75c209zdWR/f38ZPny4/Prrr/L999+Lq6urtGnTRme5ffv2FZVKJe+//75s2bJFYmJixNPTU1xcXHTGKv9rc/bsWenWrZsAkD179mgfDx48EBH9sU5LSxNPT0+pXLmyzJkzRzZv3izDhg0TADJ06NASjeG2bdvExsZGWrRoIXFxcbJ582bp16+f3lWljx8/Lg4ODlKnTh358ccf5eeff5YOHTpI9erVDRqbZcuWCQAJCwuTtWvXSlxcnAQFBYmNjY3s3LlT209zBdqAgAD55JNPJD4+XmJiYsTW1lb69+9f5DI0Y5H/qrqa16JJkyby008/ycaNG6V169ZiZWUlf//9t4iIJCcny/DhwwWArF69WjsW6enpIiLy+eefi0qlkgEDBsiGDRtk9erVEhISIo6OjnL8+HGd5VtbW4uvr69ER0fLb7/9Jr/++qscOXJEAMh3332nU9utW7fE1tZWoqKitG3GLMvGxkZq164t06ZNk61bt8onn3wiKpVKJk6cqPcaPH71XGPe37169RILCwsZM2aMbNmyRWbMmCHe3t6iVqt13p8FefTokbRp00asrKxk9OjRsnHjRlm3bp189NFH8uOPPxo1RiJ5VyEfNWqUrFy5UhITE2XNmjXStWtXsbe3l1OnTmn7aT5rnp6e0q1bN1m3bp1s2LBBbty4IYMHDxYHBweJiYmR7du3y4YNG2Ty5Mny9ddfF7kuVDyGG9Lx4osvire3t2RlZWnbMjMzpWLFisWGm3Xr1mlvUaCRnZ0t1apVk9dee03bNnjwYHFycpKLFy/qzG/atGkCQPulqflhrF+/vmRnZ2v7/f777wJA5wupIAWFm/yys7Plzp074ujoKDNnztS2v/766+Lo6KgTeHJycqROnToG/YDWqFFDatSoIffv3y+0T9OmTcXd3V0yMzN16qlXr554eXlJbm6uiPz75f/222/rTP/ll18KAElJSRERkZMnTwoAvdtaaH7Miwo3IiLvvPOO3hhr5B/rMWPGCADZt2+fTr+hQ4eKSqWS06dPi4hxY1irVi1p2LChPHr0SGeeERERUrVqVW1o7tGjh9jb2+vcwiA7O1tq1apV7Njk5ORItWrVpH79+johPDMzU9zd3aVZs2baNk24+fLLL3Xm8fbbb4udnZ12fApTWLjx8PCQjIwMbVtqaqpYWFhIdHS0tm3q1KkFrsulS5fEyspKhg8frtOemZkpVapUke7du+ssH4DMnz9fr7ZGjRrprKuISGxsrACQo0ePlnhZP/30k07fTp06SUBAgN5rUFC4Ke79ffz4cQEgH374oU6/H3/8Ue/9XZDFixcXGOryM3SM8svOzpaHDx9KzZo1dT6Dms9ay5Yt9aapV6+edO3atch6qGS4W4q07t69iwMHDqBr1646d9N1cnIy6CZv4eHhqFKlChYsWKBt+/XXX3HlyhWdu5Nv2LABbdq0QbVq1ZCdna19hIeHAwASExN15tu5c2edO5AHBgYCQIG7kopz584dfPjhh3juuedgZWUFKysrODk54e7duzh58qS2X2JiItq2bYtKlSpp2ywsLAy6i/Zff/2Fv//+GwMHDiz0QoV3797Fvn370K1bNzg5OWnbLS0tERkZicuXL+P06dM607z88ss6f+d/HbZv3w4AeOONN3T6de/evdT37W/btg116tRBkyZNdNr79esHEdHenFCjuDE8e/YsTp06pa398fdFp06dkJKSon09tm/fjtDQUHh4eGjnZ2lpiR49ehRb9+nTp3HlyhVERkbq3LjVyckJr732Gvbu3Yt79+7pTFPQ6/7gwQOkpaUVu7yCtGnTBs7Oztq/PTw84O7ubtD7+ddff0V2djb69Omj8xrZ2dmhVatWBZ4Z+Phd7DX69++PpKQknffYggUL8OKLL6JevXolWpZKpdL7nggMDDT4c1rc+1vzvZD/M9itWzeD3t+bNm2CnZ2dzndRYQwZo+zsbHzxxReoU6cObGxsYGVlBRsbG5w5c0bnu0SjoHFo0qQJNm3ahDFjxiAhIeGJ7v5Nung0E2ndunULIqLzo6FRUFt+VlZWiIyMxNdff43bt2+jQoUKWLhwIapWrYoOHTpo+129ehXr16+HtbV1gfO5fv26zt/5L1Wuuft4Sb4Ievfujd9++w0ff/wxXnzxRbi4uEClUqFTp04687tx40aJX4dr164BQJFnhGhe66pVq+o9V61aNW0NjyvuddD0r1Klik4/KyurUr/c+40bNwo8zbmktV+9ehUAMHr06ELvYaR5X9y4cUNvHQH99S6sbgCFvu65ubm4deuWzkG3pfn+K2h+mnkaMj/N61TYXcEfD2wA4ODgUOBZbm+88QZGjx6NhQsXIjo6GidOnMD+/fsRGxv7RMvKH+ZtbW3x4MGDYtYqj6Hv7/yfQUPf39euXUO1atX06jakFk09j49RVFQUZs2ahQ8//BCtWrWCq6srLCwsMGjQoALHsqD33FdffQUvLy/ExcVhypQpsLOzQ4cOHTB16lTUrFmz2DqpcAw3pOXq6gqVSqX9UntcamqqQfPo378/pk6diuXLl6NHjx5Yt24dRowYofO/9kqVKiEwMBCff/55gfPQ/ECWtvT0dGzYsAHjx4/HmDFjtO1ZWVm4efOmTt+KFSuW+HWoXLkyAODy5cuF9tF8EaakpOg9pzmI8vGtRobQfCGnpqbC09NT256dna0XNp5UxYoVS7V2Tf+xY8cWes+xgIAA7bILGgdDxkbzGhVWu4WFBVxdXQ2u+2nTvE4rV66Ej49Psf3znwSg4erqiv/85z9YvHgxPvvsMyxYsAB2dnbo1atXiZdV1jRjd/Xq1RK9vytXroxdu3YhNzfXoIBTnKVLl6JPnz744osvdNqvX79e4On7BY2Fo6MjJk6ciIkTJ+Lq1avarThdunTBqVOnnrjGZxl3S5GWo6MjGjdujLVr1+Lhw4fa9jt37uidKVCY2rVrIzg4GAsWLMAPP/yArKws9O/fX6dPREQEjh07hho1aqBx48Z6j7IKNyqVCiKi/R+hxvfff4+cnBydtlatWmHbtm06W5Fyc3OxYsWKYpfz/PPPo0aNGpg/f77emSIajo6OCA4OxurVq3X+l5ebm4ulS5fCy8sLzz//vDGrh9atWwMAli1bptP+008/6ZyFVhhjtkiEhobixIkTOHTokE774sWLoVKp0KZNGwOrzhMQEICaNWviyJEjBb4nGjdurN1N0KZNG/z222864TMnJwdxcXEGLcfT0xM//PAD5LE7z9y9exerVq3SnkFlaoWNRYcOHWBlZYW///670NfJUP3798eVK1ewceNGLF26FK+88orOj3JpLqs0tGzZEgD0xnnlypUGvb/Dw8Px4MEDLFy4sFTqUalUet8lv/zyC/75558Szc/DwwP9+vVDr169cPr0ab3do2QcbrkhHZMmTULnzp3RoUMHvPfee8jJycHUqVPh5OSkt3WjMAMGDMDgwYNx5coVNGvWTPs/7seXER8fj2bNmuHdd99FQEAAHjx4gAsXLmDjxo2YM2dOmVzky8XFBS1btsTUqVNRqVIl+Pr6IjExEfPmzdP7n9a4ceOwfv16hIaGYty4cbC3t8ecOXNw9+5dAPqb5PObNWsWunTpgqZNm2LkyJGoXr06Ll26hF9//VUbPqKjo9G+fXu0adMGo0ePho2NDWJjY3Hs2DH8+OOPhf6vuzC1a9fGm2++iRkzZsDa2hrt2rXDsWPHMG3aNIMuwFe/fn0AwJQpUxAeHg5LS0sEBgbqHH+lMXLkSCxevBidO3fGpEmT4OPjg19++QWxsbEYOnSo0cEMAL799luEh4ejQ4cO6NevHzw9PXHz5k2cPHkShw4d0gbL//73v1i3bh3atm2LTz75BA4ODpg1a5Z2bIpiYWGBL7/8Em+88QYiIiIwePBgZGVlYerUqbh9+zYmT55sdN1lQTMWM2fORN++fWFtbY2AgAD4+vpi0qRJGDduHM6dO4eOHTvC1dUVV69exe+//67dEmCIsLAweHl54e2330Zqaqref0JKc1mloW7duujVqxemT58OS0tLtG3bFsePH8f06dOhVquL/Uz26tULCxYswJAhQ3D69Gm0adMGubm52LdvH2rXro2ePXsaVU9ERAQWLlyIWrVqITAwEAcPHsTUqVON+u4KDg5GREQEAgMD4erqipMnT2LJkiXlJmSbNZMezkzl0po1a6R+/fpiY2Mj1atXl8mTJ8u7774rrq6uOv3yn0GjkZ6eLvb29kWemXDt2jV59913xc/PT6ytrcXNzU2CgoJk3LhxcufOHRH590ybqVOn6k2PfGdcFKSgM4IuX74sr732mri6uoqzs7N07NhRjh07VuC67Ny5U4KDg8XW1laqVKki77//vkyZMkUAFHsauojInj17JDw8XNRqtdja2kqNGjX0zmTauXOntG3bVhwdHcXe3l6aNm0q69ev1+mjOZtk//79xa5fVlaWjBo1Stzd3cXOzk6aNm0qe/bs0Vu/wqYdNGiQVK5cWVQqlc7ZOgW9PhcvXpTevXtLxYoVxdraWgICAmTq1Kk6ZyEZO4ZHjhyR7t27i7u7u1hbW0uVKlWkbdu2MmfOHJ1+u3fvlqZNm+qMzdy5cw06k01EZO3atRIcHCx2dnbi6OgooaGhsnv3bp0+mrOlHj9jTuTf8ShuOYWdLfXOO+/o9S3o9R07dqxUq1ZNLCws9MZq7dq10qZNG3FxcRFbW1vx8fGRbt26ydatW3WW7+joWGSNH330kQAo8BIOpbEszWuY/zUo6GwpQ97fDx48kKioKL33t1qt1vtsFeT+/fvyySefSM2aNcXGxkYqVqwobdu2laSkJJ36DBmjW7duycCBA8Xd3V0cHBzkpZdekp07d0qrVq2kVatWeuuxYsUKvXmOGTNGGjduLK6urmJrayv+/v4ycuRI7WUzqOR4V3Aq1qNHj/DCCy/A09MTW7ZsMXU5JhUWFoYLFy7gr7/+MnUpRIS8W1U0b94cy5YtQ+/evU1dDpUT3C1FegYOHIj27dujatWqSE1NxZw5c3Dy5EnMnDnT1KU9VVFRUWjYsCG8vb1x8+ZNLFu2DPHx8Zg3b56pSyN6JsXHx2PPnj0ICgqCvb09jhw5gsmTJ6NmzZqFHohOzyaGG9KTmZmJ0aNH49q1a7C2tkajRo2wceNGtGvXztSlPVU5OTn45JNPkJqaCpVKhTp16mDJkiV48803TV0a0TPJxcUFW7ZswYwZM5CZmYlKlSohPDwc0dHRhV5Tip5N3C1FREREisJTwYmIiEhRGG6IiIhIURhuiIiISFGeuQOKc3NzceXKFTg7Oxt9kTQiIiIyDRFBZmamQfcIe+bCzZUrV+Dt7W3qMoiIiKgEkpOTi70S9DMXbjT3p0lOTjbokvRERERkehkZGfD29tb+jhflmQs3ml1RLi4uDDdERERmxpBDSnhAMRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDVF6OnD5csHPXb6c9zwREZkNhht6tqWnAx07Aq1aAcnJus8lJ+e1d+zIgENEZEYYbujZlpkJpKUB584BrVv/G3CSk/P+Pncu7/nMTFNWSURERmC4oWeblxeQkAD4+/8bcJKS/g02/v55z3t5mbZOIiIy2DN3V3AiPd7eeQFGE2iaN89r1wQbb28TFkdERMbilhsiIC/ALFmi27ZkCYMNEZEZYrghAvKOsYmM1G2LjNQ/yJiIiMo9hhuixw8e9vcHdu/WPQaHAYeIyKww3NCz7fJl/YOHmzXTP8i4sOvgEBFRucMDiunZ5uwMuLvn/fvxg4cfP8jY3T2vHxERmQWGG3q2qdXA5s1517HJf7q3tzeQmJgXbNRq09RHRERGY7ghUqsLDy+8vg0RkdnhMTdERESkKAw3T4o3XSQiIipXGG6eBG+6SEREVO4w3DwJ3nSRiIio3GG4eRK86SIREVG5w7OlnhRvukhERFSucMtNaeBNF4mIiMoNhpvSwJsuEhERlRsMN0+KN10kIiIqVxhungRvukhERFTu8IDiJ8GbLhIREZU7DDdPgjddJCIiKncYbp4Ub7pIRERUrvCYGyIiIlIUhhsiIiJSFIYbIiIiUhSTh5vY2Fj4+fnBzs4OQUFB2LlzZ6F9+/XrB5VKpfeoW7fuU6yYiIiIyjOThpu4uDiMGDEC48aNw+HDh9GiRQuEh4fj0qVLBfafOXMmUlJStI/k5GS4ubnh9ddff8qVExERUXmlEhEx1cKDg4PRqFEjzJ49W9tWu3ZtdO3aFdHR0cVOv3btWrz66qs4f/48fHx8DFpmRkYG1Go10tPT4eLiUuLaiYiI6Okx5vfbZFtuHj58iIMHDyIsLEynPSwsDElJSQbNY968eWjXrl2RwSYrKwsZGRk6DyIiIlIuk4Wb69evIycnBx4eHjrtHh4eSE1NLXb6lJQUbNq0CYMGDSqyX3R0NNRqtfbhzTt1ExERKZrJDyhWqVQ6f4uIXltBFi5ciAoVKqBr165F9hs7dizS09O1j2TeyJKIiEjRTHaF4kqVKsHS0lJvK01aWpre1pz8RATz589HZGQkbGxsiuxra2sLW1vbJ66XiIiIzIPJttzY2NggKCgI8fHxOu3x8fFo1qxZkdMmJibi7NmzGDhwYFmWSERERGbIpPeWioqKQmRkJBo3boyQkBDMnTsXly5dwpAhQwDk7VL6559/sHjxYp3p5s2bh+DgYNSrV88UZRMREVE5ZtJw06NHD9y4cQOTJk1CSkoK6tWrh40bN2rPfkpJSdG75k16ejpWrVqFmTNnmqJkIiIiKudMep0bU+B1boiIiMyPWVznhoiIiKgsMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaKYPNzExsbCz88PdnZ2CAoKws6dO4vsn5WVhXHjxsHHxwe2traoUaMG5s+f/5SqJSIiovLOypQLj4uLw4gRIxAbG4vmzZvj22+/RXh4OE6cOIHq1asXOE337t1x9epVzJs3D8899xzS0tKQnZ39lCsnIiKi8kolImKqhQcHB6NRo0aYPXu2tq127dro2rUroqOj9fpv3rwZPXv2xLlz5+Dm5laiZWZkZECtViM9PR0uLi4lrp2IiIieHmN+v022W+rhw4c4ePAgwsLCdNrDwsKQlJRU4DTr1q1D48aN8eWXX8LT0xPPP/88Ro8ejfv37xe6nKysLGRkZOg8iIiISLlMtlvq+vXryMnJgYeHh067h4cHUlNTC5zm3Llz2LVrF+zs7LBmzRpcv34db7/9Nm7evFnocTfR0dGYOHFiqddPRERE5ZPJDyhWqVQ6f4uIXptGbm4uVCoVli1bhiZNmqBTp06IiYnBwoULC916M3bsWKSnp2sfycnJpb4OREREVH6YbMtNpUqVYGlpqbeVJi0tTW9rjkbVqlXh6ekJtVqtbatduzZEBJcvX0bNmjX1prG1tYWtrW3pFk9ERETllsm23NjY2CAoKAjx8fE67fHx8WjWrFmB0zRv3hxXrlzBnTt3tG1//fUXLCws4OXlVab1EhERkXkw6W6pqKgofP/995g/fz5OnjyJkSNH4tKlSxgyZAiAvF1Kffr00fbv3bs3KlasiP79++PEiRPYsWMH3n//fQwYMAD29vamWg0iIiIqR0x6nZsePXrgxo0bmDRpElJSUlCvXj1s3LgRPj4+AICUlBRcunRJ29/JyQnx8fEYPnw4GjdujIoVK6J79+747LPPTLUKREREVM6Y9Do3psDr3BAREZkfs7jODREREVFZYLghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRbEydgIRQWJiInbu3IkLFy7g3r17qFy5Mho2bIh27drB29u7LOokIiIiMojBW27u37+PL774At7e3ggPD8cvv/yC27dvw9LSEmfPnsX48ePh5+eHTp06Ye/evWVZMxEREVGhDN5y8/zzzyM4OBhz5sxBhw4dYG1trdfn4sWL+OGHH9CjRw/897//xVtvvVWqxRIREREVRyUiYkjHY8eOoV69egbN9OHDh7h48SJq1qz5RMWVhYyMDKjVaqSnp8PFxcXU5RAREZEBjPn9Nni3lKHBBgBsbGzKZbAhIiIi5SvR2VKbN2/Grl27tH/PmjULL7zwAnr37o1bt26VWnFERERExipRuHn//feRkZEBADh69ChGjRqFTp064dy5c4iKiirVAomIiIiMYfSp4ABw/vx51KlTBwCwatUqRERE4IsvvsChQ4fQqVOnUi2QiIiIyBgl2nJjY2ODe/fuAQC2bt2KsLAwAICbm5t2iw4RERGRKZRoy81LL72EqKgoNG/eHL///jvi4uIAAH/99Re8vLxKtUAiIiIiY5Roy80333wDKysrrFy5ErNnz4anpycAYNOmTejYsWOpFkhERERkDIOvc6MUvM4NERGR+SmT69w8ztLSEmlpaXrtN27cgKWlZUlmSURERFQqShRuCtvYk5WVBRsbmycqiIiIiOhJGHVA8VdffQUAUKlU+P777+Hk5KR9LicnBzt27ECtWrVKt0IiIiIiIxgVbv73v/8ByNtyM2fOHJ1dUDY2NvD19cWcOXNKt0IiIiIiIxgVbs6fPw8AaNOmDVavXg1XV9cyKYqIiIiopEp0nZvt27eXdh1EREREpcLgcBMVFYVPP/0Ujo6Oxd4/KiYm5okLIyIiIioJg8PN4cOH8ejRI+2/C6NSqZ68KiIiQ6WnA5mZQEFXR798GXB2BtTqp18XEZkML+JHROYrPR3o2BFISwMSEgBv73+fS04GWrcG3N2BzZsZcIjMXJlfxO9xycnJuHz58pPOhojIeJmZecHm3Lm8IJOcnNeuCTbnzuU9n5lpyiqJ6CkrUbjJzs7Gxx9/DLVaDV9fX/j4+ECtVuO///2vdtcVEVGZ8/LK22Lj7/9vwElK+jfY+PvnPc8b+hI9U0p0ttSwYcOwZs0afPnllwgJCQEA7NmzBxMmTMD169d5rRsienq8vfMCjCbQNG+e164JNo/vqiKiZ0KJjrlRq9VYvnw5wsPDddo3bdqEnj17Ij09vdQKLG085oZIoZKS/g02ALB7N9CsmenqIaJSVebH3NjZ2cHX11ev3dfXl/eWIqKnLzkZiIzUbYuM/PcYHCJ6ppQo3Lzzzjv49NNPkZWVpW3LysrC559/jmHDhpVacURExXr84GF//7wtNo8fg8OAQ/TMMfiYm1dffVXn761bt8LLywsNGjQAABw5cgQPHz5EaGho6VZIRFSYy5f1Dx7OfwxO69ZAYiIPKiZ6hhgcbtT5rhHx2muv6fztzYP2iOhpc3bOu44NoHvw8OMBx909rx8RPTN4ET8iMm+8QjHRM8GY3+8SnQpORFRuqNWFhxfuiiJ6Jhl8QHHHjh2RlJRUbL/MzExMmTIFs2bNeqLCiIiIiErC4C03r7/+Orp37w5nZ2e8/PLLaNy4MapVqwY7OzvcunULJ06cwK5du7Bx40ZERERg6tSpZVk3ERERUYGMOubm4cOHWLlyJeLi4rBz507cvn07byYqFerUqYMOHTrgrbfeQkBAQFnV+8R4zA0REZH5Meb3+4kOKE5PT8f9+/dRsWJFWFtbl3Q2TxXDDRERkfl5agcUq9VqvVPEiYiIiEypRFcoJiIiIiqvTB5uYmNj4efnBzs7OwQFBWHnzp2F9k1ISIBKpdJ7nDp16ilWTEREROWZScNNXFwcRowYgXHjxuHw4cNo0aIFwsPDcenSpSKnO336NFJSUrSPmjVrPqWKiYiIqLwzabiJiYnBwIEDMWjQINSuXRszZsyAt7c3Zs+eXeR07u7uqFKlivZhaWn5lComIiKi8s5k4ebhw4c4ePAgwsLCdNrDwsKKvVhgw4YNUbVqVYSGhmL79u1F9s3KykJGRobOg4iIiJTLqLOlXF1doVKpiu138+bNYvtcv34dOTk58PDw0Gn38PBAampqgdNUrVoVc+fORVBQELKysrBkyRKEhoYiISEBLVu2LHCa6OhoTJw4sdh6iIiISBmMCjczZswo9QLyhyURKTRABQQE6FwgMCQkBMnJyZg2bVqh4Wbs2LGIiorS/p2RkcE7mBMRESmYUeGmb9++pbbgSpUqwdLSUm8rTVpamt7WnKI0bdoUS5cuLfR5W1tb2NralrhOIiIiMi8mO+bGxsYGQUFBiI+P12mPj49Hs2bNDJ7P4cOHUbVq1dIuj4iIiMyUUVtu/P39Dep37tw5g/pFRUUhMjISjRs3RkhICObOnYtLly5hyJAhAPJ2Kf3zzz9YvHgxgLzdYr6+vqhbty4ePnyIpUuXYtWqVVi1apUxq0FEREQKZlS4uXDhAnx8fNC7d2+4u7s/8cJ79OiBGzduYNKkSUhJSUG9evWwceNG+Pj4AABSUlJ0rnnz8OFDjB49Gv/88w/s7e1Rt25d/PLLL+jUqdMT10JERETKYNSNM3/66ScsWLAACQkJCA8Px4ABA9CpUydYWJj8QscG440ziYiIzI8xv99GpZLu3btj06ZNOHv2LIKCgjBy5Eh4eXlhzJgxOHPmzBMVTURERFQaSrTJxdPTE+PGjcOZM2fw448/Yt++fahVqxZu3bpV2vURERERGcWoY24e9+DBA6xcuRLz58/Hvn378Prrr8PBwaE0ayMiIiIymtHhZt++fZg3bx7i4uJQo0YNDBgwAKtWrYKrq2tZ1EdERERkFKPCTd26dZGWlobevXtj586dCAwMLKu6iIiIiErEqLOlLCws4OjoCCsrqyLvMWXIvaVMhWdLERERmR9jfr+N2nKzYMGCJyqMiIiIqKyZ7N5SRERERGXBfK6+R0RERGQAhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIPPloqKijJ4pjExMSUqhoiIiOhJGRxuDh8+rPP3wYMHkZOTg4CAAADAX3/9BUtLSwQFBZVuhURERERGMDjcbN++XfvvmJgYODs7Y9GiRdp7St26dQv9+/dHixYtSr9KIiIiIgMZdfsFDU9PT2zZsgV169bVaT927BjCwsJw5cqVUiuwtPH2C0RERObHmN/vEh1QnJGRgatXr+q1p6WlITMzsySzJCIiIioVJQo3r7zyCvr374+VK1fi8uXLuHz5MlauXImBAwfi1VdfLe0aiYiIiAxm1L2lNObMmYPRo0fjzTffxKNHj/JmZGWFgQMHYurUqaVaIBEREZExSnTMjcbdu3fx999/Q0Tw3HPPwdHRsTRrKxM85oaIiMj8GPP7XaItNxqOjo4IDAx8klkQERERlaoShZu7d+9i8uTJ+O2335CWlobc3Fyd58+dO1cqxREREREZq0ThZtCgQUhMTERkZCSqVq0KlUpV2nURERERlUiJws2mTZvwyy+/oHnz5qVdDxEREdETKdGp4K6urnBzcyvtWoiIiIieWInCzaeffopPPvkE9+7dK+16iIiIiJ5IiXZLTZ8+HX///Tc8PDzg6+sLa2trnecPHTpUKsURERERGatE4aZr166lXAYRERFR6Xiii/iZI17Ej4iIyPw8tYv4HTx4ECdPnoRKpUKdOnXQsGHDJ5kdERER0RMrUbhJS0tDz549kZCQgAoVKkBEkJ6ejjZt2mD58uWoXLlyaddJREREZJASnS01fPhwZGRk4Pjx47h58yZu3bqFY8eOISMjA++++25p10hERERksBIdc6NWq7F161a8+OKLOu2///47wsLCcPv27dKqr9TxmBsiIiLzY8zvd4m23OTm5uqd/g0A1tbWeveZIiIiInqaShRu2rZti/feew9XrlzRtv3zzz8YOXIkQkNDS604IiIiImOVKNx88803yMzMhK+vL2rUqIHnnnsOfn5+yMzMxNdff13aNRIREREZrERnS3l7e+PQoUOIj4/HqVOnICKoU6cO2rVrV9r1ERERERmFF/EjIiKicq/MDijetm0b6tSpg4yMDL3n0tPTUbduXezcudO4aomIiIhKkVHhZsaMGXjrrbcKTExqtRqDBw9GTExMqRVHREREZCyjws2RI0fQsWPHQp8PCwvDwYMHn7goIiIiopIyKtxcvXq1wOvbaFhZWeHatWtPXBQRERFRSRkVbjw9PXH06NFCn//zzz9RtWrVJy6KiIiIqKSMCjedOnXCJ598ggcPHug9d//+fYwfPx4RERGlVhwRERGRsYw6Ffzq1ato1KgRLC0tMWzYMAQEBEClUuHkyZOYNWsWcnJycOjQIXh4eJRlzU+Ep4ITERGZH2N+v426iJ+HhweSkpIwdOhQjB07FppcpFKp0KFDB8TGxpbrYENERETKZ/QVin18fLBx40bcunULZ8+ehYigZs2acHV1LYv6iIiIiIxSotsvAICrqytefPHF0qyFiIiI6ImV6MaZREREROUVww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKYrJw01sbCz8/PxgZ2eHoKAg7Ny506Dpdu/eDSsrK7zwwgtlWyARERGZFZOGm7i4OIwYMQLjxo3D4cOH0aJFC4SHh+PSpUtFTpeeno4+ffogNDT0KVVKRERE5kIlmlt7m0BwcDAaNWqE2bNna9tq166Nrl27Ijo6utDpevbsiZo1a8LS0hJr167FH3/8YfAyjbllOhEREZUPxvx+m2zLzcOHD3Hw4EGEhYXptIeFhSEpKanQ6RYsWIC///4b48ePN2g5WVlZyMjI0HkQERGRcpks3Fy/fh05OTnw8PDQaffw8EBqamqB05w5cwZjxozBsmXLYGVl2A3No6OjoVartQ9vb+8nrp2IiIjKL5MfUKxSqXT+FhG9NgDIyclB7969MXHiRDz//PMGz3/s2LFIT0/XPpKTk5+4ZiIiIiq/DNv8UQYqVaoES0tLva00aWlpeltzACAzMxMHDhzA4cOHMWzYMABAbm4uRARWVlbYsmUL2rZtqzedra0tbG1ty2YliIiIqNwx2ZYbGxsbBAUFIT4+Xqc9Pj4ezZo10+vv4uKCo0eP4o8//tA+hgwZgoCAAPzxxx8IDg5+WqUTERFROWayLTcAEBUVhcjISDRu3BghISGYO3cuLl26hCFDhgDI26X0zz//YPHixbCwsEC9evV0pnd3d4ednZ1eOxERET27TBpuevTogRs3bmDSpElISUlBvXr1sHHjRvj4+AAAUlJSir3mDREREdHjTHqdG1PgdW6IiIjMj1lc54aIiIioLDDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3BAREZGiMNwQEZFppacDly8X/Nzly3nPExmB4YaIiEwnPR3o2BFo1QpITtZ9Ljk5r71jRwYcMgrDDRERmU5mJpCWBpw7B7Ru/W/ASU7O+/vcubznMzNNWSWZGYYbIiIyHS8vICEB8Pf/N+AkJf0bbPz985738jJtnWRWrExdABERPeO8vfMCjCbQNG+e164JNt7eJiyOzBG33BARkel5ewNLlui2LVnCYEMlwnBDRESml5wMREbqtkVG6h9kTGQAhhsiIjKtxw8e9vcHdu/WPQaHAYeMxHBDRESmc/my/sHDzZrpH2Rc2HVwiArAA4qJiMh0nJ0Bd/e8fz9+8PDjBxm7u+f1IzIQww0REZmOWg1s3px3HZv8p3t7ewOJiXnBRq02TX1klhhuiIjItNTqwsMLr29DJcBjboiIiEhRGG6IiIhIURhuiIiISFFMHm5iY2Ph5+cHOzs7BAUFYefOnYX23bVrF5o3b46KFSvC3t4etWrVwv/+97+nWC0RERGVdyY9oDguLg4jRoxAbGwsmjdvjm+//Rbh4eE4ceIEqlevrtff0dERw4YNQ2BgIBwdHbFr1y4MHjwYjo6O+L//+z8TrAERERGVNyoREVMtPDg4GI0aNcLs2bO1bbVr10bXrl0RHR1t0DxeffVVODo6Ykn+e5IUIiMjA2q1Gunp6XBxcSlR3URERPR0GfP7bbLdUg8fPsTBgwcRFham0x4WFoakpCSD5nH48GEkJSWhVatWhfbJyspCRkaGzoOIiIiUy2Th5vr168jJyYGHh4dOu4eHB1JTU4uc1svLC7a2tmjcuDHeeecdDBo0qNC+0dHRUKvV2oc37zBLRESkaCY/oFilUun8LSJ6bfnt3LkTBw4cwJw5czBjxgz8+OOPhfYdO3Ys0tPTtY9k3oCNiIhI0Ux2QHGlSpVgaWmpt5UmLS1Nb2tOfn5+fgCA+vXr4+rVq5gwYQJ69epVYF9bW1vY2tqWTtFERERU7plsy42NjQ2CgoIQHx+v0x4fH49mzZoZPB8RQVZWVmmXR0RERGbKpKeCR0VFITIyEo0bN0ZISAjmzp2LS5cuYciQIQDydin9888/WLx4MQBg1qxZqF69OmrVqgUg77o306ZNw/Dhw022DkRERFS+mDTc9OjRAzdu3MCkSZOQkpKCevXqYePGjfDx8QEApKSk4NKlS9r+ubm5GDt2LM6fPw8rKyvUqFEDkydPxuDBg021CkRERFTOmPQ6N6bA69wQERGZH7O4zg0RERFRWWC4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiKiJ5OeDly+XPBzly/nPf8UMdwQERFRyaWnAx07Aq1aAcnJus8lJ+e1d+z4VAMOww0RERGVXGYmkJYGnDsHtG79b8BJTs77+9y5vOczM59aSQw3REREVHJeXkBCAuDv/2/ASUr6N9j4++c97+X11EqyempLIiIiImXy9s4LMJpA07x5Xrsm2Hh7P9VyuOWGiIiInpy3N7BkiW7bkiVPPdgADDdERERUGpKTgchI3bbISP2DjJ8ChhsiIiJ6Mo8fPOzvD+zerXsMzlMOOAw3REREVHKXL+sfPNysmf5BxoVdB6cM8IBiIiIiKjlnZ8DdPe/fjx88/PhBxu7uef2eEoYbIiIiKjm1Gti8Oe86NvlP9/b2BhIT84KNWv3USmK4ISIioiejVhceXp7i9W00eMwNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpCsMNERERKQrDDRERESkKww0REREpyjN3hWIRAQBkZGSYuBIiIiIylOZ3W/M7XpRnLtxkZmYCALw1N/YiIiIis5GZmQl1MfepUokhEUhBcnNzceXKFTg7O0OlUpXqvDMyMuDt7Y3k5GS4uLiU6rzLA6WvH6D8deT6mT+lryPXz/yV1TqKCDIzM1GtWjVYWBR9VM0zt+XGwsICXmV8Ey8XFxfFvmkB5a8foPx15PqZP6WvI9fP/JXFOha3xUaDBxQTERGRojDcEBERkaIw3JQiW1tbjB8/Hra2tqYupUwoff0A5a8j18/8KX0duX7mrzys4zN3QDEREREpG7fcEBERkaIw3BAREZGiMNwQERGRojDcEBERkaIw3Bhox44d6NKlC6pVqwaVSoW1a9cWO01iYiKCgoJgZ2cHf39/zJkzp+wLfQLGrmNCQgJUKpXe49SpU0+nYCNFR0fjxRdfhLOzM9zd3dG1a1ecPn262OnMZRxLsn7mNIazZ89GYGCg9sJgISEh2LRpU5HTmMvYaRi7juY0fgWJjo6GSqXCiBEjiuxnbuOoYcj6mdsYTpgwQa/WKlWqFDmNKcaP4cZAd+/eRYMGDfDNN98Y1P/8+fPo1KkTWrRogcOHD+Ojjz7Cu+++i1WrVpVxpSVn7DpqnD59GikpKdpHzZo1y6jCJ5OYmIh33nkHe/fuRXx8PLKzsxEWFoa7d+8WOo05jWNJ1k/DHMbQy8sLkydPxoEDB3DgwAG0bdsW//nPf3D8+PEC+5vT2GkYu44a5jB++e3fvx9z585FYGBgkf3McRwBw9dPw5zGsG7dujq1Hj16tNC+Jhs/IaMBkDVr1hTZ54MPPpBatWrptA0ePFiaNm1ahpWVHkPWcfv27QJAbt269VRqKm1paWkCQBITEwvtY87jaMj6mfsYurq6yvfff1/gc+Y8do8rah3NdfwyMzOlZs2aEh8fL61atZL33nuv0L7mOI7GrJ+5jeH48eOlQYMGBvc31fhxy00Z2bNnD8LCwnTaOnTogAMHDuDRo0cmqqpsNGzYEFWrVkVoaCi2b99u6nIMlp6eDgBwc3MrtI85j6Mh66dhbmOYk5OD5cuX4+7duwgJCSmwjzmPHWDYOmqY2/i988476Ny5M9q1a1dsX3McR2PWT8OcxvDMmTOoVq0a/Pz80LNnT5w7d67QvqYav2fuxplPS2pqKjw8PHTaPDw8kJ2djevXr6Nq1aomqqz0VK1aFXPnzkVQUBCysrKwZMkShIaGIiEhAS1btjR1eUUSEURFReGll15CvXr1Cu1nruNo6PqZ2xgePXoUISEhePDgAZycnLBmzRrUqVOnwL7mOnbGrKO5jR8ALF++HIcOHcL+/fsN6m9u42js+pnbGAYHB2Px4sV4/vnncfXqVXz22Wdo1qwZjh8/jooVK+r1N9X4MdyUIZVKpfO3/P+LQedvN1cBAQEICAjQ/h0SEoLk5GRMmzatXH4oHzds2DD8+eef2LVrV7F9zXEcDV0/cxvDgIAA/PHHH7h9+zZWrVqFvn37IjExsdAff3McO2PW0dzGLzk5Ge+99x62bNkCOzs7g6czl3EsyfqZ2xiGh4dr/12/fn2EhISgRo0aWLRoEaKiogqcxhTjx91SZaRKlSpITU3VaUtLS4OVlVWB6VYpmjZtijNnzpi6jCINHz4c69atw/bt2+Hl5VVkX3McR2PWryDleQxtbGzw3HPPoXHjxoiOjkaDBg0wc+bMAvua49gBxq1jQcrz+B08eBBpaWkICgqClZUVrKyskJiYiK+++gpWVlbIycnRm8acxrEk61eQ8jyG+Tk6OqJ+/fqF1muq8eOWmzISEhKC9evX67Rt2bIFjRs3hrW1tYmqKnuHDx8ud5uJNUQEw4cPx5o1a5CQkAA/P79ipzGncSzJ+hWkPI9hfiKCrKysAp8zp7ErSlHrWJDyPH6hoaF6Z9b0798ftWrVwocffghLS0u9acxpHEuyfgUpz2OYX1ZWFk6ePIkWLVoU+LzJxq9MD1dWkMzMTDl8+LAcPnxYAEhMTIwcPnxYLl68KCIiY8aMkcjISG3/c+fOiYODg4wcOVJOnDgh8+bNE2tra1m5cqWpVqFYxq7j//73P1mzZo389ddfcuzYMRkzZowAkFWrVplqFYo0dOhQUavVkpCQICkpKdrHvXv3tH3MeRxLsn7mNIZjx46VHTt2yPnz5+XPP/+Ujz76SCwsLGTLli0iYt5jp2HsOprT+BUm/9lEShjHxxW3fuY2hqNGjZKEhAQ5d+6c7N27VyIiIsTZ2VkuXLggIuVn/BhuDKQ5XS//o2/fviIi0rdvX2nVqpXONAkJCdKwYUOxsbERX19fmT179tMv3AjGruOUKVOkRo0aYmdnJ66urvLSSy/JL7/8YpriDVDQugGQBQsWaPuY8ziWZP3MaQwHDBggPj4+YmNjI5UrV5bQ0FDtj76IeY+dhrHraE7jV5j8P/5KGMfHFbd+5jaGPXr0kKpVq4q1tbVUq1ZNXn31VTl+/Lj2+fIyfiqR/39kDxEREZEC8IBiIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIiIiUhSGGyIiIlIUhhsiIiJSFIYbIir3+vXrh65du5q6DCIyEww3REREpCgMN0Rk1mJiYlC/fn04OjrC29sbb7/9Nu7cuaPT57vvvoO3tzccHBzwyiuvICYmBhUqVDBNwURU5hhuiMisWVhY4KuvvsKxY8ewaNEibNu2DR988IH2+d27d2PIkCF477338Mcff6B9+/b4/PPPTVgxEZU13hWciMq9fv364fbt21i7dm2xfVesWIGhQ4fi+vXrAICePXvizp072LBhg7bPm2++iQ0bNuD27dtlVDERmRK33BCRWdu+fTvat28PT09PODs7o0+fPrhx4wbu3r0LADh9+jSaNGmiM03+v4lIWRhuiMhsXbx4EZ06dUK9evWwatUqHDx4ELNmzQIAPHr0CAAgIlCpVDrTcYM1kbJZmboAIqKSOnDgALKzszF9+nRYWOT9X+2nn37S6VOrVi38/vvvetMRkXIx3BCRWUhPT8cff/yh01a5cmVkZ2fj66+/RpcuXbB7927MmTNHp8/w4cPRsmVLxMTEoEuXLti2bRs2bdqktzWHiJSDBxQTUbnXr18/LFq0SK+9b9++aNCgAaZOnYrbt2+jZcuWeOONN9CnTx/cunVLe7r3d999h4kTJ+LmzZvo0KEDGjdujG+++QYpKSlPeU2I6GlguCGiZ85bb72FU6dOYefOnaYuhYjKAHdLEZHiTZs2De3bt4ejoyM2bdqERYsWITY21tRlEVEZ4ZYbIlK87t27IyEhAZmZmfD398fw4cMxZMgQU5dFRGWE4YaIiIgUhde5ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJFYbghIiIiRWG4ISIiIkVhuCEiIiJF+X+EeqDHyWoCFQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Compute the mutual information between successive characters conditioned on up to 5 characters in between:\n", + "maxLag = 5;\n", + "\n", + "condMisVsLag = np.zeros(maxLag);\n", + "conditionalCharacters = None;\n", + "\n", + "nextChar = processedStr[maxLag:];\n", + "for lag in range(1,maxLag+1): # To go from 1 up to maxLag\n", + " sourceChar = processedStr[maxLag-lag:-lag];\n", + " if (lag == 1):\n", + " # Nothing to condition on, just compute MI\n", + " condMisVsLag[lag-1] = simpleinfotheory.mutualinformationempirical(sourceChar,nextChar)[0];\n", + " conditionalCharacters = sourceChar; # Initialise the characters to now be conditioned on\n", + " else:\n", + " condMisVsLag[lag-1] = simpleinfotheory.conditionalmutualinformationempirical(sourceChar,nextChar,conditionalCharacters);\n", + " conditionalCharacters = np.column_stack( (conditionalCharacters,sourceChar) ); # Add to the characters to be conditioned on\n", + " print('cond MI over lag %d is %.4f\\n' % (lag, condMisVsLag[lag-1]));\n", + "\n", + "plt.scatter(range(1,maxLag+1), condMisVsLag, c=\"red\", marker=\"x\");\n", + "plt.xlabel('Lag')\n", + "plt.ylabel('Cond MI (bits)');\n", + "plt.title('Average MI between characters separated by the\\ngiven lag conditioned on intervening chars');\n", + "plt.savefig('CondMIversusLag.png')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "498d4c88-23f4-4544-922d-e812255fd914", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.ipynb new file mode 100644 index 0000000..c2a86f5 --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.ipynb @@ -0,0 +1,196 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Entropy of written English text\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 12. (Optional extension) Entropy of written English text\n", + "\n", + "Let's compute the Shannon information contents of letters in English language text ourselves, using the collected scripts from the 1990s comedy [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld).\n", + "\n", + "1. Download the collection of text extracted from Seinfeld scripts following the links on Module 2 on Canvas.\n", + "1. Open the data in a text file to inspect it (you should always do this!). We have each character's line on a different line of text. There is much punctuation in here as well.\n", + "1. Load the data into Python: (_note_ you may need to alter the filename/path to match your own)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [], + "source": [ + "filename = './Seinfeld-scripts-textOnly.txt'\n", + "with open(filename, 'rt') as f:\n", + " str = f.read()" + ] + }, + { + "cell_type": "markdown", + "id": "dc2ec16c-3f84-49a0-ba1a-99c0b07f53d2", + "metadata": {}, + "source": [ + "4. Now we need to pre-process it to remove punctuation characters, digits, and newlines (which we'll turn into spaces), and convert all upper case characters into lower case. We'll also convert it to a numpy array. Afterwards, let's check that we're only left with characters and spaces by examining the set of unique symbols in `processedStr` (leave the \";\" off so we see the output!):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18ab6e30-72c9-481c-b191-2c4c81081f3f", + "metadata": {}, + "outputs": [], + "source": [ + "p = re.compile('[!\"#\\$%&\\'\\(\\)\\*\\+\\,-\\.\\/:;<=>\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "5. Now compute the average entropy of these characters, as derived from their probabilities of occurrence in the Seinfeld script, using your `simpleinfotheory.entropyempirical()` function. Please note:\n", + " - You will need to have imported your `simpleinfotheory` scripts as above\n", + " - I would suggest that you edit your function `entropyempirical()` in `simpleinfotheory.py` to uncomment the line `[symbols, counts] = np.unique(xn, axis=0, return_counts=True)` instead of the subsequent for loop (which can be commented out), but still include the line `probabilities = counts / xnSamples`. (You can see how this is done in the simpleinfotheory.py solution code). This will run much faster. You will need to restart the kernel for this to take effect.\n", + "\n", + " How does this compare to the stated value of the entropy of characters from [Mackay](http://www.inference.org.uk/itprnn/book.pdf) in Table 2.9 (sec 2.3; or see slide 26 of our lecture) as estimated from \"_The Frequently Asked Questions Manual for Linux_\"? Did you expect it to be the same, and why or why not?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1225213-86e2-49f3-9ff9-dfe0f6d34876", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the entropy of the characters:\n" + ] + }, + { + "cell_type": "markdown", + "id": "9ec8b8b0-92ea-444b-9b02-aeb3c01ab679", + "metadata": {}, + "source": [ + "6. Next, compute the Shannon information content of each character, and again compare these to those quoted by Mackay.
\n", + "You will have noticed that the `simpleinfotheory.entropyempirical()` function returns the probabilities of each symbol as well as the result in a tuple `(result, symbols, probabilities)` (see more details in its header). So, when you call the function, make sure that you have accepted all output variables as follows: `(result, symbols, probabilities) = simpleinfotheory.entropyempirical(processedStr)`. You can then send the probabilities as an argument to your `simpleinfotheory.infocontent()` code. On comparing to Mackay's results for each character, remember that your Shannon information contents are for the characters in a sorted order, but that order may be different to what the book displays -- yours will be displayed for each character in the order they appear in `symbols` (which is as returned by `np.unique(processedStr)` above)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3a37011-0238-47d0-85a3-b384573fa2c0", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the Shannon information content of each character:\n" + ] + }, + { + "cell_type": "markdown", + "id": "e7f91288-9d24-4736-ae88-b8f3562d09a4", + "metadata": {}, + "source": [ + "7. _Next level challenge_: can you move on to compute joint entropies for consecutive appearance of two characters, and then the conditional entropy of the second given the first.
\n", + " _Hint_: to select all but the last item in a numpy array `x`, you can refer to `x[:-1]`, whilst to select all but the first item in an array `x`, you can refer to `x[1:]`
\n", + " What does this tell us about how reading one character reduces our uncertainty about the next, and does this make sense?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9a980f0-ad8a-4aea-9fbe-5d0a78282db8", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the joint entropies for two characters:\n", + "\n", + "# Compute the conditional entropy of the second character given the first:\n" + ] + }, + { + "cell_type": "markdown", + "id": "37b50b62-6af9-4b66-84c3-b2c180ad9d28", + "metadata": {}, + "source": [ + "A more serious challenge would be to display the joint Shannon information contents, and the conditional Shannon information contents, as per Figures 2.2 and 2.3 of Mackay. This cannot be done with a simple modification to our simple Matlab scripts, as they were not set up to return the probabilities in a nicely ordered way for all possible combinations. (That was sacrificed to make your other tasks easier!). But you could attempt to pull out a list of all observed joint symbols and their probabilities, and sort them nicely yourself ready for display in such a figure. We will work further on this in the next module (and solutions are deferred to that module)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.m b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.m new file mode 100644 index 0000000..36e5ab6 --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters.m @@ -0,0 +1,34 @@ +str = fileread('Seinfeld-scripts-textOnly.txt'); +processedStr = regexprep(str, '[!"#$%&''()\*,-./;<=>?\[\\\]_`{}~]', ''); % Remove punctuation characters +processedStr = regexprep(processedStr, '[0-9]', ''); % Removed digits +processedStr = replace(processedStr, newline, ' '); % Replace newline characters with spaces +processedStr = lower(processedStr); % Convert all upper case into lower case +unique(processedStr) + +% Check that this is adding a path to your scripts correctly: +addpath('../../Module1-IntroToInfoTheory/MatlabCode/completed'); + +% Compute the entropy of individual characters: +[result, symbols, probabilities] = entropyempirical(processedStr); +fprintf('Entropy of individual characters: %.4f bits\n', result); + +% Now compute the info content of individual characters: +characterInfoContents = infocontent(probabilities); + +% To just dump them to the screen: +% characterInfoContents +% To display more nicely: +for ix = 1:length(symbols) + fprintf('Info content of %s is %.4f bits\n', symbols(ix), characterInfoContents(ix)); +end + +% Now compute joint entropies for two characters: +% Need a matrix with first column being first character, and second column +% being the second +characterPairSamples = [processedStr(1:end-1)',processedStr(2:end)']; +pairEntropy = jointentropyempirical(characterPairSamples); +fprintf('Entropy of characters pairs: %.4f bits\n', pairEntropy); +% Finally compute the conditional entropy of the second character given the +% first: +conditionalEntropy = conditionalentropyempirical(characterPairSamples(:,2), characterPairSamples(:,1)); +fprintf('Conditional entropy of character given previous: %.4f bits\n', conditionalEntropy); diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters_Solution.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters_Solution.ipynb new file mode 100644 index 0000000..c4656e2 --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/entropyOfCharacters_Solution.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Entropy of written English text\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 12. (Optional extension) Entropy of written English text\n", + "\n", + "Let's compute the Shannon information contents of letters in English language text ourselves, using the collected scripts from the 1990s comedy [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld).\n", + "\n", + "1. Download the collection of text extracted from Seinfeld scripts following the links on Module 2 on Canvas.\n", + "1. Open the data in a text file to inspect it (you should always do this!). We have each character's line on a different line of text. There is much punctuation in here as well.\n", + "1. Load the data into Python: (_note_ you may need to alter the filename/path to match your own)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [], + "source": [ + "filename = './Seinfeld-scripts-textOnly.txt'\n", + "with open(filename, 'rt') as f:\n", + " str = f.read()" + ] + }, + { + "cell_type": "markdown", + "id": "dc2ec16c-3f84-49a0-ba1a-99c0b07f53d2", + "metadata": {}, + "source": [ + "4. Now we need to pre-process it to remove punctuation characters, digits, and newlines (which we'll turn into spaces), and convert all upper case characters into lower case. We'll also convert it to a numpy array. Afterwards, let's check that we're only left with characters and spaces by examining the set of unique symbols in `processedStr` (leave the \";\" off so we see the output!):" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "75054903-0006-40fc-8b52-f5ec78c46e3e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([' ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',\n", + " 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',\n", + " 'z'], dtype='\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "5. Now compute the average entropy of these characters, as derived from their probabilities of occurrence in the Seinfeld script, using your `simpleinfotheory.entropyempirical()` function. Please note:\n", + " - You will need to have imported your `simpleinfotheory` scripts as above\n", + " - I would suggest that you edit your function `entropyempirical()` in `simpleinfotheory.py` to uncomment the line `[symbols, counts] = np.unique(xn, axis=0, return_counts=True)` instead of the subsequent for loop (which can be commented out), but still include the line `probabilities = counts / xnSamples`. (You can see how this is done in the `simpleinfotheory.py` solution code). This will run much faster. You will need to restart the kernel for this to take effect.\n", + "\n", + " How does this compare to the stated value of the entropy of characters from [Mackay](http://www.inference.org.uk/itprnn/book.pdf) in Table 2.9 (sec 2.3; or see slide 26 of our lecture) as estimated from \"_The Frequently Asked Questions Manual for Linux_\"? Did you expect it to be the same, and why or why not?" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "37767929-ed8a-4a8a-a8f9-1c8a2842b0cf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Entropy of single characters from Seinfeld scripts is 4.0846 bits\n" + ] + } + ], + "source": [ + "# Compute the entropy of the characters:\n", + "(result, symbols, probabilities) = simpleinfotheory.entropyempirical(processedStr)\n", + "print(\"Entropy of single characters from Seinfeld scripts is %.4f bits\" % result)" + ] + }, + { + "cell_type": "markdown", + "id": "24ca71e8-9b68-40ed-8124-473aa4275567", + "metadata": {}, + "source": [ + "6. Next, compute the Shannon information content of each character, and again compare these to those quoted by Mackay.
\n", + "You will have noticed that the `simpleinfotheory.entropyempirical()` function returns the probabilities of each symbol as well as the result in a tuple `(result, symbols, probabilities)` (see more details in its header). So, when you call the function, make sure that you have accepted all output variables as follows: `(result, symbols, probabilities) = simpleinfotheory.entropyempirical(processedStr)`. You can then send the probabilities as an argument to your `simpleinfotheory.infocontent()` code. On comparing to Mackay's results for each character, remember that your Shannon information contents are for the characters in a sorted order, but that order may be different to what the book displays -- yours will be displayed for each character in the order they appear in `symbols` (which is as returned by `np.unique(processedStr)` above)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0c84c738-8497-481c-970f-f63ac0215b79", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Info content of [' '] is 2.3116 bits\n", + "Info content of ['a'] is 4.0322 bits\n", + "Info content of ['b'] is 6.4422 bits\n", + "Info content of ['c'] is 5.9460 bits\n", + "Info content of ['d'] is 5.2915 bits\n", + "Info content of ['e'] is 3.4355 bits\n", + "Info content of ['f'] is 6.4115 bits\n", + "Info content of ['g'] is 5.4955 bits\n", + "Info content of ['h'] is 4.3057 bits\n", + "Info content of ['i'] is 4.1372 bits\n", + "Info content of ['j'] is 8.3484 bits\n", + "Info content of ['k'] is 6.3208 bits\n", + "Info content of ['l'] is 4.9140 bits\n", + "Info content of ['m'] is 5.5827 bits\n", + "Info content of ['n'] is 4.2814 bits\n", + "Info content of ['o'] is 3.8170 bits\n", + "Info content of ['p'] is 6.4067 bits\n", + "Info content of ['q'] is 11.2288 bits\n", + "Info content of ['r'] is 4.6424 bits\n", + "Info content of ['s'] is 4.4777 bits\n", + "Info content of ['t'] is 3.7499 bits\n", + "Info content of ['u'] is 5.1079 bits\n", + "Info content of ['v'] is 7.1339 bits\n", + "Info content of ['w'] is 5.5464 bits\n", + "Info content of ['x'] is 9.9738 bits\n", + "Info content of ['y'] is 5.1689 bits\n", + "Info content of ['z'] is 10.5129 bits\n" + ] + } + ], + "source": [ + "# Compute the Shannon information content of each character:\n", + "characterInfoContents = simpleinfotheory.infocontent(probabilities)\n", + "# To display more nicely:\n", + "for ix in range(symbols.size):\n", + " print('Info content of %s is %.4f bits' % (symbols[ix], characterInfoContents[ix]));" + ] + }, + { + "cell_type": "markdown", + "id": "4fa01559-3c8e-4987-86eb-03b83a9032da", + "metadata": {}, + "source": [ + "7. _Next level challenge_: can you move on to compute joint entropies for consecutive appearance of two characters, and then the conditional entropy of the second given the first.
\n", + " _Hint_: to select all but the last item in a numpy array `x`, you can refer to `x[:-1]`, whilst to select all but the first item in an array `x`, you can refer to `x[1:]`
\n", + " What does this tell us about how reading one character reduces our uncertainty about the next, and does this make sense?" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0b962f0e-208a-4017-9d2f-3bd26850bbc0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Entropy of characters pairs: 7.4496 bits\n", + "Conditional entropy of character given previous: 3.3650 bits\n" + ] + } + ], + "source": [ + "# Compute the joint entropies for two characters:\n", + "# Need a matrix with first column being first character, and second column\n", + "# being the second\n", + "characterPairSamples = np.column_stack( (processedStr[:-1],processedStr[1:]) );\n", + "pairEntropy,__,__ = simpleinfotheory.jointentropyempirical(characterPairSamples);\n", + "print('Entropy of characters pairs: %.4f bits' % pairEntropy);\n", + "# Compute the conditional entropy of the second character given the first:\n", + "conditionalEntropy = simpleinfotheory.conditionalentropyempirical(processedStr[1:], processedStr[:-1]);\n", + "print('Conditional entropy of character given previous: %.4f bits' % conditionalEntropy);" + ] + }, + { + "cell_type": "markdown", + "id": "231703d1-a22d-4a19-a22f-fea28c7a2558", + "metadata": {}, + "source": [ + "A more serious challenge would be to display the joint Shannon information contents, and the conditional Shannon information contents, as per Figures 2.2 and 2.3 of Mackay. This cannot be done with a simple modification to our simple Matlab scripts, as they were not set up to return the probabilities in a nicely ordered way for all possible combinations. (That was sacrificed to make your other tasks easier!). But you could attempt to pull out a list of all observed joint symbols and their probabilities, and sort them nicely yourself ready for display in such a figure. We will work further on this in the next module (and solutions are deferred to that module)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "170d20cd-1072-4a1a-b8d4-c6adab0a2d4f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLag.m b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLag.m new file mode 100644 index 0000000..330e5bb --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLag.m @@ -0,0 +1,17 @@ +% assumes processedStr holds the text as previously processed - you can run +% the previous solution code entropyOfCharacters.m to pull this up + +% Compute MI as a function of lag: + +maxLag = 10; + +misVsLag = zeros(1, maxLag); + +for lag=1:maxLag + misVsLag(lag) = mutualinformationempirical(processedStr(1:end-lag), processedStr(1+lag:end)); +end + +plot(1:maxLag, misVsLag, 'rx'); +xlabel('Lag') +ylabel('MI (bits)'); +title('Average MI between characters separated by the given lag'); diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise.ipynb new file mode 100644 index 0000000..131ad4c --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise.ipynb @@ -0,0 +1,260 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Mutual information of written English text\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` etc. we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 6. (Optional extension) Mutual information between successive letters in written English\n", + "\n", + "In this extension activity, we will continue our analysis of written English extracted from the [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld) scripts as begun in the previous module.\n", + "\n", + "1. Download the scripts from the links on Module 2 on Canvas, load into Python and preprocess as per steps 1-4 of the activity from the previous module, such that we have the characters stored in the numpy array `processedStr`:
\n", + "_Note:_ you may need to alter the filename/path to match your own --" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [], + "source": [ + "filename = './Seinfeld-scripts-textOnly.txt'\n", + "with open(filename, 'rt') as f:\n", + " str = f.read()\n", + "p = re.compile('[!\"#\\$%&\\'\\(\\)\\*\\+\\,-\\.\\/:;<=>\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "2. How can we now compute the mutual information between one character and the character that comes next in the text? We will need to provide samples of a previous character and the next character to our `simpleinfotheory.mutualinformationempirical()` function.
\n", + " _Hint_: to select all but the last item in a numpy array `x`, you can refer to `x[:-1]`, whilst to select all but the first item in an array `x`, you can refer to `x[1:]`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1225213-86e2-49f3-9ff9-dfe0f6d34876", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the mutual information between successive characters:\n" + ] + }, + { + "cell_type": "markdown", + "id": "ab60c5ec-592a-4e15-97c7-280f7ce91253", + "metadata": {}, + "source": [ + "3. Compare the mutual information that you computed above to the average entropy of each character computed as per step 5 of the activity in the previous module. Consider the following:\n", + " 1. What proportion of our uncertainty about the next character in the written text is reduced by observing the previous character?\n", + " 1. How much code could we save in communicating a character if our coding scheme took the previous character into account?\n", + " 1. The mutual information computes a measure of the relationship between the consecutive characters here. You're probably familiar with using correlation to measure a relationship between variables -- could correlation be used here? We will see more about how MI and correlation are related in the coming weeks.\n", + "4. Are there relationships between previous characters and later characters beyond those which are consecutive?
\n", + " Can you modify your call to `simpleinfotheory.mutualinformationempirical()` above to compute the mutual information between characters that are not consecutive but separated by a lag of 2 (i.e. with one character in between them)? Is there still a substantial relationship? Is this information solely contained in the earlier character or is it perhaps also included in the immediately previous character?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7dc60b50-fc49-40a8-9d9c-9afc67e081fc", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute the mutual information between characters separated by a lag of two:\n" + ] + }, + { + "cell_type": "markdown", + "id": "75e18172-0164-46e0-9be3-18970c294c30", + "metadata": {}, + "source": [ + "5. Can you see how this relationship changes over longer lags still? Plot the mutual information as a function of lag (up to say 10). At what point would you say there is no longer a relationship? We will discuss statistical approaches to answering that in the coming weeks." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aaf73d8c-a21a-4503-9281-b3bd8e7f670f", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute and plot the MI as a function of lag:\n" + ] + }, + { + "cell_type": "markdown", + "id": "7445b88c-d912-4976-9368-11478dbbcf74", + "metadata": {}, + "source": [ + "# 7. (Optional extension) Pointwise mutual information between successive letters in written English\n", + "\n", + "_Further challenge_ -- It would be interesting to inspect the **local or pointwise mutual information** between each possible pair of consecutive letters. (See Part 3 of the lecture)\n", + "\n", + "1. To do this, first note how `simpleinfotheory.jointentropyempirical()` returns the set of symbols and their probabilities, as well as the joint entropy value.\n", + "2. Now we will alter `simpleinfotheory.mutualinformationempirical()` to similarly retrieve and return all of the relevant probabilities for each consecutive character pair:\n", + " 1. Notice how the calls for the joint entropy, $Y$ entropy and $X$ entropy already retrieve these for us.\n", + " 2. Then alter the return statement so that all of these relevant values are returned: `return result, xySymbols, xyProbs, xSymbols, xProbs, ySymbols, yProbs` (this is already done in the solution code for `simpleinfotheory.py`).\n", + " 3. You will need to restart the kernel to reload the library. You'll also need to update the function calls above, since they're now returning a list. If you append `[0]` to them, such as `simpleinfotheory.mutualinformationempirical(...)[0]`, then this will just pick out the main `result` return variable for the above as desired.\n", + "3. Next, call `simpleinfotheory.mutualinformationempirical()` again as per step 2 of the previous exercise for lag 1, but this time storing all of these return values:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "539b82af-b94e-4ca1-9362-ba70078c30cb", + "metadata": {}, + "outputs": [], + "source": [ + "# Call the mutual information empirical again, this time storing all of the return values.\n" + ] + }, + { + "cell_type": "markdown", + "id": "9ec8b8b0-92ea-444b-9b02-aeb3c01ab679", + "metadata": {}, + "source": [ + "4. Now, we loop over all possible joint symbols and compute the pointwise mutual information -- fill in the line of the code marked with `???` to compute the pointwise MI and then run this code block:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3a37011-0238-47d0-85a3-b384573fa2c0", + "metadata": {}, + "outputs": [], + "source": [ + "pointwiseMIs = np.zeros((xSymbols.size, ySymbols.size)); # Create array to store the pointwise MI values for each possible character pair\n", + "for firstCharIndex in range(xSymbols.size):\n", + " firstChar = xSymbols[firstCharIndex];\n", + " probFirst = xProbs[firstCharIndex];\n", + " for secondCharIndex in range(ySymbols.size):\n", + " secondChar = ySymbols[secondCharIndex];\n", + " probSecond = yProbs[secondCharIndex];\n", + " jointSymbolIndex = np.argwhere((xySymbols[:,0] == firstChar) & (xySymbols[:,1] == secondChar));\n", + " if (jointSymbolIndex.size == 0):\n", + " pointwiseMIs[firstCharIndex, secondCharIndex] = 0; # No occurence, so set to 0\n", + " continue;\n", + " probJoint = xyProbs[jointSymbolIndex];\n", + " # Compute the pointwise MI from probJoint, probFirst and probSecond\n", + " pointwiseMIs[firstCharIndex, secondCharIndex] = np.log2( ??? );" + ] + }, + { + "cell_type": "markdown", + "id": "e7f91288-9d24-4736-ae88-b8f3562d09a4", + "metadata": {}, + "source": [ + "5. Can you plot these values using `plt.imshow()`? Run the command `plt.colorbar()` to insert a colour bar to show the scale. The plot will have the first letters along the y axis, and second letters along the x axis. You can label these using:\n", + "\n", + " plt.xlabel('Second letter')\n", + " plt.xticks(ticks=range(0,27), labels=ySymbols.flatten())\n", + " plt.ylabel('First letter');\n", + " plt.yticks(ticks=range(0,27), labels=xSymbols.flatten())\n", + " cbar = plt.colorbar()\n", + " cbar.set_label('MI (bits)');" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9a980f0-ad8a-4aea-9fbe-5d0a78282db8", + "metadata": {}, + "outputs": [], + "source": [ + "# Make the heatmap plot:\n", + "\n", + "# Add the labels pasting in the code from above:\n" + ] + }, + { + "cell_type": "markdown", + "id": "37b50b62-6af9-4b66-84c3-b2c180ad9d28", + "metadata": {}, + "source": [ + "6. Examine the values and determine whether you can identify character pairs where the second is highly predictable from the first, and where the first character is misinformative about the second. Can you explain these results?" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise_Solution.ipynb b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise_Solution.ipynb new file mode 100644 index 0000000..203ca9a --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/miAsFunctionOfLagAndPointwise_Solution.ipynb @@ -0,0 +1,355 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "87e17b68-7539-4539-aeab-9df6fc819f22", + "metadata": {}, + "source": [ + "# Mutual information of written English text\n", + "\n", + "Author: J. Lizier, Isabelle De Backer, 2022-; based on the original Matlab tutorials.\n", + "\n", + "The following block aims to import all the relevant libraries to analyse data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "55ca6967-2ca7-45e8-9858-6bb0356c2bee", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import math\n", + "\n", + "# Specifics required for the text processing here:\n", + "import string\n", + "import re" + ] + }, + { + "cell_type": "markdown", + "id": "bc5119d3-c9d3-4139-ad43-3346098ea85d", + "metadata": {}, + "source": [ + "# Preparing your environment\n", + "\n", + "As per `Module_2_notebook.ipynb` etc. we need to use the functions we have defined in our previous work in other notebooks. So gather the new functions you wrote in this module into your `simpleinfotheory.py` script, and make sure it is referencable from here (you may need to change the folder referenced below) before you run the import line in the next cell:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8df5b524-117e-4a02-989e-9647b336dcc5", + "metadata": {}, + "outputs": [], + "source": [ + "# Option 3: edit simpleinfotheory.py and past your functions into that as you write them\n", + "import sys\n", + "sys.path.append('../../Module1-IntroToInfoTheory/PythonCode/completed/')\n", + "import simpleinfotheory" + ] + }, + { + "cell_type": "markdown", + "id": "4ecca927-eb6b-4840-8ea5-d4ea2bf79869", + "metadata": {}, + "source": [ + "# 6. (Optional extension) Mutual information between successive letters in written English\n", + "\n", + "In this extension activity, we will continue our analysis of written English extracted from the [Seinfeld](https://en.wikipedia.org/wiki/Seinfeld) scripts as begun in the previous module.\n", + "\n", + "1. Download the scripts from the links on Module 2 on Canvas, load into Python and preprocess as per steps 1-4 of the activity from the previous module, such that we have the characters stored in the numpy array `processedStr`:
\n", + "_Note:_ you may need to alter the filename/path to match your own --" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "0cebd6af-93fe-4377-9bdf-adfc995bcd10", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array([' ', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l',\n", + " 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y',\n", + " 'z'], dtype='\\?@\\[\\]\\\\\\^_`{\\|}~0-9]*');\n", + "processedStr = p.sub('', str); # Remove punctuation characters and digits\n", + "processedStr = ' '.join(processedStr.split('\\n')); # Replace newline characters with spaces\n", + "processedStr = processedStr.lower(); # Convert all upper case into lower case\n", + "processedStr = np.array(list(processedStr)); # Finally convert this into a numpy array so we can work with it\n", + "np.unique(processedStr)" + ] + }, + { + "cell_type": "markdown", + "id": "4e6b1195-3c3b-4570-ac49-742810f94025", + "metadata": {}, + "source": [ + "2. How can we now compute the mutual information between one character and the character that comes next in the text? We will need to provide samples of a previous character and the next character to our `simpleinfotheory.mutualinformationempirical()` function.
\n", + " _Hint_: to select all but the last item in a numpy array `x`, you can refer to `x[:-1]`, whilst to select all but the first item in an array `x`, you can refer to `x[1:]`" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "f1225213-86e2-49f3-9ff9-dfe0f6d34876", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.7196084319029961" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Compute the mutual information between successive characters:\n", + "simpleinfotheory.mutualinformationempirical(processedStr[:-1],processedStr[1:])[0]" + ] + }, + { + "cell_type": "markdown", + "id": "ab60c5ec-592a-4e15-97c7-280f7ce91253", + "metadata": {}, + "source": [ + "3. Compare the mutual information that you computed above to the average entropy of each character computed as per step 5 of the activity in the previous module. Consider the following:\n", + " 1. What proportion of our uncertainty about the next character in the written text is reduced by observing the previous character?\n", + " 1. How much code could we save in communicating a character if our coding scheme took the previous character into account?\n", + " 1. The mutual information computes a measure of the relationship between the consecutive characters here. You're probably familiar with using correlation to measure a relationship between variables -- could correlation be used here? We will see more about how MI and correlation are related in the coming weeks.\n", + "4. Are there relationships between previous characters and later characters beyond those which are consecutive?
\n", + " Can you modify your call to `simpleinfotheory.mutualinformationempirical()` above to compute the mutual information between characters that are not consecutive but separated by a lag of 2 (i.e. with one character in between them)? Is there still a substantial relationship? Is this information solely contained in the earlier character or is it perhaps also included in the immediately previous character?" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "7dc60b50-fc49-40a8-9d9c-9afc67e081fc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.3280241319399746" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Compute the mutual information between characters separated by a lag of two:\n", + "simpleinfotheory.mutualinformationempirical(processedStr[:-2],processedStr[2:])[0]" + ] + }, + { + "cell_type": "markdown", + "id": "75e18172-0164-46e0-9be3-18970c294c30", + "metadata": {}, + "source": [ + "5. Can you see how this relationship changes over longer lags still? Plot the mutual information as a function of lag (up to say 10). At what point would you say there is no longer a relationship? We will discuss statistical approaches to answering that in the coming weeks." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "aaf73d8c-a21a-4503-9281-b3bd8e7f670f", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHFCAYAAAAOmtghAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABIjklEQVR4nO3deVyU5f7/8feAsqiAoYEbIphHTVzxaEguaZLrOVamaeFe2mKZ1imzc1KPhmZ5LBXNjktapi2eSnPDXBPN5Wh2tPXnAhqkYgJaoeD1+4Mvk+OADgYO3Lyej8c8aq657ns+Mzdzz9vrvu57bMYYIwAAAIvwcHcBAAAARYlwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVw839ef/112Ww2RUREuLuUEqdDhw6y2WwKDw9Xfhe03rp1q2w2m2w2mxYtWmRvX7RokWw2m/bs2XPV9bvarzASExM1fvx4nT17tsjWWRrVqVNHPXr0cHcZ1yU+Pt7h7wk3xi+//KLx48dr8+bNRb7uzZs3y2azXXPd48ePl81m0+nTp4u8hsstXbpUM2bMcGo/evSobDabXnnllWJ9/sLIe09Ksjp16mjQoEHuLkMS4cZuwYIFkqSDBw/qiy++cHM1JY+fn5+OHDmijRs3Oj22YMEC+fv7u6GqgiUmJmrChAllPtyUZoQb9/jll180YcKEYgk3JU1B4aYkGjZsmHbs2OHuMkoNwo2kPXv26Msvv1T37t0lSfPnz7/hNRhj9Ouvv97w53VV7dq1ddttt9lDYJ7MzEy9//776tu3r5sqg7vl5OQoKyvL3WW4pKR/zorDxYsXlZ2d7e4y8AfVqlVLt912m7vLKDUIN/o9zEyZMkVt2rTRsmXL9Msvv0jK3TEEBQUpNjbWabmzZ8/K19dXo0ePtrdlZGTo6aefVlhYmLy8vFSzZk2NGjVK58+fd1jWZrPp8ccf19y5c9WwYUN5e3vrrbfekiRNmDBBrVu3VmBgoPz9/dWiRQvNnz/f6ZBQVlaWxowZo2rVqqlChQpq166d9u7dm+/QYGpqqoYPH65atWrJy8tLYWFhmjBhQqF2ekOGDNGKFSscRkOWLVsmSbr//vtdXk9Bfv75Zw0ePFiBgYGqWLGievbsqcOHDzv127Bhgzp16iR/f39VqFBB0dHR+uyzz+yPjx8/Xs8884wkKSwszH7IbPPmzXrmmWcUEBCgnJwce/+RI0fKZrNp2rRp9ra0tDR5eHho5syZ9jZXt60xRvHx8WrWrJl8fX110003qXfv3k6vpUOHDoqIiNDu3bvVtm1bVahQQeHh4ZoyZYouXbp0zffr0qVLmjlzpv15KleurNtuu02ffPKJU9+1a9eqRYsW8vX1VYMGDZxC6qlTp/Too4/q1ltvVaVKlRQUFKSOHTtq27ZtDv3yhutffvllTZo0SWFhYfL29tamTZv022+/acyYMWrWrJkCAgIUGBioqKgoffzxx4WuvU6dOjp48KC2bNli33516tQp9La42udszpw5atq0qSpVqiQ/Pz81aNBAzz///DXfd1eWc+Xzdvl7OXnyZNWuXVs+Pj5q2bKlw9+zJP3www8aPHiw6tWrpwoVKqhmzZrq2bOnvvrqK4d+eYd9lixZojFjxqhmzZry9vbWDz/84NI2Pnr0qG6++WZJufuhvPf+8v3J999/r/79+ysoKEje3t5q2LChZs+e7fQ+ffPNN+rSpYsqVKigqlWrasSIEcrMzLzm+3u55ORk3XPPPfL391dAQIAefPBBnTp1yv740KFDFRgYaN9fX65jx45q1KhRgevu0KGDPv30Ux07dsz+OvM77DN9+nSFhYWpUqVKioqK0s6dO5367NmzR3/5y18UGBgoHx8fNW/eXO+9955Lr/H48ePq3bu3/Pz8VLlyZT3wwAPavXu302H+Kw9L9erVS6GhofnuK1q3bq0WLVrY79+ofdKVCrNPOHv2rH17VqpUSd27d9fhw4dls9k0fvz4Qj+3TBn3yy+/mICAAPPnP//ZGGPMv//9byPJLFq0yN7nqaeeMr6+viY9Pd1h2fj4eCPJHDhwwBhjzPnz502zZs1M1apVzfTp082GDRvMa6+9ZgICAkzHjh3NpUuX7MtKMjVr1jRNmjQxS5cuNRs3bjT/+9//jDHGDBo0yMyfP98kJCSYhIQE889//tP4+vqaCRMmODx/v379jIeHh3nuuefM+vXrzYwZM0xISIgJCAgwAwcOtPdLSUkxISEhJjQ01Lzxxhtmw4YN5p///Kfx9vY2gwYNuuZ71L59e9OoUSOTkZFhKlasaOLj4+2PtW7d2gwYMMDs3r3bSDILFy60P7Zw4UIjyezevfuq68/rFxISYoYMGWLWrFlj5s2bZ4KCgkxISIj5+eef7X2XLFlibDab6dWrl1mxYoVZuXKl6dGjh/H09DQbNmwwxhiTnJxsRo4caSSZFStWmB07dpgdO3aY9PR0s3btWiPJJCYm2tfZoEED4+vrazp37mxvW758uZFkDh06ZIwp3LZ96KGHTPny5c2YMWPM2rVrzdKlS02DBg1McHCwSU1NdXhfq1SpYurVq2fmzp1rEhISzKOPPmokmbfeeuua2yU2NtbYbDYzbNgw8/HHH5s1a9aYyZMnm9dee83eJzQ01NSqVcvceuutZvHixWbdunXmvvvuM5LMli1b7P2++eYb88gjj5hly5aZzZs3m1WrVpmhQ4caDw8Ps2nTJnu/I0eO2P9277jjDvPBBx+Y9evXmyNHjpizZ8+aQYMGmSVLlpiNGzeatWvXmqefftp4eHg4vZ5r1f7f//7XhIeHm+bNm9u333//+99Cb4uCPmfvvvuukWRGjhxp1q9fbzZs2GDmzp1rnnjiiau+564s5+rnLe+9DAkJMbfffrv58MMPzfvvv2/+/Oc/m/Llyzv8jW7ZssWMGTPGfPDBB2bLli3mP//5j+nVq5fx9fU133zzjb3fpk2b7K+5d+/e5pNPPjGrVq0yaWlpLm3j3377zf4ZGTp0qP29/+GHH4wxxhw8eNAEBASYxo0bm8WLF5v169ebMWPGGA8PDzN+/Hh7HampqSYoKMjUrFnTLFy40Kxevdo88MADpnbt2kaSw99Ufl588UUjyYSGhppnnnnGrFu3zkyfPt1UrFjRNG/e3Fy4cMEYY8yXX35pJJk333zTYfmDBw8aSWb27NkFPsfBgwdNdHS0qVatmv117tixw2Hb1KlTx3Tp0sV89NFH5qOPPjKNGzc2N910kzl79qx9PRs3bjReXl6mbdu2Zvny5Wbt2rVm0KBBTvvD/Jw7d87ccsstJjAw0MyePdusW7fOPPXUUyYsLMxp+bz3JM/HH39sJJmEhASHdX799ddGknn99dftbTdqnxQaGurw3ePqPiEnJ8fcfvvtxsfHx0yZMsWsX7/eTJgwwdSrV89IMi+++OI1n/tKZT7cLF682Egyc+fONcYYk5mZaSpVqmTatm1r73PgwAEjycybN89h2VatWpnIyEj7/bi4OOPh4eH0Zf7BBx8YSWb16tX2NkkmICDAnDlz5qr15eTkmIsXL5qJEyeaKlWq2HfceR/eZ5991qF/3s738j+w4cOHm0qVKpljx4459H3llVeMJHPw4MGr1pAXbowxZuDAgaZly5YONWzevLlIws3dd9/t0L59+3YjyUyaNMkYk/ulFhgYaHr27On0HjVt2tS0atXK3jZt2jQjyRw5csSh7/nz542Xl5eZOHGiMcaY48eP299HX19f89tvvxljcncGNWrUsC/n6rbdsWOHkWReffVVh37JycnG19fX/O1vf7O3tW/f3kgyX3zxhUPfW2+91dx1111Xfc+2bt1qJJlx48ZdtV9oaKjx8fFx2Pa//vqrCQwMNMOHDy9wuezsbHPx4kXTqVMnh+2St9OvW7eu/QvmWusYOnSoad68eaFrb9SokWnfvr1Te1F8zh5//HFTuXLlqz5/flxZztXPW957WaNGDfPrr7/a+2VkZJjAwEBz5513Fvgc2dnZ5sKFC6ZevXrmqaeesrfnhZt27dpd87UUtI1PnTpV4BfKXXfdZWrVquX0D73HH3/c+Pj42N/nZ5991thsNrN//36Hfp07dy5UuLn8tRljzDvvvGMkmbffftve1r59e9OsWTOHfo888ojx9/c3mZmZV32e7t27m9DQUKf2vG3TuHFjk52dbW/ftWuXkWTeffdde1uDBg1M8+bNzcWLFx3W0aNHD1O9enWTk5NT4PPPnj3bSDJr1qxxaB8+fPg1w83FixdNcHCw6d+/v8Oyf/vb34yXl5c5ffq0MebG7ZOMcQ43Vypon/Dpp58aSWbOnDkO/ePi4q473JT5w1Lz58+Xr6+v/bBKpUqVdN9992nbtm36/vvvJUmNGzdWZGSkFi5caF/u66+/1q5duzRkyBB726pVqxQREaFmzZopOzvbfrvrrrvyPUOgY8eOuummm5xq2rhxo+68804FBATI09NT5cuX1z/+8Q+lpaXp5MmTkqQtW7ZIkvr06eOwbO/evVWuXDmHtlWrVumOO+5QjRo1HOrq2rWrw7pcMWTIEO3Zs0dfffWV5s+fr7p166pdu3YuL381DzzwgMP9Nm3aKDQ0VJs2bZKUO0n4zJkzGjhwoMPruHTpkrp06aLdu3c7HZa4UoUKFRQVFaUNGzZIkhISElS5cmU988wzunDhgj7//HNJuYe+7rzzTvtyrm7bVatWyWaz6cEHH3ToV61aNTVt2tTpb6BatWpq1aqVQ1uTJk107Nixq76ONWvWSJIee+yxq/aTpGbNmql27dr2+z4+PvrTn/7k9Bxz585VixYt5OPjo3Llyql8+fL67LPP9PXXXzut8y9/+YvKly/v1P7+++8rOjpalSpVsq9j/vz5DusoTO35KYrPWatWrXT27Fn169dPH3/8sctn5biyXGE/b/fcc498fHzs9/38/NSzZ09t3brVfvg0OztbL730km699VZ5eXmpXLly8vLy0vfff5/v9rn33nvzrb8w2/hKv/32mz777DPdfffdqlChgsNr69atm3777Tf7IZtNmzapUaNGatq0qcM6+vfvf83nudyV+4Q+ffqoXLly9n2CJD355JPav3+/tm/fLin3kOWSJUs0cOBAVapUqVDPd6Xu3bvL09PTfr9JkyaSZP/s/PDDD/rmm2/sdV75nqSkpOjbb78tcP1btmyRn5+funTp4tDer1+/a9ZWrlw5Pfjgg1qxYoXS09Ml5c5/W7Jkif7617+qSpUqkm7cPqkgruwTCvo+c+V9KEiZDjc//PCDtm7dqu7du8sYo7Nnz+rs2bPq3bu3JDnMSxgyZIh27Nihb775RpK0cOFCeXt7O7z5P/30kw4cOKDy5cs73Pz8/GSMcdoRVq9e3ammXbt2KSYmRpL05ptvavv27dq9e7fGjRsnSfbJkGlpaZKk4OBgh+XLlStn/6O+vK6VK1c61ZV3PLowp1u2a9dO9erV0xtvvKElS5ZoyJAhRXZ6YrVq1fJty3utP/30k6TcAHfla5k6daqMMTpz5sw1n+fOO+/Uzp07df78eW3YsEEdO3ZUlSpVFBkZqQ0bNujIkSM6cuSIQ7hxddv+9NNPMsYoODjYqe/OnTud3usrt5UkeXt7X3PS66lTp+Tp6Znve3YlV55j+vTpeuSRR9S6dWt9+OGH2rlzp3bv3q0uXbrkW0t+f7srVqxQnz59VLNmTb399tvasWOHdu/erSFDhui33367rtrzUxSfs9jYWC1YsEDHjh3Tvffeq6CgILVu3VoJCQlXfW5Xlivs562gv/sLFy7o3LlzkqTRo0fr73//u3r16qWVK1fqiy++0O7du9W0aVOXt09ht/GV0tLSlJ2drZkzZzq9tm7dujm8trS0tAJfV2Fc2T9v/5a3T5Ckv/71r6pTp4593s+iRYt0/vz56w7Pl7vys+Pt7S3p9/1w3j7p6aefdnpPHn30UUlX37+mpaU57cMl5/16QfI+W3lzH9etW6eUlBQNHjzY3udG7ZPy4+o+IS0tTeXKlVNgYKDD8q6+D/kpd+0u1rVgwQIZY/TBBx/ogw8+cHr8rbfe0qRJk+Tp6al+/fpp9OjRWrRokSZPnqwlS5aoV69eDv8irFq1qnx9fZ0ma17++OXyCwXLli1T+fLltWrVKod/zX300UcO/fL+AH/66SfVrFnT3p6dne3wwc973iZNmmjy5Mn51lWjRo182wsyePBgvfDCC7LZbBo4cGChlr2a1NTUfNtuueUWSb+/fzNnzizwrAFXPgydOnXS3//+d23dulWfffaZXnzxRXv7+vXrFRYWZr+fx9VtW7VqVdlsNm3bts2+I7xcfm3X4+abb1ZOTo5SU1Pz/SIrrLffflsdOnTQnDlzHNoLmgCa39/u22+/rbCwMC1fvtzh8SvPpPqjtRfF50zK/TsePHiwzp8/r61bt+rFF19Ujx499N133yk0NLTA57/WcoX9vBX0d+/l5WUfeXj77bc1YMAAvfTSSw79Tp8+rcqVKzstX9D2Kcw2vtJNN90kT09PxcbGFhgc8j47VapUKfB1FUZqamq++7fLv4A9PDz02GOP6fnnn9err76q+Ph4derUSfXr1y/Uc12PvL+1sWPH6p577sm3z9XqqFKlinbt2uXU7ur7dOutt6pVq1ZauHChhg8froULF6pGjRr2fyDn1Xgj9kn5cXWfUKVKFWVnZ+vMmTMOAaewfy+XK7PhJicnR2+99Zbq1q2rf//7306Pr1q1Sq+++qrWrFmjHj166KabblKvXr20ePFiRUVFKTU11eGQlCT16NFDL730kqpUqWL/kBeWzWZTuXLlHIZCf/31Vy1ZssShX96hoOXLlzvMiv/ggw+czoDq0aOHVq9erbp16+Z7GKywBg4cqC+++EINGzZ02PH8Ue+8847DcHpiYqKOHTumYcOGSZKio6NVuXJlHTp0SI8//vhV13Xlv7Au16pVK/n7+2vGjBlKTU1V586dJeWO6EydOlXvvfeebr31VocvIVe3bY8ePTRlyhSdOHHCaYi1KHXt2lVxcXGaM2eOJk6c+IfXZ7PZnHZyBw4c0I4dOxQSEuLyOry8vBx2YqmpqU5nRrhae0H/WiyKz9nlKlasqK5du+rChQvq1auXDh48eNVwc63lCvt5W7FihaZNm2b/x0xmZqZWrlyptm3b2vcD+W2fTz/9VCdOnLCH/2txdRsX9NmpUKGC7rjjDu3bt09NmjSRl5dXgc91xx136OWXX9aXX37pcGhq6dKlLtWa55133lFkZKT9/nvvvafs7Gx16NDBod+wYcM0fvx4PfDAA/r22281depUl9Z/vSMSeerXr6969erpyy+/dAqermjfvr3ee+89rVmzxn7YUvr9LFRXDB48WI888og+//xzrVy5UqNHj3b4/rhR+6T8uLpPaN++vV5++WUtX75cjzzyiL29MO/DlcpsuFmzZo1+/PFHTZ061emDIkkRERGaNWuW5s+fb7/C65AhQ7R8+XI9/vjjqlWrlsNhC0kaNWqUPvzwQ7Vr105PPfWUmjRpokuXLikpKUnr16/XmDFj1Lp166vW1b17d02fPl39+/fXww8/rLS0NL3yyitOO6VGjRqpX79+evXVV+Xp6amOHTvq4MGDevXVVxUQECAPj9+POE6cOFEJCQlq06aNnnjiCdWvX1+//fabjh49qtWrV2vu3LmqVauWy+9djRo1nEaSisKePXs0bNgw3XfffUpOTta4ceNUs2ZN+/BupUqVNHPmTA0cOFBnzpxR7969FRQUpFOnTunLL7/UqVOn7P8qbdy4sSTptdde08CBA1W+fHnVr19ffn5+8vT0VPv27bVy5UqFhYWpbt26knLDk7e3tz777DM98cQTDrW5um2jo6P18MMPa/DgwdqzZ4/atWunihUrKiUlRZ9//rkaN27s8OG9Xm3btlVsbKwmTZqkn376ST169JC3t7f27dunChUqaOTIkYVaX48ePfTPf/5TL774otq3b69vv/1WEydOVFhYmMuXC+jRo4dWrFihRx99VL1791ZycrL++c9/qnr16vb5a4WpvXHjxlq2bJmWL1+u8PBw+fj4qHHjxkXyOXvooYfk6+ur6OhoVa9eXampqYqLi1NAQID+/Oc//6HlCvt58/T0VOfOnTV69GhdunRJU6dOVUZGhiZMmODw3i5atEgNGjRQkyZNtHfvXk2bNq1Qn1tXt7Gfn59CQ0P18ccfq1OnTgoMDFTVqlVVp04dvfbaa7r99tvVtm1bPfLII6pTp44yMzP1ww8/aOXKlfaLfI4aNUoLFixQ9+7dNWnSJAUHB+udd96xH9Z31YoVK1SuXDl17txZBw8e1N///nc1bdrU6Uu6cuXKGjBggObMmaPQ0FD17NnTpfU3btxYK1as0Jw5cxQZGSkPDw+1bNmyUDW+8cYb6tq1q+666y4NGjRINWvW1JkzZ/T111/rv//9r95///0Clx04cKD+9a9/6cEHH9SkSZN0yy23aM2aNVq3bp0kOezHC5J3VKFfv37KyspyugzIjdon5cfVfUKXLl0UHR2tMWPGKCMjQ5GRkdqxY4cWL14sybX3wUmhpyBbRK9evYyXl5c5efJkgX3uv/9+U65cOfupcjk5OSYkJOSqZ3qcO3fOvPDCC6Z+/frGy8vLftrkU0895XDKnSTz2GOP5buOBQsWmPr16xtvb28THh5u4uLizPz5853O/vntt9/M6NGjTVBQkPHx8TG33Xab2bFjhwkICHA6y+DUqVPmiSeeMGFhYaZ8+fImMDDQREZGmnHjxplz585d9b26/GypghTF2VLr1683sbGxpnLlysbX19d069bNfP/99079t2zZYrp3724CAwNN+fLlTc2aNU337t3N+++/79Bv7NixpkaNGsbDw8PpDI3XXnvNSDIPPfSQwzJ5Z3N88sknTs/r6rY1Jncbtm7d2lSsWNH4+vqaunXrmgEDBpg9e/bY+xT0vg4cODDfMziulJOTY/71r3+ZiIgIez1RUVFm5cqV9j6hoaGme/fuTsu2b9/e4UykrKws8/TTT5uaNWsaHx8f06JFC/PRRx851ZJ3Fsm0adPyrWnKlCmmTp06xtvb2zRs2NC8+eabTmd5uFr70aNHTUxMjPHz87OfFpznj37O3nrrLXPHHXeY4OBg4+XlZWrUqGH69Oljv6xDQVxdzpXPW957OXXqVDNhwgRTq1Yt4+XlZZo3b27WrVvnsL6ff/7ZDB061AQFBZkKFSqY22+/3Wzbts1pO+adLXXlZ8EY17exMcZs2LDBNG/e3Hh7ezudfXnkyBEzZMgQU7NmTVO+fHlz8803mzZt2tjPasxz6NAh07lzZ+Pj42MCAwPN0KFD7acvu3q21N69e03Pnj1NpUqVjJ+fn+nXr5/56aef8l1m8+bNRpKZMmXKVdd9uTNnzpjevXubypUrG5vNZv87vdrfufI5e+fLL780ffr0MUFBQaZ8+fKmWrVqpmPHjvazcK8mKSnJ3HPPPfbXeO+995rVq1cbSebjjz92ek/y079/fyPJREdHF/g8N2KflN/ZUq7uE86cOWMGDx5sKleubCpUqGA6d+5sdu7caSQ5XN7CVTZj8vmxIJRaiYmJio6O1jvvvFPoMxMA3DhHjx5VWFiYpk2bpqefftrd5ZR6Y8aM0Zw5c5ScnJzvpNjS5KWXXtILL7ygpKSkQo3OWc3SpUv1wAMPaPv27WrTpk2hli2zh6WsICEhQTt27FBkZKR8fX315ZdfasqUKapXr16Bk9sAwEp27typ7777TvHx8Ro+fHipCzazZs2SJDVo0EAXL17Uxo0b9frrr+vBBx8sU8Hm3Xff1YkTJ9S4cWN5eHho586dmjZtmtq1a1foYCMRbko1f39/rV+/XjNmzFBmZqaqVq1qn6x5+ZlWAGBVUVFRqlChgnr06KFJkya5u5xCq1Chgv71r3/p6NGjysrKUu3atfXss8/qhRdecHdpN5Sfn5+WLVumSZMm6fz586pevboGDRp03duUw1IAAMBSyvRF/AAAgPUQbgAAgKUQbgAAgKWUuQnFly5d0o8//ig/P78i+00kAABQvIwxyszMVI0aNa55Yb8yF25+/PFHly8nDwAASpbk5ORrniZf5sKNn5+fpNw3x9/f383VAAAAV2RkZCgkJMT+PX41ZS7c5B2K8vf3J9wAAFDKuDKlhAnFAADAUgg3AADAUgg3AADAUgg3AADAUgg3AADAUgg3AADAUgg3AADAUgg3AADAUgg3AADAUgg3f1R6unT8eP6PHT+e+zgAALhhCDd/RHq61KWL1L69lJzs+Fhycm57ly4EHAAAbiDCzR+RmSmdPCkdPix16PB7wElOzr1/+HDu45mZ7qwSAIAyhXDzR9SqJW3eLIWH/x5wEhN/Dzbh4bmPX+On2QEAQNEpc78KXuRCQnIDTF6giY7Obc8LNiEhbiwOAICyh5GbohASIi1Z4ti2ZAnBBgAANyDcFIXkZCk21rEtNtZ5kjEAACh2hJs/6vLJw+Hh0vbtjnNwCDgAANxQhJs/4vhx58nDbdo4TzIu6Do4AACgyDGh+I/w85OCgnL///LJw5dPMg4Kyu0HAABuCLeP3MTHxyssLEw+Pj6KjIzUtm3bCuw7aNAg2Ww2p1ujRo1uYMWXCQiQ1q6VtmxxnjwcEpLbvnZtbj8AAHBDuDXcLF++XKNGjdK4ceO0b98+tW3bVl27dlVSUlK+/V977TWlpKTYb8nJyQoMDNR99913gyu/TEBAwdexqVWLYAMAwA1mM8YYdz1569at1aJFC82ZM8fe1rBhQ/Xq1UtxcXHXXP6jjz7SPffcoyNHjig0NNSl58zIyFBAQIDS09Pl7+9/3bUDAIAbpzDf324bublw4YL27t2rmJgYh/aYmBglJia6tI758+frzjvvvGqwycrKUkZGhsMNAABYl9vCzenTp5WTk6Pg4GCH9uDgYKWmpl5z+ZSUFK1Zs0bDhg27ar+4uDgFBATYbyFcWA8AAEtz+4Rim83mcN8Y49SWn0WLFqly5crq1avXVfuNHTtW6enp9lsy150BAMDS3HYqeNWqVeXp6ek0SnPy5Emn0ZwrGWO0YMECxcbGysvL66p9vb295e3t/YfrBQAApYPbRm68vLwUGRmphIQEh/aEhAS1adPmqstu2bJFP/zwg4YOHVqcJQIAgFLIrRfxGz16tGJjY9WyZUtFRUVp3rx5SkpK0ogRIyTlHlI6ceKEFi9e7LDc/Pnz1bp1a0VERLijbAAAUIK5Ndz07dtXaWlpmjhxolJSUhQREaHVq1fbz35KSUlxuuZNenq6PvzwQ7322mvuKBkAAJRwbr3OjTtwnRsAAEqfUnGdGwAAgOJAuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJbi9nATHx+vsLAw+fj4KDIyUtu2bbtq/6ysLI0bN06hoaHy9vZW3bp1tWDBghtULQAAKOnKufPJly9frlGjRik+Pl7R0dF644031LVrVx06dEi1a9fOd5k+ffrop59+0vz583XLLbfo5MmTys7OvsGVAwCAkspmjDHuevLWrVurRYsWmjNnjr2tYcOG6tWrl+Li4pz6r127Vvfff78OHz6swMDA63rOjIwMBQQEKD09Xf7+/tddOwAAuHEK8/3ttsNSFy5c0N69exUTE+PQHhMTo8TExHyX+eSTT9SyZUu9/PLLqlmzpv70pz/p6aef1q+//lrg82RlZSkjI8PhBgAArMtth6VOnz6tnJwcBQcHO7QHBwcrNTU132UOHz6szz//XD4+PvrPf/6j06dP69FHH9WZM2cKnHcTFxenCRMmFHn9AACgZHL7hGKbzeZw3xjj1Jbn0qVLstlseuedd9SqVSt169ZN06dP16JFiwocvRk7dqzS09Ptt+Tk5CJ/DQAAoORw28hN1apV5enp6TRKc/LkSafRnDzVq1dXzZo1FRAQYG9r2LChjDE6fvy46tWr57SMt7e3vL29i7Z4AABQYrlt5MbLy0uRkZFKSEhwaE9ISFCbNm3yXSY6Olo//vijzp07Z2/77rvv5OHhoVq1ahVrvQAAoHRw62Gp0aNH69///rcWLFigr7/+Wk899ZSSkpI0YsQISbmHlAYMGGDv379/f1WpUkWDBw/WoUOHtHXrVj3zzDMaMmSIfH193fUyAABACeLW69z07dtXaWlpmjhxolJSUhQREaHVq1crNDRUkpSSkqKkpCR7/0qVKikhIUEjR45Uy5YtVaVKFfXp00eTJk1y10sAAAAljFuvc+MOXOcGAIDSp1Rc5wYAAKA4EG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAIClEG4AAICluD3cxMfHKywsTD4+PoqMjNS2bdsK7Lt582bZbDan2zfffHMDKwYAACWZW8PN8uXLNWrUKI0bN0779u1T27Zt1bVrVyUlJV11uW+//VYpKSn2W7169W5QxQAAoKRza7iZPn26hg4dqmHDhqlhw4aaMWOGQkJCNGfOnKsuFxQUpGrVqtlvnp6eN6hiAABQ0rkt3Fy4cEF79+5VTEyMQ3tMTIwSExOvumzz5s1VvXp1derUSZs2bSrOMgEAQClTzl1PfPr0aeXk5Cg4ONihPTg4WKmpqfkuU716dc2bN0+RkZHKysrSkiVL1KlTJ23evFnt2rXLd5msrCxlZWXZ72dkZBTdiwAAACWO28JNHpvN5nDfGOPUlqd+/fqqX7++/X5UVJSSk5P1yiuvFBhu4uLiNGHChKIrGAAAlGhuOyxVtWpVeXp6Oo3SnDx50mk052puu+02ff/99wU+PnbsWKWnp9tvycnJ110zAAAo+dwWbry8vBQZGamEhASH9oSEBLVp08bl9ezbt0/Vq1cv8HFvb2/5+/s73AAAgHW59bDU6NGjFRsbq5YtWyoqKkrz5s1TUlKSRowYISl31OXEiRNavHixJGnGjBmqU6eOGjVqpAsXLujtt9/Whx9+qA8//NCdLwMAAJQgbg03ffv2VVpamiZOnKiUlBRFRERo9erVCg0NlSSlpKQ4XPPmwoULevrpp3XixAn5+vqqUaNG+vTTT9WtWzd3vQQAAFDC2Iwxxt1F3EgZGRkKCAhQeno6h6gAACglCvP97fafXwAAAChKhBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAphBsAAGAp5a5noaysLO3atUtHjx7VL7/8optvvlnNmzdXWFhYUdcHAABQKIUKN4mJiZo5c6Y++ugjXbhwQZUrV5avr6/OnDmjrKwshYeH6+GHH9aIESPk5+dXXDUDAAAUyOXDUn/961/Vu3dv1axZU+vWrVNmZqbS0tJ0/Phx/fLLL/r+++/1wgsv6LPPPtOf/vQnJSQkFGfdAAAA+XJ55CYmJkbvv/++vLy88n08PDxc4eHhGjhwoA4ePKgff/yxyIoEAABwlcsjN4899liBweZKjRo1UufOnV3qGx8fr7CwMPn4+CgyMlLbtm1zabnt27erXLlyatasmUv9AQBA2XBdZ0slJyfr+PHj9vu7du3SqFGjNG/evEKtZ/ny5Ro1apTGjRunffv2qW3bturatauSkpKuulx6eroGDBigTp06XU/5AADAwq4r3PTv31+bNm2SJKWmpqpz587atWuXnn/+eU2cONHl9UyfPl1Dhw7VsGHD1LBhQ82YMUMhISGaM2fOVZcbPny4+vfvr6ioqOspHwAAWNh1hZv//e9/atWqlSTpvffeU0REhBITE7V06VItWrTIpXVcuHBBe/fuVUxMjEN7TEyMEhMTC1xu4cKF+n//7//pxRdfdOl5srKylJGR4XADAADWdV3h5uLFi/L29pYkbdiwQX/5y18kSQ0aNFBKSopL6zh9+rRycnIUHBzs0B4cHKzU1NR8l/n+++/13HPP6Z133lG5cq7NhY6Li1NAQID9FhIS4tJyAACgdLqucNOoUSPNnTtX27ZtU0JCgrp06SJJ+vHHH1WlSpVCrctmszncN8Y4tUlSTk6O+vfvrwkTJuhPf/qTy+sfO3as0tPT7bfk5ORC1QcAAEqX67pC8dSpU3X33Xdr2rRpGjhwoJo2bSpJ+uSTT+yHq66latWq8vT0dBqlOXnypNNojiRlZmZqz5492rdvnx5//HFJ0qVLl2SMUbly5bR+/Xp17NjRaTlvb2/7KBMAALC+6wo3HTp00OnTp5WRkaGbbrrJ3v7www+rYsWKLq3Dy8tLkZGRSkhI0N13321vT0hI0F//+len/v7+/vrqq68c2uLj47Vx40Z98MEH/PQDAACQdJ3hpmPHjlqxYoVDsJGkwMBA9erVSxs3bnRpPaNHj1ZsbKxatmypqKgozZs3T0lJSRoxYoSk3ENKJ06c0OLFi+Xh4aGIiAiH5YOCguTj4+PUDgAAyq7rCjebN2/WhQsXnNp/++03ly/CJ0l9+/ZVWlqaJk6cqJSUFEVERGj16tUKDQ2VJKWkpFzzmjcAAACXsxljjKudDxw4IElq1qyZNm7cqMDAQPtjOTk5Wrt2rd544w0dPXq0yAstKhkZGQoICFB6err8/f3dXQ4AAHBBYb6/CzVy06xZM9lsNtlstnwn7/r6+mrmzJmFqxYAAKAIFSrcHDlyRMYYhYeHa9euXbr55pvtj3l5eSkoKEienp5FXiQAAICrChVu8ubCXLp0qViKAQAA+KNcDjeffPKJunbtqvLly+uTTz65at+8KxYDAADcaC5PKPbw8FBqaqqCgoLk4VHwhY1tNptycnKKrMCixoRiAABKn2KZUHz5oSgOSwEAgJLqun5bCgAAoKS67nDz2WefqUePHqpbt65uueUW9ejRQxs2bCjK2gAAAArtusLNrFmz1KVLF/n5+enJJ5/UE088IX9/f3Xr1k2zZs0q6hoBAABcVqgrFOepWbOmxo4da/917jyzZ8/W5MmT9eOPPxZZgUWNCcUAAJQ+hfn+vq6Rm4yMDHXp0sWpPSYmRhkZGdezSgAAgCJxXeHmL3/5i/7zn/84tX/88cfq2bPnHy4KAADgerl8Kvjrr79u//+GDRtq8uTJ2rx5s6KioiRJO3fu1Pbt2zVmzJiirxIAAMBFLs+5CQsLc22FNpsOHz78h4oqTsy5AQCg9CmWi/gdOXLkDxcGAABQ3LiIHwAAsBSXw82UKVN0/vx5l/p+8cUX+vTTT6+7KAAAgOvlcrg5dOiQQkND9cgjj2jNmjU6deqU/bHs7GwdOHBA8fHxatOmje6//37mswAAALdwec7N4sWLdeDAAc2ePVsPPPCA0tPT5enpKW9vb/3yyy+SpObNm+vhhx/WwIED5e3tXWxFAwAAFOS6rlBsjNGBAwd09OhR/frrr6pataqaNWumqlWrFkeNRYqzpQAAKH2K5Wypy9lsNjVt2lRNmza9rgIBAACKC2dLAQAASyHcIFd6unT8eP6PHT+e+zgAAKUA4Qa5waVLF6l9eyk52fGx5OTc9i5dCDgAgFKBcAMpM1M6eVI6fFjq0OH3gJOcnHv/8OHcxzMz3VklAAAuIdxAqlVL2rxZCg//PeAkJv4ebMLDcx+vVcu9dQIA4IJCnS11zz33uNRvxYoV11UM3CgkJDfA5AWa6Ojc9rxgExLixuIAAHBdocJNQEBAcdWBkiAkRFqy5PdgI+XeJ9gAAEqR67qIX2nGRfyu4vI5NnkYuQEAlACF+f5mzg1yXR5swsOl7dsd5+BceRYVAAAlVKEOSw0ZMsSlfgsWLLiuYuAmx487Tx6+cg5Ohw7Sli1MKgYAlHiFCjeLFi1SaGiomjdvrjJ2NMva/PykoKDc/7/8ENTlAScoKLcfAAAlXKHCzYgRI7Rs2TIdPnxYQ4YM0YMPPqjAwMDiqg03SkCAtHZt7nVsrhyZCQnJHbHx88vtBwBACVeoOTfx8fFKSUnRs88+q5UrVyokJER9+vTRunXrGMkp7QICCj7kVKsWwQYAUGr8obOljh07pkWLFmnx4sW6ePGiDh06pEqVKhVlfUWOs6UAACh9btjZUjabTTabTcYYXbp06Y+sCgAAoEgUOtxkZWXp3XffVefOnVW/fn199dVXmjVrlpKSkq5r1CY+Pl5hYWHy8fFRZGSktm3bVmDfzz//XNHR0apSpYp8fX3VoEED/etf/yr0cwIAAOsq1ITiRx99VMuWLVPt2rU1ePBgLVu2TFWqVLnuJ1++fLlGjRql+Ph4RUdH64033lDXrl116NAh1a5d26l/xYoV9fjjj6tJkyaqWLGiPv/8cw0fPlwVK1bUww8/fN11AAAA6yjUnBsPDw/Vrl1bzZs3l81mK7Cfq78t1bp1a7Vo0UJz5syxtzVs2FC9evVSXFycS+u45557VLFiRS1ZssSl/sy5AQCg9CnM93ehRm4GDBhw1VBTGBcuXNDevXv13HPPObTHxMQoMTHRpXXs27dPiYmJmjRpUoF9srKylJWVZb+fkZFxfQUDAIBSodAX8Ssqp0+fVk5OjoKDgx3ag4ODlZqaetVla9WqpVOnTik7O1vjx4/XsGHDCuwbFxenCRMmFEnNAACg5HP7b0tdORJkjLnm6NC2bdu0Z88ezZ07VzNmzNC7775bYN+xY8cqPT3dfkvmN5IAALC0Qo3cFKWqVavK09PTaZTm5MmTTqM5VwoLC5MkNW7cWD/99JPGjx+vfv365dvX29tb3t7eRVM0AAAo8dw2cuPl5aXIyEglJCQ4tCckJKhNmzYur8cY4zCnBgAAlG1uG7mRpNGjRys2NlYtW7ZUVFSU5s2bp6SkJI0YMUJS7iGlEydOaPHixZKk2bNnq3bt2mrQoIGk3OvevPLKKxo5cqTbXgMAAChZ3Bpu+vbtq7S0NE2cOFEpKSmKiIjQ6tWrFRoaKklKSUlRUlKSvf+lS5c0duxYHTlyROXKlVPdunU1ZcoUDR8+3F0vAQAAlDB/6LelSiOucwMAQOlzw35bCgAAoKQh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEsh3AAAAEtxe7iJj49XWFiYfHx8FBkZqW3bthXYd8WKFercubNuvvlm+fv7KyoqSuvWrbuB1QIAgJLOreFm+fLlGjVqlMaNG6d9+/apbdu26tq1q5KSkvLtv3XrVnXu3FmrV6/W3r17dccdd6hnz57at2/fDa4cAACUVDZjjHHXk7du3VotWrTQnDlz7G0NGzZUr169FBcX59I6GjVqpL59++of//iHS/0zMjIUEBCg9PR0+fv7X1fdAADgxirM97fbRm4uXLigvXv3KiYmxqE9JiZGiYmJLq3j0qVLyszMVGBgYIF9srKylJGR4XADAADW5bZwc/r0aeXk5Cg4ONihPTg4WKmpqS6t49VXX9X58+fVp0+fAvvExcUpICDAfgsJCflDdQMAgJLN7ROKbTabw31jjFNbft59912NHz9ey5cvV1BQUIH9xo4dq/T0dPstOTn5D9cMAABKrnLueuKqVavK09PTaZTm5MmTTqM5V1q+fLmGDh2q999/X3feeedV+3p7e8vb2/sP1wsAAEoHt43ceHl5KTIyUgkJCQ7tCQkJatOmTYHLvfvuuxo0aJCWLl2q7t27F3eZAACglHHbyI0kjR49WrGxsWrZsqWioqI0b948JSUlacSIEZJyDymdOHFCixcvlpQbbAYMGKDXXntNt912m33Ux9fXVwEBAW57HQAAoORwa7jp27ev0tLSNHHiRKWkpCgiIkKrV69WaGioJCklJcXhmjdvvPGGsrOz9dhjj+mxxx6ztw8cOFCLFi260eUDAIASyK3XuXEHrnMDAEDpUyqucwMAAFAcCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDcAAMBSCDewlvR06fjx/B87fjz3cQCApRFuYB3p6VKXLlL79lJysuNjycm57V26EHAAwOIIN7COzEzp5Enp8GGpQ4ffA05ycu79w4dzH8/MdGeVAIBiRriBddSqJW3eLIWH/x5wEhN/Dzbh4bmP16rl3joBAMWqnLsLAIpUSEhugMkLNNHRue15wSYkxI3FAQBuBEZuYD0hIdKSJY5tS5YQbACgjCDcwHqSk6XYWMe22FjnScYAAEsi3MBaLp88HB4ubd/uOAeHgAMAlke4gXUcP+48ebhNG+dJxgVdBwcAYAluDzfx8fEKCwuTj4+PIiMjtW3btgL7pqSkqH///qpfv748PDw0atSoG1coSj4/PykoyHnycN4k4/Dw3Mf9/NxZJQCgmLk13CxfvlyjRo3SuHHjtG/fPrVt21Zdu3ZVUlJSvv2zsrJ08803a9y4cWratOkNrhYlXkCAtHattGWL8+ThkJDc9rVrc/sBACzLZowx7nry1q1bq0WLFpozZ469rWHDhurVq5fi4uKuumyHDh3UrFkzzZgxo1DPmZGRoYCAAKWnp8vf3/96ygYAADdYYb6/3TZyc+HCBe3du1cxMTEO7TExMUpMTCyy58nKylJGRobDDQAAWJfbws3p06eVk5Oj4OBgh/bg4GClpqYW2fPExcUpICDAfgvhWicAAFia2ycU22w2h/vGGKe2P2Ls2LFKT0+335I5FRgAAEtz288vVK1aVZ6enk6jNCdPnnQazfkjvL295e3tXWTrAwAAJZvbRm68vLwUGRmphIQEh/aEhAS1adPGTVUBAIDSzq0/nDl69GjFxsaqZcuWioqK0rx585SUlKQRI0ZIyj2kdOLECS1evNi+zP79+yVJ586d06lTp7R//355eXnp1ltvdcdLAAAAJYxbw03fvn2VlpamiRMnKiUlRREREVq9erVCQ0Ml5V6078pr3jRv3tz+/3v37tXSpUsVGhqqo0eP3sjSAQBACeXW69y4A9e5AQCg9CkV17kBAAAoDoQbAABgKYQbAABgKYQbAABgKYQbAABgKYQbAABgKYQboCRKT5eOH8//sePHcx8HAOSLcAOUNOnpUpcuUvv20pU/9JqcnNvepQsBBwAKQLgBSprMTOnkSenwYalDh98DTnJy7v3Dh3Mfz8x0Z5UAUGIRboCSplYtafNmKTz894CTmPh7sAkPz328Vi331gkAJZRbf1sKQAFCQnIDTF6giY7Obc8LNiEhbiwOAEo2Rm6AkiokRFqyxLFtyRKCDQBcA+EGKKmSk6XYWMe22FjnScYAAAeEG6AkunzycHi4tH274xwcAg4AFIhwA5Q0x487Tx5u08Z5knFB18EBgDKOCcVASePnJwUF5f7/5ZOHL59kHBSU2w8A4IRwA5Q0AQHS2rW517G58nTvkBBpy5bcYBMQ4J76AKCEI9wAJVFAQMHhhevbAMBVMecGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGQPFJTy/4SsrHj+c+DgBFjHADoHikp0tdukjt2zv/FlZycm57ly4EHABFjnADoHhkZkonTzr/2OflPwp68mRuPwAoQoQbAMWjVi3nH/tMTHT+UVCuuAygiPHzCwCKz+U/9nn4sBQdndueF2zyfhQUAIoQIzcAildIiLRkiWPbkiWlK9gwMRooVQg3AIpXcrIUG+vYFhvrPMm4pGJiNFDqEG4AFJ/LJw+Hh0vbtzvOwSkNAcdqE6MZhUIZQLgBUDyOH3eePNymjfMk44K+aEsKK02MZhQKZQThBkDx8POTgoKcJw/nTTIOD8993M/PnVW65vKa8yZGXx5sSsv8ISuNQjEChasg3AAoHgEB0tq10pYtzl/+ISG57WvX5vYrDawwMdoqo1BWGoGySkgrYa/D7eEmPj5eYWFh8vHxUWRkpLZt23bV/lu2bFFkZKR8fHwUHh6uuXPn3qBKARRaQEDBX5S1apWeYCOV/onReawwCmWVESirhLQS+DrcGm6WL1+uUaNGady4cdq3b5/atm2rrl27KikpKd/+R44cUbdu3dS2bVvt27dPzz//vJ544gl9+OGHN7hyAGWKFSZGX660j0JZZQTKKiGtJL4O40atWrUyI0aMcGhr0KCBee655/Lt/7e//c00aNDAoW348OHmtttuc/k509PTjSSTnp5e+IIBlD3JycaEhxsj5f43KSm3PSnJsT052b11FsbltefdLn9tpYUVXseVf0fbt+f/91bS3YDXUZjvb7eN3Fy4cEF79+5VTEyMQ3tMTIwSExPzXWbHjh1O/e+66y7t2bNHFy9eLLZaAZRhVpoYLVlrFKq0j0BJ1jhMKJW41+G2cHP69Gnl5OQoODjYoT04OFipqan5LpOamppv/+zsbJ0+fTrfZbKyspSRkeFwAwCXWWlitFVOz89jpXlQpT2kSSXqdbh9QrHNZnO4b4xxartW//za88TFxSkgIMB+CyltfywA3M8qE6OtNAplpREoq4S0EvQ63BZuqlatKk9PT6dRmpMnTzqNzuSpVq1avv3LlSunKlWq5LvM2LFjlZ6ebr8ll7Y/FgAoKlYZhbLSCJRVQloJex1uCzdeXl6KjIxUQkKCQ3tCQoLatGmT7zJRUVFO/devX6+WLVuqfPny+S7j7e0tf39/hxsAlFlWGIWyygiUVUJaCXwd5W7YM+Vj9OjRio2NVcuWLRUVFaV58+YpKSlJI0aMkJQ76nLixAktXrxYkjRixAjNmjVLo0eP1kMPPaQdO3Zo/vz5evfdd935MgAAN1LeCFRmpnNQyxuB8vMr+UEtL6RJ+Ye0Dh1KR0grga/DreGmb9++SktL08SJE5WSkqKIiAitXr1aoaGhkqSUlBSHa96EhYVp9erVeuqppzR79mzVqFFDr7/+uu699153vQQAgDsEBBQcXkr69W3yWCWklcDXYTN5M3LLiIyMDAUEBCg9PZ1DVAAAlBKF+f52+9lSAAAARYlwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALIVwAwAALMWtP7/gDnkXZM7IyHBzJQAAwFV539uu/LBCmQs3mZmZkqSQvB/2AgAApUZmZqYCrvE7VWXut6UuXbqkH3/8UX5+frLZbO4up0TKyMhQSEiIkpOT+f2tEoDtUbKwPUoetknJUlzbwxijzMxM1ahRQx4eV59VU+ZGbjw8PFSrtPxirJv5+/uzoyhB2B4lC9uj5GGblCzFsT2uNWKThwnFAADAUgg3AADAUgg3cOLt7a0XX3xR3t7e7i4FYnuUNGyPkodtUrKUhO1R5iYUAwAAa2PkBgAAWArhBgAAWArhBgAAWArhBgAAWArhBnZxcXH685//LD8/PwUFBalXr1769ttv3V0W/k9cXJxsNptGjRrl7lLKrBMnTujBBx9UlSpVVKFCBTVr1kx79+51d1llUnZ2tl544QWFhYXJ19dX4eHhmjhxoi5duuTu0sqMrVu3qmfPnqpRo4ZsNps++ugjh8eNMRo/frxq1KghX19fdejQQQcPHrwhtRFuYLdlyxY99thj2rlzpxISEpSdna2YmBidP3/e3aWVebt379a8efPUpEkTd5dSZv3888+Kjo5W+fLltWbNGh06dEivvvqqKleu7O7SyqSpU6dq7ty5mjVrlr7++mu9/PLLmjZtmmbOnOnu0sqM8+fPq2nTppo1a1a+j7/88suaPn26Zs2apd27d6tatWrq3Lmz/TceixOngqNAp06dUlBQkLZs2aJ27dq5u5wy69y5c2rRooXi4+M1adIkNWvWTDNmzHB3WWXOc889p+3bt2vbtm3uLgWSevTooeDgYM2fP9/edu+996pChQpasmSJGysrm2w2m/7zn/+oV69eknJHbWrUqKFRo0bp2WeflSRlZWUpODhYU6dO1fDhw4u1HkZuUKD09HRJUmBgoJsrKdsee+wxde/eXXfeeae7SynTPvnkE7Vs2VL33XefgoKC1Lx5c7355pvuLqvMuv322/XZZ5/pu+++kyR9+eWX+vzzz9WtWzc3VwZJOnLkiFJTUxUTE2Nv8/b2Vvv27ZWYmFjsz1/mfjgTrjHGaPTo0br99tsVERHh7nLKrGXLlum///2vdu/e7e5SyrzDhw9rzpw5Gj16tJ5//nnt2rVLTzzxhLy9vTVgwAB3l1fmPPvss0pPT1eDBg3k6empnJwcTZ48Wf369XN3aZCUmpoqSQoODnZoDw4O1rFjx4r9+Qk3yNfjjz+uAwcO6PPPP3d3KWVWcnKynnzySa1fv14+Pj7uLqfMu3Tpklq2bKmXXnpJktS8eXMdPHhQc+bMIdy4wfLly/X2229r6dKlatSokfbv369Ro0apRo0aGjhwoLvLw/+x2WwO940xTm3FgXADJyNHjtQnn3yirVu3qlatWu4up8zau3evTp48qcjISHtbTk6Otm7dqlmzZikrK0uenp5urLBsqV69um699VaHtoYNG+rDDz90U0Vl2zPPPKPnnntO999/vySpcePGOnbsmOLi4gg3JUC1atUk5Y7gVK9e3d5+8uRJp9Gc4sCcG9gZY/T4449rxYoV2rhxo8LCwtxdUpnWqVMnffXVV9q/f7/91rJlSz3wwAPav38/weYGi46Odro0wnfffafQ0FA3VVS2/fLLL/LwcPwK8/T05FTwEiIsLEzVqlVTQkKCve3ChQvasmWL2rRpU+zPz8gN7B577DEtXbpUH3/8sfz8/OzHTAMCAuTr6+vm6soePz8/p/lOFStWVJUqVZgH5QZPPfWU2rRpo5deekl9+vTRrl27NG/ePM2bN8/dpZVJPXv21OTJk1W7dm01atRI+/bt0/Tp0zVkyBB3l1ZmnDt3Tj/88IP9/pEjR7R//34FBgaqdu3aGjVqlF566SXVq1dP9erV00svvaQKFSqof//+xV+cAf6PpHxvCxcudHdp+D/t27c3Tz75pLvLKLNWrlxpIiIijLe3t2nQoIGZN2+eu0sqszIyMsyTTz5pateubXx8fEx4eLgZN26cycrKcndpZcamTZvy/c4YOHCgMcaYS5cumRdffNFUq1bNeHt7m3bt2pmvvvrqhtTGdW4AAIClMOcGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGAABYCuEGQIk3aNAg9erVy91lACglCDcAAMBSCDcASrXp06ercePGqlixokJCQvToo4/q3LlzDn3efPNNhYSEqEKFCrr77rs1ffp0Va5c2T0FAyh2hBsApZqHh4def/11/e9//9Nbb72ljRs36m9/+5v98e3bt2vEiBF68skntX//fnXu3FmTJ092Y8UAihs/nAmgxBs0aJDOnj2rjz766Jp933//fT3yyCM6ffq0JOn+++/XuXPntGrVKnufBx98UKtWrdLZs2eLqWIA7sTIDYBSbdOmTercubNq1qwpPz8/DRgwQGlpaTp//rwk6dtvv1WrVq0clrnyPgBrIdwAKLWOHTumbt26KSIiQh9++KH27t2r2bNnS5IuXrwoSTLGyGazOSzHgDVgbeXcXQAAXK89e/YoOztbr776qjw8cv+t9t577zn0adCggXbt2uW0HADrItwAKBXS09O1f/9+h7abb75Z2dnZmjlzpnr27Knt27dr7ty5Dn1Gjhypdu3aafr06erZs6c2btyoNWvWOI3mALAOJhQDKPEGDRqkt956y6l94MCBatq0qaZNm6azZ8+qXbt2euCBBzRgwAD9/PPP9tO933zzTU2YMEFnzpzRXXfdpZYtW2rWrFlKSUm5wa8EwI1AuAFQ5jz00EP65ptvtG3bNneXAqAYcFgKgOW98sor6ty5sypWrKg1a9borbfeUnx8vLvLAlBMGLkBYHl9+vTR5s2blZmZqfDwcI0cOVIjRoxwd1kAignhBgAAWArXuQEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJZCuAEAAJby/wGEuj+MOigkjAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Compute and plot the MI as a function of lag:\n", + "maxLag = 10;\n", + "\n", + "misVsLag = np.zeros(maxLag);\n", + "\n", + "for lag in range(1,maxLag+1): # To go from 1 up to maxLag\n", + " misVsLag[lag-1] = simpleinfotheory.mutualinformationempirical(processedStr[:-lag],processedStr[lag:])[0];\n", + "\n", + "plt.scatter(range(1,maxLag+1), misVsLag, c=\"red\", marker=\"x\");\n", + "plt.xlabel('Lag')\n", + "plt.ylabel('MI (bits)');\n", + "plt.title('Average MI between characters separated by the given lag');" + ] + }, + { + "cell_type": "markdown", + "id": "7445b88c-d912-4976-9368-11478dbbcf74", + "metadata": {}, + "source": [ + "# 7. (Optional extension) Pointwise mutual information between successive letters in written English\n", + "\n", + "_Further challenge_ -- It would be interesting to inspect the **local or pointwise mutual information** between each possible pair of consecutive letters. (See Part 3 of the lecture)\n", + "\n", + "1. To do this, first note how `simpleinfotheory.jointentropyempirical()` returns the set of symbols and their probabilities, as well as the joint entropy value.\n", + "2. Now we will alter `simpleinfotheory.mutualinformationempirical()` to similarly retrieve and return all of the relevant probabilities for each consecutive character pair:\n", + " 1. Notice how the calls for the joint entropy, $Y$ entropy and $X$ entropy already retrieve these for us.\n", + " 2. Then alter the return statement so that all of these relevant values are returned: `return result, xySymbols, xyProbs, xSymbols, xProbs, ySymbols, yProbs` (this is already done in the solution code for `simpleinfotheory.py`).\n", + " 3. You will need to restart the kernel to reload the library. You'll also need to update the function calls above, since they're now returning a list. If you append `[0]` to them, such as `simpleinfotheory.mutualinformationempirical(...)[0]`, then this will just pick out the main `result` return variable for the above as desired.\n", + "3. Next, call `simpleinfotheory.mutualinformationempirical()` again as per step 2 of the previous exercise for lag 1, but this time storing all of these return values:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "539b82af-b94e-4ca1-9362-ba70078c30cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.7196084319029961\n" + ] + } + ], + "source": [ + "# Call the mutual information empirical again, this time storing all of the return values.\n", + "result, xySymbols, xyProbs, xSymbols, xProbs, ySymbols, yProbs = simpleinfotheory.mutualinformationempirical(processedStr[:-1],processedStr[1:]);\n", + "print(result)" + ] + }, + { + "cell_type": "markdown", + "id": "9ec8b8b0-92ea-444b-9b02-aeb3c01ab679", + "metadata": {}, + "source": [ + "4. Now, we loop over all possible joint symbols and compute the pointwise mutual information -- fill in the line of the code marked with `???` to compute the pointwise MI and then run this code block:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "b3a37011-0238-47d0-85a3-b384573fa2c0", + "metadata": {}, + "outputs": [], + "source": [ + "pointwiseMIs = np.zeros((xSymbols.size, ySymbols.size)); # Create array to store the pointwise MI values for each possible character pair\n", + "for firstCharIndex in range(xSymbols.size):\n", + " firstChar = xSymbols[firstCharIndex];\n", + " probFirst = xProbs[firstCharIndex];\n", + " for secondCharIndex in range(ySymbols.size):\n", + " secondChar = ySymbols[secondCharIndex];\n", + " probSecond = yProbs[secondCharIndex];\n", + " jointSymbolIndex = np.argwhere((xySymbols[:,0] == firstChar) & (xySymbols[:,1] == secondChar));\n", + " if (jointSymbolIndex.size == 0):\n", + " pointwiseMIs[firstCharIndex, secondCharIndex] = 0; # No occurence, so set to 0\n", + " continue;\n", + " probJoint = xyProbs[jointSymbolIndex];\n", + " # Compute the pointwise MI from probJoint, probFirst and probSecond\n", + " pointwiseMIs[firstCharIndex, secondCharIndex] = np.log2( probJoint / (probFirst * probSecond) );" + ] + }, + { + "cell_type": "markdown", + "id": "e7f91288-9d24-4736-ae88-b8f3562d09a4", + "metadata": {}, + "source": [ + "5. Can you plot these values using `plt.imshow()`? Run the command `plt.colorbar()` to insert a colour bar to show the scale. The plot will have the first letters along the y axis, and second letters along the x axis. You can label these using:\n", + "\n", + " plt.xlabel('Second letter')\n", + " plt.xticks(ticks=range(0,27), labels=ySymbols.flatten())\n", + " plt.ylabel('First letter');\n", + " plt.yticks(ticks=range(0,27), labels=xSymbols.flatten())\n", + " cbar = plt.colorbar()\n", + " cbar.set_label('MI (bits)');" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "d4351b95-71de-4509-9278-6790516c007e", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAg0AAAHFCAYAAABxS8rQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAABw3ElEQVR4nO3deVhU1f8H8PewDPuOICqC4IYrruWS4K7llmVWKKKmqamZpWVWrkVuadnXtcQl0yxbzAV3TEvNNcs1tyQBxYVFlmGZ8/vDH5Mj29zDwAzwfj3PfR653M89Zy7D+OGcc+9HJYQQICIiIiqGhak7QEREROUDkwYiIiIyCJMGIiIiMgiTBiIiIjIIkwYiIiIyCJMGIiIiMgiTBiIiIjIIkwYiIiIyCJMGIiIiMgiTBiNavXo1VCoVVCoVYmJi8n1fCIHatWtDpVIhNDRU73sqlQpjx44ttg1DjzNUeno6pk+fXmB/qeKIiYkp9H1Zmq5fvw6VSoXVq1crjo2Li8P06dNx+vTpfN/bvn07pk+fXuL+mdJ7772HmjVrwsrKCq6uroUeV1avlZ8FZAgmDaXAyckJX375Zb79Bw4cwJUrV+Dk5GSCXhUsPT0dM2bM4AdFBde8eXMcPnwYzZs3N3VXDBYXF4cZM2YUmjTMmDGj7DtlJD/99BM+/PBDhIeH48CBA9izZ0+hx5bVa+VnARmCSUMpGDhwIDZv3oyUlBS9/V9++SXatGmDmjVrmqhnVFk5OzvjySefhLOzs6m7YtbS09PLpJ2//voLADB+/Hi0a9cOLVu2LJN2iUqKSUMpeOmllwAAGzZs0O1LTk7G5s2bMWzYMKO0sXz5ctStWxc2NjZo0KABNm7cmO+YhIQEvPrqq6hRowbUajVq1aqFGTNmICcnB8DDoeMqVaoAAGbMmKGbWomIiMDZs2ehUqnw7bff6s534sQJqFQqNGzYUK+dPn36oEWLFnr7vvnmG7Rp0wYODg5wdHRE9+7dcerUqXx9PH78OPr06QN3d3fY2tqiWbNm2LRpk94xedM++/fvx+jRo+Hp6QkPDw/0798fcXFxxV6rq1ev4sUXX0S1atVgY2MDb29vdO7cWe8vWJVKVeAQsL+/PyIiIvT23bx5EyNHjoSvry/UajWqVauG559/Hrdu3dIdk5SUhDfffBMBAQGwsbGBl5cXnn76aVy4cEF3TFZWFmbPno369evDxsYGVapUwdChQ5GYmKjX3r59+xAaGgoPDw/Y2dmhZs2aeO655/T+g1u6dCmaNm0KR0dHODk5oX79+nj33Xd13398emLRokVQqVS4fPlyvtf89ttvQ61W486dO7p9e/bsQefOneHs7Ax7e3u0a9cOe/fuLfK6F+Xvv//Gyy+/DC8vL9jY2CAoKAj/+9//9PrbqlUrAMDQoUN1783p06cjIiJCd2zefpVKhevXrwN4OA24ZMkSBAcHw87ODm5ubnj++edx9epVvT6EhoaiUaNG+OWXX9C2bVvY29vrfj8NueYF0Wq1mDt3ru5n6uXlhfDwcPz777+6Y/z9/fHee+8BALy9vQt97wEwymvduHEjVCoVPv/8c71zT5s2DZaWlti9e3eRnwVEegQZTVRUlAAgjh07JgYPHixat26t+97SpUuFg4ODSElJEQ0bNhQhISF6sQDEa6+9VmwbAISvr69o0KCB2LBhg9iyZYvo0aOHACC+/fZb3XHx8fHC19dX+Pn5ieXLl4s9e/aIWbNmCRsbGxERESGEECIzM1NER0cLAGL48OHi8OHD4vDhw+Ly5ctCCCF8fHzEyJEjdef8+OOPhZ2dnQAgbt68KYQQIjs7Wzg7O4vJkyfrjvvwww+FSqUSw4YNE1u3bhXff/+9aNOmjXBwcBBnz57VHbdv3z6hVqvFU089Jb755hsRHR0tIiIiBAARFRWV77oGBASIcePGiZ07d4ovvvhCuLm5iY4dOxZ7zerVqydq164t1q1bJw4cOCA2b94s3nzzTbF//3696zpt2rR8sX5+fmLIkCG6r//991/h4+MjPD09xSeffCL27NkjvvnmGzFs2DBx/vx5IYTQ/YwdHBzEzJkzxc6dO8XmzZvF66+/Lvbt2yeEECI3N1f06NFDODg4iBkzZojdu3eLL774QlSvXl00aNBApKenCyGEuHbtmrC1tRVdu3YVP/74o4iJiRHr168XgwcPFvfv3xdCCLFhwwYBQIwbN07s2rVL7NmzRyxbtkyMHz9e1+/9+/cLALrXnJiYKNRqtZg6dare683JyRHVqlUT/fv31+1bt26dUKlUol+/fuL7778XP//8s+jVq5ewtLQUe/bsKfLaX7t2Ld/P8+zZs8LFxUU0btxYrF27VuzatUu8+eabwsLCQkyfPl0IIURycrLu5/7ee+/p3puxsbHi8uXL4vnnnxcAdPsPHz4sMjMzhRBCjBgxQlhbW4s333xTREdHi6+//lrUr19feHt7i4SEBF0/QkJChLu7u/D19RWLFy8W+/fvFwcOHDDomhdm5MiRAoAYO3asiI6OFsuWLRNVqlQRvr6+IjExUQghxMmTJ8Xw4cMFABEdHa17XQUx1msdNWqUUKvV4tixY0IIIfbu3SssLCzEe++9J4Qo/rOAKA+TBiN6NGnI+5D+66+/hBBCtGrVSvefdUmTBjs7O70PhJycHFG/fn1Ru3Zt3b5XX31VODo6in/++Ucvfv78+QKA7j/vxMTEQv/DHDRokAgICNB93aVLFzFixAjh5uYm1qxZI4QQ4tdffxUAxK5du4QQQty4cUNYWVmJcePG6Z0rNTVVVK1aVbzwwgu6ffXr1xfNmjUT2dnZesf26tVL+Pj4iNzcXCHEf9d1zJgxesfNnTtXABDx8fGFXq87d+4IAGLRokWFHiOE4UnDsGHDhLW1tTh37lyh55o5c6YAIHbv3l3oMXn/0W/evFlv/7FjxwQAsWTJEiGEEN99950AIE6fPl3oucaOHStcXV0L/b4Q+ZMGIYTo37+/qFGjhu46CyHE9u3bBQDx888/CyGESEtLE+7u7qJ3795658vNzRVNmzbVS4wLUlDS0L17d1GjRg2RnJyc73XY2tqKe/fuCSH+uxaPxuZ57bXXREF/8xw+fFgAEAsWLNDbHxsbK+zs7PSS25CQEAFA7N27V+9YQ655Qc6fP1/g+/To0aMCgHj33Xd1+6ZNmyYA6BKJohjjtWZmZopmzZqJWrVqiXPnzglvb28REhIicnJydMcU9VlAlIfTE6UkJCQEgYGBWLVqFf78808cO3bMaFMTnTt3hre3t+5rS0tLDBw4EJcvX9YNg27duhUdO3ZEtWrVkJOTo9t69uwJ4OGiTEPauXr1Kq5du4bMzEwcOnQIPXr0QMeOHbF7924AD4etbWxs0L59ewDAzp07kZOTg/DwcL12bW1tERISohsev3z5Mi5cuICwsDAA0Dv26aefRnx8PC5evKjXnz59+uh93aRJEwDAP//8U+hrcHd3R2BgIObNm4dPPvkEp06dglarLfa1F2bHjh3o2LEjgoKCijymbt266NKlS6HHbN26Fa6urujdu7feaw8ODkbVqlV11yk4OBhqtRojR47EmjVr8g2xA0Dr1q2RlJSEl156CT/99JPetEJRhg4din///VdvEV5UVBSqVq2qe5/89ttvuHfvHoYMGaLXT61Wix49euDYsWNIS0szqD0AyMzMxN69e/Hss8/C3t4+3889MzMTR44cMfh8j9u6dStUKhUGDRqkd+6qVauiadOm+Rb5ubm5oVOnTnr7DLnmBdm/fz8A5BvSb926NYKCgko0nVMQJa/VxsYGmzZtwt27d9G8eXMIIbBhwwZYWloatU9U8TFpKCUqlQpDhw7FV199hWXLlqFu3bp46qmnjHLuqlWrFrrv7t27AIBbt27h559/hrW1td6Wtx7BkP9Y8v7T27NnDw4dOoTs7Gx06tQJXbp00X0A7tmzB+3atYOdnZ2uXQBo1apVvra/+eYbXbt5x7311lv5jhszZkyBffTw8ND72sbGBgCQkZFR6GtQqVTYu3cvunfvjrlz56J58+aoUqUKxo8fj9TU1GKvweMSExNRo0aNEh9z69YtJCUlQa1W53v9CQkJutceGBiIPXv2wMvLC6+99hoCAwMRGBiITz/9VHeuwYMHY9WqVfjnn3/w3HPPwcvLC0888YQusStMz5494ePjg6ioKADA/fv3sWXLFoSHh+v+M8n7OT3//PP5+jlnzhwIIXDv3r2iL9oj7t69i5ycHCxevDjf+Z5++mkAhr03C3Pr1i0IIeDt7Z3v/EeOHMl3bh8fn3znMOSaF/baCjtntWrVdN83FqWvtXbt2njqqaeQmZmJsLCwAvtJVBwrU3egIouIiMAHH3yAZcuW4cMPPzTaeRMSEgrdl/cfq6enJ5o0aVJou9WqVSu2nRo1aqBu3brYs2cP/P390bJlS7i6uqJz584YM2YMjh49iiNHjujdDubp6QkA+O677+Dn51foufOOmzJlCvr371/gMfXq1Su2j4bw8/PT3QJ76dIlbNq0CdOnT0dWVhaWLVsG4GECotFo8sU+/kFfpUoVvUVtBTHkmLzFnNHR0QV+/9Hbcp966ik89dRTyM3NxfHjx7F48WJMmDAB3t7eePHFFwE8HDUYOnQo0tLS8Msvv2DatGno1asXLl26VOjPwdLSEoMHD8Znn32GpKQkfP3119BoNBg6dKhePwFg8eLFePLJJws8z6OjXsVxc3PTtfvaa68VeEytWrUMPt/jPD09oVKpcPDgQV1S+ajH96lUqgLPY8g1f1ze7158fHy+pDEuLk53LY1F6Wv94osvsG3bNrRu3Rqff/45Bg4ciCeeeMKofaJKwLSzIxXLo2sa8rz99tuib9++Ii4uTrevtNY0BAYG6va98sorolq1arr54cKkpKQIAHrzn48aM2aM8PT0FM2aNdMtmhJCiJo1a4pu3boJAOL333/X7b927ZqwsrISc+bMKfa11KlTRzz99NPFHlfQdRWi4Hl6QwUHB4tWrVrpvq5Xr16+vuzdu1cAKHBNw4ULFwo9d96ahsfnyh/11VdfCQDiyJEjivuelJQkAIhJkyYVesyPP/4oAIht27YJIQq/Vnnz8EuWLBEtW7YUbdq00ft+amqqcHV1FaNHj1bcTyEKXtPQpUsX0bRpU6HRaIqMPXPmjN76jkdNnDhRANAtGM1z6NAhAUB88803xfYtJCRENGzY0KDXYcg1v3DhggCgtwBVCCF+//13AUBv0amSNQ3GeK1nzpwRdnZ2Ijw8XGg0GtGiRQvh5+en9/lQ3GcBkRBCcKShlH388cdGP6enpyc6deqE999/Hw4ODliyZAkuXLigd9vlzJkzsXv3brRt2xbjx49HvXr1kJmZievXr2P79u1YtmwZatSoAScnJ/j5+eGnn35C586d4e7uDk9PT/j7+wN4uK5hyZIluHPnDhYtWqQ7f+fOnREVFQU3Nze92y39/f0xc+ZMTJ06FVevXkWPHj3g5uaGW7du4ffff4eDg4NuZGL58uXo2bMnunfvjoiICFSvXh337t3D+fPncfLkSb3bPWWdOXMGY8eOxYABA1CnTh2o1Wrs27cPZ86cwTvvvKM7bvDgwXj//ffxwQcfICQkBOfOncPnn38OFxcXvfPNnDkTO3bsQIcOHfDuu++icePGSEpKQnR0NCZOnIj69etjwoQJ+Oabb9C3b1+88847aN26NTIyMnDgwAH06tULHTt2xIsvvoj169fj6aefxuuvv47WrVvD2toa//77L/bv34++ffvi2WefxbJly7Bv3z4888wzqFmzJjIzM7Fq1SoA/00fjRgxAnZ2dmjXrh18fHyQkJCAyMhIuLi46G5bLEz9+vXRpk0bREZGIjY2FitWrND7vqOjIxYvXowhQ4bg3r17eP755+Hl5YXExET88ccfSExMxNKlSxX9TD799FO0b98eTz31FEaPHg1/f3+kpqbi8uXL+Pnnn7Fv3z4AD6cJ7OzssH79egQFBcHR0RHVqlVDtWrV0LhxYwDAnDlz0LNnT1haWqJJkyZo164dRo4ciaFDh+L48ePo0KEDHBwcEB8fj0OHDqFx48YYPXp0kf0z5JoXpF69ehg5ciQWL14MCwsL9OzZE9evX8f7778PX19fvPHGG4quU56Svta0tDS88MILqFWrFpYsWQK1Wo1NmzahefPmGDp0KH788UcAKPazgAgARxqMqbC/iB9X0pGG1157TSxZskQEBgYKa2trUb9+fbF+/fp8xyYmJorx48eLWrVqCWtra+Hu7i5atGghpk6dKh48eKA7bs+ePaJZs2bCxsYm31/W9+/fFxYWFsLBwUFkZWXp9q9fv14A0Ls171E//vij6Nixo3B2dhY2NjbCz89PPP/88/lu0fvjjz/ECy+8ILy8vIS1tbWoWrWq6NSpk1i2bJnumJKMNNy6dUtERESI+vXrCwcHB+Ho6CiaNGkiFi5cqLdyXKPRiMmTJwtfX19hZ2cnQkJCxOnTp/PdPSHEw9Xpw4YNE1WrVhXW1taiWrVq4oUXXhC3bt3Su26vv/66qFmzprC2thZeXl7imWee0RuhyM7OFvPnzxdNmzYVtra2wtHRUdSvX1+8+uqr4u+//xZCPFwh/+yzzwo/Pz9hY2MjPDw8REhIiNiyZYvuPGvWrBEdO3YU3t7eQq1W6/pz5swZg67VihUrdCNYj9/RkOfAgQPimWeeEe7u7sLa2lpUr15dPPPMM3q3+RakoJGGvP3Dhg0T1atXF9bW1qJKlSqibdu2Yvbs2XrHbdiwQdSvX19YW1vrrezXaDTilVdeEVWqVBEqlUoAENeuXdPFrVq1SjzxxBPCwcFB2NnZicDAQBEeHi6OHz+uO6awkQZDrnlhcnNzxZw5c0TdunWFtbW18PT0FIMGDcp3S6WSkYaSvtZBgwYJe3t7vdudhRDi22+/FQDEwoULdfuK+iwgEkIIlRBClH2qQkREROUN754gIiIigzBpICIiIoMwaSAiIiKDMGkgIiIigzBpICIiIoMwaSAiIiKDVLiHO2m1WsTFxcHJyanQR8QSEZF5EkIgNTUV1apVg4VF6f1dm5mZiaysLKOcS61Ww9bW1ijnMncVLmmIi4uDr6+vqbtBREQlEBsbW2zhN1mZmZmo5eeIhNu5Rjlf1apVce3atUqROFS4pCGv0E9w3/dgaa3sB3jv6cKrJRbGy015pUQASMtSS8WlZ8jFqS44Ko6xDb4v1VZ2rly53axM5W9Ha5scqbb83OVe2+X4KopjcrLkroeTq/L3IwCkJij/WQMALJU/582xiuFlsfXibPIXBzPErcvKiz5Ze8ldRytruRLqGTcdFMcIK7ln7PkGJErFpW/OXynXEDnPJCmOeXDVpfiDHqHNzETszNl6RduMLSsrCwm3c/HPCX84O5VsNCMlVQu/FteRlZXFpKE8ypuSsLS2hZXCpMHCXvkvrpWD3PCWpZXcf/4WqvzV7AyKs1H+Zra0l2tLK5k0WFhYK46xtMmWasvaQfL62yu/jhaWctfD0l7uPy0LO8kPLomkwdJeLmmzkuyizGuT+b0GAEtrub9CZfoomzRYOcj9jlqq5X4AQuIzwULyP9KymF52dFLB0alk7WhRuabBK1zSQEREZIhcoUVuCQsp5Aq55L68YtJARESVkhYCWpQsayhpfHnDWy6JiIjIIOV+pEGj0UCj+W9RVUpKigl7Q0RE5YUWWpR0cqHkZyhfyv1IQ2RkJFxcXHQbb7ckIiJD5AphlK0yKfdJw5QpU5CcnKzbYmNjTd0lIiKiCqncT0/Y2NjAxkbutiMiIqq8uBBSuXKfNBAREcnQQiCXSYMi5X56goiIiMoGRxqIiKhS4vSEckwaiIioUjLG3Q+V7e6JCps03GqvhYWdsvtnPXfbK27nQb9MxTEAkKOVmxnKyZarYxC4W3lhoSuurlJtaSVrJsjIgVwNiZt7lRXRyWPbNUl5jLNcfYaUNLln9ssWMXIMV/4eubrYW6qt25dcpeIc6yYpjslIl6zPIFnXRKaORFDQv1JtXTpRUyou90m596SNRnl9mGqNbik6PidNg38Ut0JlpcImDUREREXR/v9W0nNUJkwaiIioUso1wt0TJY0vb5g0EBFRpZQrYIQql8bpS3lhdrdcRkdHo3379nB1dYWHhwd69eqFK1eumLpbRERElZ7ZJQ1paWmYOHEijh07hr1798LCwgLPPvsstNrKNnNERESlSWukrTIxu+mJ5557Tu/rL7/8El5eXjh37hwaNWqU73hWuSQiIhlaqJALVYnPUZmY3UjDlStX8PLLLyMgIADOzs6oVasWAODGjRsFHs8ql0REVF7cvHkTgwYNgoeHB+zt7REcHIwTJ06YulsGM7ukoXfv3rh79y5WrlyJo0eP4ujRowCArKysAo9nlUsiIpKhFcbZDHX//n20a9cO1tbW2LFjB86dO4cFCxbAVfKZOKZgVtMTd+/exfnz57F8+XI89dRTAIBDhw4VGcMql0REJCPXCNMTSuLnzJkDX19fREVF6fb5+/uXqP2yZlYjDW5ubvDw8MCKFStw+fJl7Nu3DxMnTjR1t4iIiIqUkpKitz261i7Pli1b0LJlSwwYMABeXl5o1qwZVq5caYLeyjOrpMHCwgIbN27EiRMn0KhRI7zxxhuYN2+eqbtFREQVUN5IQ0k3APD19dVbXxcZGZmvvatXr2Lp0qWoU6cOdu7ciVGjRmH8+PFYu3ZtWb90aWY1PQEAXbp0wblz5/T2iUpWEISIiEqfVqigFSW8e+L/42NjY+Hs7KzbX9C0uVarRcuWLfHRRx8BAJo1a4azZ89i6dKlCA8PL1E/yorZJQ3GYhdrBUsbZS8vp+89xe2kJCsvcgUAtufspOKqXcyVikubpryIUd135Ar2XJwoV2ip9lLlr+3yKLkCXslBch8UbjuUF7pS9b0j1ZbLdgepOLuLUmFI/FJ5MaLA0fel2ro0zlEqLjNTeR9VN+R+1+zPyL2Pqw5XXnzq4h9yhaeqnJYKw+1ucp8jWq3y35t/b7kpayNDrgigqTk7O+slDQXx8fFBgwYN9PYFBQVh8+bNpdk1o6qwSQMREVFRynohZLt27XDxon5Wf+nSJfj5+ZWoD2XJrNY0hIaGYsKECabuBhERVQK5sDDKZqg33ngDR44cwUcffYTLly/j66+/xooVK/Daa6+V4qs0LrNKGoiIiMqK+P81DSXZhII1Ea1atcIPP/yADRs2oFGjRpg1axYWLVqEsLCwUnyVxsXpCSIiojLSq1cv9OrVy9TdkGZ2Iw05OTkYO3asrsrle++9x7sniIjI6Ix5y2VlYXYjDWvWrMHw4cNx9OhRHD9+HCNHjoSfnx9GjBhR4PEsWEVERDJyhQVyRcn+ds6tZH/Tmt1Ig6+vLxYuXIh69eohLCwM48aNw8KFCws9ngWriIiIyobZJQ1PPvkkVKr/hnvatGmDv//+G7m5Bd9XzIJVREQkQwsVtLAo4cbpiXKFBauIiEhGWT+noSIwu5GGI0eO5Pu6Tp06sLSUe/IfERERGYfZjTTExsZi4sSJePXVV3Hy5EksXrwYCxYsMHW3iIiogjHOQsjKtRLS7JKG8PBwZGRkoHXr1rC0tMS4ceMwcuRIU3eLiIgqmIdrGkpYsKqSTU+YVdIQExOj+/fSpUtN1xEiIiLKx6ySBmOyyAEsFC6DcLFTXl3Ne7rcWotLr6il4uz3pkvFXblXdPW1gsRPlBu2qz/xH6m4O2vcFceIWGUV9PKoXOQqeLYb+afimO0XG0m1ZVlHKgxu5+WGS60slVc+zK7hIdWWfd0kqbiUe8orf3r8LdUUEjrKVYK8HVdFcYxwz5Jqy32n3N1it7vJ3ZpueV55ddKcQPOtWqlVWDui4HNweoKIiKjC45oG5Zg0EBFRpZT3rIWSnaNyJQ1md8slERERmSeONBARUaWUK1TIVVDaurBzVCZmOdKg1WoxZ84c1K5dGzY2NqhZsyY+/PBDU3eLiIgqkNz/XwhZ0q0yMcuRhilTpmDlypVYuHAh2rdvj/j4eFy4cKHAY1nlkoiIqGyYXdKQmpqKTz/9FJ9//jmGDBkCAAgMDET79u0LPD4yMhIzZswoyy4SEVEFoBUW0Jbw7gltJbt7wuzGVc6fPw+NRoPOnTsbdDyrXBIRkQxOTyhndiMNdnZ2io5nlUsiIqKyYXYpUp06dWBnZ4e9e/eauitERFSBafHfHRSym9bUL6KMmd1Ig62tLd5++21MnjwZarUa7dq1Q2JiIs6ePYvhw4ebuntERFRBGOfhTmb3t3epMrukAQDef/99WFlZ4YMPPkBcXBx8fHwwatQoU3eLiIioUjPLpMHCwgJTp07F1KlTTd0VIiKqoIxTe4IjDRWCut1dWNorWyD5z3Xl1ekcO1srjgGAwE0ZUnF/D7GXihOaHMUxHofkKnH+M6KeVFzNKfcUx6S+L3cdLY87ScWdXdNEcYx1O7mFumrJR478HS73HnHL0RR/0GOS6sq1lfO78mqVAICayt/Hd1vKVaus//kDqbjLYa6KY4SF3G17NwdJlkKFXOVJTS3l75HqPyr7jMzJzkVZ3QOnhQpalOyJjiWNL2/KRYoUGhqKCRMmmLobRERUgeSNNJR0q0wq16slIiIiaRV2eoKIiKgoxng4U2V7uJPZvdq0tDSEh4fD0dERPj4+WLBggam7REREFZBWqIyyVSZmlzRMmjQJ+/fvxw8//IBdu3YhJiYGJ06cMHW3iIiIKj2zmp548OABvvzyS6xduxZdu3YFAKxZswY1atQoNIZVLomISIbWCNMTle3hTmb1aq9cuYKsrCy0adNGt8/d3R316hV+C19kZCRcXFx0m6+vb1l0lYiIyrm8Kpcl3SoTs3q1QqLEKKtcEhERlQ2zShpq164Na2trHDlyRLfv/v37uHTpUqExNjY2cHZ21tuIiIiKkwuVUTZZkZGRUKlU5eo5RGa1psHR0RHDhw/HpEmT4OHhAW9vb0ydOhUWFmaV2xARUQVgjOkF2fhjx45hxYoVaNJE+VNmTcns/jeeN28eOnTogD59+qBLly5o3749WrRoYepuERERGcWDBw8QFhaGlStXws3NzdTdUcSsRhqAh6MN69atw7p163T7Jk2aZMIeERFRRZQLlGh6Ie8cQP4792xsbGBjU3Ddmddeew3PPPMMunTpgtmzZ5eo/bJmdkmDsWTnWkKba6koxsJeeTEcrVquYNXlIcr6lsfWKV0qLvuG8gJB9xvJFdGxuyX3S3inpfKM2/K4XFtOsVqpuNvNlRefUjWRuw045Y5cMShLp2ypuLSTnopjslrJFYOy8pArmOTrrvxa3k52lGrrwii5omY1dil/b6UNTZJqK1XrLhXXsW7h68SKsv9iXcUxN/so+1zVZuQAWxQ3I8WY0xOP37k3bdo0TJ8+Pd/xGzduxMmTJ3Hs2LEStWsqFTZpICIiKooxS2PHxsbqLcQvaJQhNjYWr7/+Onbt2gVbW9sStWsqTBqIiIhKyJC7906cOIHbt2/rrdPLzc3FL7/8gs8//xwajQaWlnKj0GWFSQMREVVKAipoS7imQSiI79y5M/7880+9fUOHDkX9+vXx9ttvm33CADBpICKiSsqY0xOGcHJyQqNGjfT2OTg4wMPDI99+c2V2t1wKITB37lwEBATAzs4OTZs2xXfffWfqbhEREVV6ZjfS8N577+H777/H0qVLUadOHfzyyy8YNGgQqlSpgpCQkHzHs2AVERHJMEZp65LGx8TElCi+rJlV0pCWloZPPvkE+/bt0xWtCggIwKFDh7B8+fICk4bIyEjMmDGjrLtKRETlXK4RqlyWNL68Mauk4dy5c8jMzNSVxc6TlZWFZs2aFRgzZcoUTJw4Ufd1SkoKK10SERGVArNKGrTahw9F2bZtG6pXr673vcKerFXUU7eIiIgKYw7TE+WNWSUNDRo0gI2NDW7cuFHgVAQREZGxaGEBbQmnF0oaX96YVdLg5OSEt956C2+88Qa0Wi3at2+PlJQU/Pbbb3B0dMSQIUNM3UUiIqJKy6ySBgCYNWsWvLy8EBkZiatXr8LV1RXNmzfHu+++a+quERFRBZIrVMgt4fRCSePLG7NLGlQqFcaPH4/x48ebuitERFSBcU2DcmaXNBhLlsYKFpbKKlAKrfIfvuvfclX+bO/IVce0uyf3mNG4PlmKY9rWvirV1v2BclUFtW7K4/6dITefeKuhXLEY52PK4x4kybVVZ42m+IMKcHm03HvEofkdxTE1I+2k2ro8Sq6PaVnKf2+ct8u9H+80l6uEmviS8kq0WUnKq9ACQOCWDKm4/VWVV6sEAJfflb+XLTOVVcvNzdIiVnErcoQRqlyKEsaXN5Xr1RIREZE0s04ahBAYOXIk3N3doVKpcPr0aVN3iYiIKohcqIyyVSZmPT0RHR2N1atXIyYmBgEBAfD09DR1l4iIqILQipKvSdAqm30p98w6abhy5Qp8fHzQtm1bU3eFiIio0jPbpCEiIgJr1qwB8PCOCj8/P1y/ft20nSIiogpDa4SFkCWNL2/MNmn49NNPERgYiBUrVuDYsWOwtCx4tTWrXBIRkQwtVNCWcE1CSePLG7NNkVxcXODk5ARLS0tUrVoVVapUKfC4yMhIuLi46DYWqyIiIiodZps0GGrKlClITk7WbbGxZXWHLxERlWd5T4Qs6VaZmO30hKFY5ZKIiGRwTYNylevVEhERkbRyP9JAREQkQwsj1J6oZAshmTQQEVGlJIxw94SoZEmDWU9PTJgwgc9mICKiUpFX5bKkW2VSYUcaLK20sLRSVoEy667yCm7W6XKV8By/PSoVp32qmVx7zpmKY25P8pdq6+oEuaqOLnXvKY55EOck1ZbKTq46adUVJxTHPPhI7md2r6FkJdTzch9iFu0eKI7JrCK7CDlHKupevIviGMtGcs/5DdysvDIsAFx/VXkFz6DJN6XaetDKTypO9vrnSvxqJz+hrFqrNkMDrFXeDpWNCps0EBERFYV3TyjHpIGIiColY0wvVLbpicqVIhEREZE0jjQQEVGlxNoTypndSENqairCwsLg4OAAHx8fLFy4EKGhoZgwYYKpu0ZERBUI755QzuyShokTJ+LXX3/Fli1bsHv3bhw8eBAnT54s9HiNRoOUlBS9jYiIiIzPrKYnUlNTsWbNGnz99dfo3LkzACAqKgrVqlUrNCYyMhIzZswoqy4SEVEFwYWQypnVSMPVq1eRnZ2N1q1b6/a5uLigXr16hcawyiUREcng9IRyZjXSIMTDh7CoVKoC9xeEVS6JiIjKhlmNNAQGBsLa2hq///67bl9KSgr+/vtvE/aKiIgqIo40KGdWSYOTkxOGDBmCSZMmYf/+/Th79iyGDRsGCwuLfKMPREREJSHw322XspuSh5RHRkaiVatWcHJygpeXF/r164eLFy+W1ssrFWaVNADAJ598gjZt2qBXr17o0qUL2rVrh6CgINjaytUzICIiKkhZjzQcOHAAr732Go4cOYLdu3cjJycH3bp1Q1paWim+SuMyqzUNwMPRhvXr1+u+TktLw4wZMzBy5EgT9oqIiKhkoqOj9b6OioqCl5cXTpw4gQ4dOpioV8qYXdJw6tQpXLhwAa1bt0ZycjJmzpwJAOjbt6+i82Tct4NFprLRCd/AREXHA0ByLR/FMQBQpV2wVFx6VbVce453FcdkespVkKy77LZU3I3+VRXHuLe/I9WWw5euUnGZnZoojvHdLVdR83YzuSqXbn/LtRff0EFxTO1byqunPiT30aNOVB5nf1NuavPyQLnrr7qrvD2tl5tUW7k2ZTtt+8Bf7r1lrkx9y2VycjIAwN3dvUR9KEtmlzSMGDECN27cQHp6OtRqNVq0aIGDBw/C09PT1F0jIqIKxJhJw+MPFizuzj4hBCZOnIj27dujUaNGJepDWTK7NQ2Ojo54+eWX8eDBA9y7dw+7d+9G48aNTd0tIiKiQvn6+sLFxUW3RUZGFnn82LFjcebMGWzYsKGMemgcZjfSQEREVBaMOdIQGxsLZ2dn3f6iRhnGjRuHLVu24JdffkGNGjVK1H5ZM7uRBgDQarWYPHky3N3dUbVqVUyfPt3UXSIiogpGCJVRNgBwdnbW2wpKGoQQGDt2LL7//nvs27cPtWrVKuuXXGJmmTSsWbMGDg4OOHr0KObOnYuZM2di9+7dBR7LglVERFQevPbaa/jqq6/w9ddfw8nJCQkJCUhISEBGRoapu2Yws0wamjRpgmnTpqFOnToIDw9Hy5YtsXfv3gKPjYyM1JtH8vX1LePeEhFReVTSBzvlbYZaunQpkpOTERoaCh8fH932zTfflOKrNC6zXNPQpIn+bW0+Pj64fbvg2/imTJmCiRMn6r5OSUlh4kBERMUq61sui6qjVF6YZdJgba1/f7RKpYJWqy3wWBasIiIiKhtmmTQQERGVtkcXMpbkHJUJkwYiIqqUTP1EyPKISQMREVVKHGlQzuyShpiYmHz7fvzxxzLvBxEREekzu6TBaHJVDzcF7vyivPhUln/BCzSL1VWu0I8mO0cqLvtEdcUxNkPknnlxXyVXxrzmkL+Ut3WjgVRbVuly1zFhpPKfmyZDrvBRFY/7UnGePVKl4m4fDlAcc2mIXAEjC4tsqbhsV+W/bxZN5K6jY47cx6Pjt87FH/SYpDly18N9dLxUXNpge6m4alOUfyaIr5Qdn5OmQaziVuQII0xPcKSBiIioEhAASnoXZPm/iVIZs3y406NCQ0MxYcIEU3eDiIio0jP7kYbvv/8+33MbiIiISkoLFVQKnuhY2DkqE7NPGtzd3U3dBSIiqoB494RynJ4gIiIig5j9SENxNBoNNBqN7mtWuSQiIkNohQoqPtxJEbMfaSgOq1wSEZEMIYyzVSblPmmYMmUKkpOTdVtsbFnd4UtERFS5lPvpCVa5JCIiGVwIqVy5TxqIiIhkMGlQjkkDERFVSlwIqVy5X9NAREREZcPsRxoKqnpJRERUUsa4+6Gy3T1h9kmDLN8dAlbWyn6asS9lKG5HaOWGppKT5arM1Z+TJhWnXXxDcUzcVj+pth4Ey1XwvPB5bcUxFrfkrr/7X3KPJs9Ml6xqKuH2HeXVEksSZ1tbeXXMmp+rpdq6Okzu5+b/g/LqpFbpdlJtXR5lKRXX/+2DimNipraTauv8TLlqrUiWC7Pq5qY45m6spviDHqHNkPv8kPEwaSjpmgYjdaac4PQEERERGaRcJA0RERHo16+fqbtBREQVSN7dEyXdKpNyMT3x6aefQlS2MSAiIipV4v+3kp6jMikXSYOLi4upu0BERFTpcXqCiIgqJU5PKFcuRhqKwiqXREQkhfMTipWLkYaisMolERFJMcYoQyUbaSj3SQOrXBIREZWNcj89wSqXREQkg0+EVK7cJw1EREQyWOVSuXI/PUFERERlgyMNRERUORljISNHGgqXm5uLAwcO4P79+6XVnwJpNBo4OjqWaZtERFSx5a1pKOlWmSgaabC0tET37t1x/vx5uLkpr3amVE5ODi5duoTDhw/j1VdfVRR76+VMWCosJCluK6886f2bXJbpfDVdKu5BHbmnY2ZrlFfDy5ErxIkam+UGsKwylFeQvP+aXAJredJVKq7KbuWLbu83kGoKjsoLkwIA1L0TpeIyd3gpjnlQXbbqZ5ZUVOywXMUxKgu5T/U6n2RLxf3YLkRxTOoLcpUdnU7YSsWltpBrLyVQKowes2TJEsybNw/x8fFo2LAhFi1ahKeeesrU3TKI4jUNjRs3xtWrV0ujLwCA0NBQTJgwAQDw119/oWXLlmjYsCFGjRpVam0SEVElJIy0KfDNN99gwoQJmDp1Kk6dOoWnnnoKPXv2xI0bkn8llDHFScOHH36It956C1u3bkV8fDxSUlL0NmMKDg5Geno6tm3bViYjG0REVHmY4jHSn3zyCYYPH45XXnkFQUFBWLRoEXx9fbF06dJSepXGpXgcuUePHgCAPn36QKX672IJIaBSqZCbq3z4kIiIqDx7/I/mgp4hlJWVhRMnTuCdd97R29+tWzf89ttvpd5HY1CcNOzfv780+lGo6OhoDBw4EIsXL0Z4eHiZtk1ERBWckRYyPl7CYNq0aZg+fbrevjt37iA3Nxfe3t56+729vZGQkGCcjpQyxUlDSIjyRT6yNm7ciJEjR2LdunXo27dvmbVLREQVnzEf7hQbGwtnZ2fd/qKeVPzoKP3Dc4h8+8yV1MOdDh48iEGDBqFt27a4efMmAGDdunU4dOiQ0Tq2ZMkSjBo1Cj/99FORCYNGoynVdRVERFRBGXEhpLOzs95WUNLg6ekJS0vLfKMKt2/fzjf6YK4UJw2bN29G9+7dYWdnh5MnT+rKUqempuKjjz4ySqc2b96MCRMmYNeuXejYsWORx7LKJRERlQdqtRotWrTA7t279fbv3r0bbdu2NXp7Go0GBw8exLp167B8+XJ8//33uHbtWonOqThpmD17NpYtW4aVK1fC2tpat79t27Y4efJkiTqTJzg4GFWqVEFUVBREMU/OYJVLIiKSozLSZriJEyfiiy++wKpVq3D+/Hm88cYbuHHjhlEfK/Dbb7/hpZdegqurq+4xBrNmzcKgQYNQu3Zt1KlTB/PmzUNqaqricyte03Dx4kV06NAh335nZ2ckJSUp7kBBAgMDsWDBAoSGhsLS0hKff/55oceyyiUREUmReM5CgedQYODAgbh79y5mzpyJ+Ph4NGrUCNu3b4efn18JO/JQ3759cezYMbz88svYuXMnWrZsCXv7/57Ud/XqVRw8eBAbNmzAJ598grVr16Jr164Gn19x0uDj44PLly/D399fb/+hQ4cQEBCg9HSFqlu3Lvbv34/Q0FBYWVlh0aJFRjs3ERGRqYwZMwZjxowplXN369YN3377LdRqdYHfDwgIQEBAAIYMGYKzZ88iLi5O0fkVJw2vvvoqXn/9daxatQoqlQpxcXE4fPgw3nrrLXzwwQdKT1ekevXqYd++fboRhwULFhj1/EREVImZYKShtL322msGH9uwYUM0bNhQ0fkVJw2TJ09GcnIyOnbsiMzMTHTo0AE2NjZ46623MHbsWKWnyycmJkbv66CgINy6davE5yUiItJTwatcxsbGQqVSoUaNGgCA33//HV9//TUaNGiAkSNHSp1TqrLQhx9+iKlTp+LcuXPQarVo0KCB2VWhzM60Rq7KuvgDH2HlobyIi8ZFrqpTXAe562Xb4Y5UXEqKg+KYLH+5okLqQ3JFjISl8l++HK3UXcOwvyVXjOjmKOXXpHn1m1JtnYmuLxXnPUXZ+z7P35MzFMc8SCjb9USq63aKY4Rc/TTEPyX3n0GaRBEv1Z2Ch5KLk1XGT9e3aZSkOCbnmrIie6pM2SJo9LiXX34ZI0eOxODBg5GQkICuXbuiYcOG+Oqrr5CQkCA1O6D4E3fYsGFITU2Fvb09WrZsidatW8PR0RFpaWkYNmyY4g4QERGZQkUvjf3XX3+hdevWAIBNmzahUaNG+O233/D1119j9erVUudUnDSsWbMGGRn5/yLJyMjA2rVrpTpRlEerXhIRERmNCapclqXs7Gzd3YV79uxBnz59AAD169dHfHy81DkNThpSUlKQnJwMIQRSU1P1nsB4//59bN++HV5eXlKdICIiIuNq2LAhli1bhoMHD2L37t26gpNxcXHw8PCQOqfBs32urq5QqVRQqVSoW7duvu+rVCrMmDFDqhNERERlroIvhJwzZw6effZZzJs3D0OGDEHTpk0BAFu2bNFNWyhlcNKwf/9+CCHQqVMnbN68Ge7u7rrvqdVq+Pn5oVq1alKdICIiKmsq8XAr6TnMVWhoKO7cuYOUlBS4uf23anbkyJFwcFC+OB5QkDTkVbe8du0aatasWWBFrhs3bqBmzZpSHZGl0Wh09S+A/DXNiYiIClQBn9PwqE6dOuH777/XSxgAwN3dHf369cO+ffsUn1PxQsiAgAAkJibm23/37l3UqlVLcQdKigWriIiI8ouJiUFWVv7bxDMzM3Hw4EGpcyq+g7mwAlIPHjyAra2tVCdKYsqUKZg4caLu65SUFCYORERUvAq6puHMmTO6f587d06vFHdubi6io6NRvXp1qXMbnDTk/cesUqnwwQcf6BXAyM3NxdGjRxEcHCzViZJgwSoiIpJSQacngoODdTcudOrUKd/37ezssHjxYqlzG5w0nDp1CsDDkYY///xTrxiGWq1G06ZN8dZbb0l1goiIiIzj2rVrEEIgICAAv//+O6pUqaL7nlqthpeXFywtLaXOrejuCQAYOnQoPv30Uzg7O0s1SEREZBYq6EhDXpltrdb4j+RWvKYhKioKAHD58mVcuXIFHTp0gJ2dHYQQBd5RQUREZJYqYNKwZcsW9OzZE9bW1tiyZUuRx+Y9IVIJxUnDvXv3MGDAAOzfvx8qlQp///03AgIC8Morr8DV1dXo5asfr3pJREREBevXrx8SEhLg5eWFfv36FXqcSqVCbm6u4vMrThomTJgAa2tr3LhxA0FBQbr9AwcOxBtvvGH0pEGWykJAZaksBXSw1xR/0GMyOskN/9jvkaty6TFdrhqew0fKn18Rf8lHqi3rE39JxaV1Dir+oMekp8ktgtW4y5U+rBV+VnHMrZBGUm1lhSuvOgkAd2NdpeIA5VVeHQKTpVpKTVFerRKAxE3igIVcQVPU2HpbKu78ZFfFMXVXylWUVeXK/Zl7aZxcJVRb6xzFMQ88lL02kSF3LaRUwLsnHp2SMIvpiV27dmHnzp26+tx56tSpg3/++cdoHSMiIipNFf2JkKVBcd6elpamd7tlnjt37vDWRyIiIjOyd+9e9OrVC4GBgahduzZ69eqFPXv2SJ9PcdLQoUMHvRLYKpUKWq0W8+bNQ8eOHaU7QkREVKYqeGnszz//HD169ICTkxNef/11jB8/Hs7Oznj66afx+eefS51T8fTEvHnzEBoaiuPHjyMrKwuTJ0/G2bNnce/ePfz6669SnSAiIiLjioyMxMKFCzF27FjdvvHjx6Ndu3b48MMP9fYbSvFIQ4MGDXDmzBm0bt0aXbt2RVpaGvr3749Tp04hMDBQ0blCQ0Mxbtw4TJgwAW5ubvD29saKFSuQlpaGoUOHwsnJCYGBgdixY4fSbhIRERVJhf/WNUhvpn4RRUhJSUGPHj3y7e/WrZt0cUeJtchA1apVMWPGDGzduhXbt2/H7Nmz4eMjt9J+zZo18PT0xO+//45x48Zh9OjRGDBgANq2bYuTJ0+ie/fuGDx4MNLT0wuM12g0SElJ0duIiIgquz59+uCHH37It/+nn35C7969pc5p0PTEo8UvitOkSRNFHWjatCnee+89AA+LT3388cfw9PTEiBEjAAAffPABli5dijNnzuDJJ5/MFx8ZGYkZM2YoapOIiKgi3nL52Wef6f4dFBSEDz/8EDExMWjTpg0A4MiRI/j111/x5ptvSp3foKQhr/hFYRUu88g8LOLRJMPS0hIeHh5o3Lixbp+3tzcA4Pbtgu+ZZpVLIiKSUgGfCLlw4UK9r93c3HDu3DmcO3dOt8/V1RWrVq3S/cGuhEFJw7Vr1xSf2FDW1voPGVGpVHr78h5NXdhDKljlkoiI6KHS/P8aMDBpyCt+QUREVGFUwJGG0ia1EJKIiKi8K/GdE0Z4oqSxffzxx0hLSzPo2KNHj2Lbtm2Kzs+kgYiIqII4d+4c/Pz8MHr0aOzYsQOJiYm67+Xk5ODMmTNYsmQJ2rZtixdffBHOzs6Kzi9XtcdICqpgef369Xz7iluASUREpFgFnJ5Yu3Ytzpw5g//9738ICwtDcnIyLC0tYWNjo3t0QbNmzTBy5EgMGTJE8ZpAkyYN5iY51kVxjG28pVRbSe3lKhh6vpAkFXfjppfiGAtbqaZQZZfcLUiZM5T/9okkuaqft1rJDbKlhytf35N9SK6P1TcqrygIALFd5Srb2dooLwdp84OrVFvpDaTC4LdDeSXOKy/IXf9/PpJbYG39t/IKkpdflPso9tsu9x6x/VvumiRqlH/eefooq4Sam65BrOJWJFXApAF4eFfi8uXLsWzZMpw5cwbXr19HRkYGPD09ERwcDE9PT+lzK36nBgQE4NixY/Dw8NDbn5SUhObNm+Pq1avSnSEiIiLjUKlUaNq0KZo2bWq0cyr+c+v69esFPotBo9Hg5s2bRukUERFRaTPnhZDXr1/H8OHDUatWLdjZ2SEwMBDTpk1DVlZW6TRoIINHGrZs2aL7986dO+Hi8t9Qfm5uLvbu3Qt/f3+jdo6IiKjUmPETIS9cuACtVovly5ejdu3a+OuvvzBixAikpaVh/vz5pdKmIQxOGvr16wfg4XDHkCFD9L5nbW0Nf39/LFiwoMQdCg0NRZMmTWBra4svvvgCarUao0aNwvTp00t8biIiIh0zXtPQo0cPvWJTAQEBuHjxIpYuXVo+koa8JzLWqlULx44dK9FCiuKsWbMGEydOxNGjR3H48GFERESgXbt26Nq1a6m1SUREJOvxYoml8bTi5ORkuLu7G/WcSile03Dt2rV8CUNSUpKx+gPg4crPadOmoU6dOggPD0fLli2xd+/eAo9llUsiIpJhzDUNvr6+cHFx0W2RkZFG7euVK1ewePFijBo1yqjnVUrx3RNz5syBv78/Bg4cCAAYMGAANm/eDB8fH2zfvt0oqzQfr5Tp4+NTaMEqVrkkIiIpRpyeiI2N1XtQUmGjDNOnTy/2/6xjx46hZcuWuq/j4uLQo0cPDBgwAK+88kqxXerfv78BHQe+//57g457lOKkYfny5fjqq68AALt378aePXsQHR2NTZs2YdKkSdi1a5fiTjyuoCJWhRWsYpVLIiIyNWdnZ4Oerjh27Fi8+OKLRR7z6E0FcXFx6NixI9q0aYMVK1YY1JdHb1QwNsVJQ3x8vO4/5a1bt+KFF15At27d4O/vjyeeeMLoHSwOq1wSEZEUY9wyqTDe09PT4DWBN2/eRMeOHdGiRQtERUXBwsKwFQVRUVHKOqWA4jUNbm5uiI19+Lyu6OhodOnSBcDDRz0X9PwGIiIisySMtJWCuLg4hIaGwtfXF/Pnz0diYiISEhKQkJBQOg0aSPFIQ//+/fHyyy+jTp06uHv3Lnr27AkAOH36NGrXrm30DhIREVU2u3btwuXLl3H58mXUqFFD73vF1WMaNmyYQW2sWrVKcb8UJw0LFy5ErVq1cOPGDcydOxeOjo4AHk5bjBkzRnEHHldQEasff/yxxOclIiLSY8bPaYiIiEBERIRU7OrVq+Hn54dmzZoZveCjoqQhOzsbI0eOxPvvv4+AgAC9702YMMGY/SIiIipVxngMdGk9RrokRo0ahY0bN+Lq1asYNmwYBg0aZLTnO6iEwjTE1dUVJ0+ezJc0mIuUlBS4uLigxpLpsLBTVqbR9YTyym9JTZRXBgQAVZZclUVhJ7duxPkv5a/NpWe8VFu3j1aVistyU16d0UIj9whXp+ty1z+7k7KKfQCQdttBqi23asrbAgAh+VjbpPjiV34/ztpFI9WW/W9y1yS3U5LimPR/lL8uANA6yP2uhTa+oDjmREKN4g8qgIONXB2CxHty18T6bzvFMZneyipxajMy8e8bHyA5OdmguxFk5P0/EfjuR7C0lSzn+/9yMzNx5aN3S7W/MjQaDb7//nusWrUKv/32G5555hkMHz4c3bp1g0ol/+hrxZ+czz77LKcLiIiIzJiNjQ1eeukl7N69G+fOnUPDhg0xZswY+Pn54cGDB9LnVbymoXbt2pg1axZ+++03tGjRAg4O+n8xjB8/XrozREREZcaM1zQYk0qlgkqlghCi0GceGUpx0vDFF1/A1dUVJ06cwIkTJ/J1jEkDERGVBxV1TQOgPz1x6NAh9OrVC59//jl69Ohh8PMeCqI4abh27Zp0Y4bQaDSYNGkSNm7ciJSUFLRs2RILFy5Eq1atSrVdIiKiimDMmDHYuHEjatasiaFDh2Ljxo3w8PAwyrkVJw2lbfLkydi8eTPWrFkDPz8/zJ07F927d8fly5cLXP2p0Wig0fy3GIsFq4iIyGBmOlJQEsuWLUPNmjVRq1YtHDhwAAcOHCjwuFKrPTFx4kTMmjULDg4OenUeCvLJJ58o7kSetLQ0LF26FKtXr9Y9NGrlypXYvXs3vvzyS0yaNClfDAtWERGRlAq6piE8PLxEd0gUxaCk4dSpU8jOztb9uzAl7eSVK1eQnZ2Ndu3a6fZZW1ujdevWOH/+fIExLFhFRET0n9WrV5fauQ1KGvbv34+rV6/CxcUF+/fvL7XO5D0y4vHkQwhRaELCglVERCSjIi+ELC0GL6GsU6cOEhMTdV8PHDgQt27dMmpnateuDbVajUOHDun2ZWdn4/jx4wgKCjJqW0REVMmZccEqc2Vw0vD4gyO3b9+OtLQ0o3bGwcEBo0ePxqRJkxAdHY1z585hxIgRSE9Px/Dhw43aFhERESljdndPfPzxx9BqtRg8eDBSU1PRsmVL7Ny5E25ubqbuGhERVSCcnlDO4KQh74lSj+8zNltbW3z22Wf47LPPjH5uIiIinQp690RpMjhpEEIgIiJCt+gwMzMTo0aNyvcYaZn7PkuFRAqZ0yVJcTOOv7kqjgEAqwypMHgdlwu8OVl5XNIuH6m2ArYnFn9QAc6/rnw0ycIzU6qtZC9LqTiHIy6KY6p1SpBqK+mAXOGvmjuSpOLS3lZefCrY91+pti6hnlScTDEura3cY3PVt+UGYo/E+iuO0WRYS7VVbaHck/1uvSH3B1+OvfL/IT2OK/tdy82yhNy7isqCwb8VQ4YM0ft60KBBRu8MERFRmeFIg2IGJw1RUVGl2Q8iIqIyxTUNypndQkgiIqIywZEGxeRLXREREVGlYlYjDaGhoWjUqBEA4KuvvoKlpSVGjx6NWbNmldpztImIqJLiSINiZjfSsGbNGlhZWeHo0aP47LPPsHDhQnzxxReFHq/RaJCSkqK3ERERFSdvTUNJt8rE7JIGX19fLFy4EPXq1UNYWBjGjRuHhQsXFnp8ZGQkXFxcdBuLVREREZUOs0sannzySb2piDZt2uDvv/9Gbm5ugcdPmTIFycnJui02NrasukpEROUZa08oZlZrGmSwyiUREcngLZfKmd1Iw5EjR/J9XadOHVhayj3Bj4iIiIzD7JKG2NhYTJw4ERcvXsSGDRuwePFivP7666buFhERVTScnlDM7KYnwsPDkZGRgdatW8PS0hLjxo3DyJEjTd0tIiKqaHjLpWJmlzRYW1tj0aJFWLp0qam7QkRERI8wu6TBWBwu2sBS4QLJtEYSKWNjuSqLspJbS/7IHkjENZV7bReaOknFqZCjOCYnu2zXuqQ1Un5N0m67yjUWJHf9LwbZSsW571O+oPj4U35SbaGV5O/NA+V9VNkrf18BQLZk3CfBPyiOmXj4Bam2Lr1RtovAc6sqr4R6T2GxVm2GBliruBkpqv/fSnqOyqTCJg1ERERF4vSEYmaVNMTExJi6C0REVEnwlkvlzO7uCSIiIjJPZpc0pKWlITw8HI6OjvDx8cGCBQsQGhqKCRMmmLprRERUkZSTWy41Gg2Cg4OhUqlw+vTp0m+wCGaXNEyaNAn79+/HDz/8gF27diEmJgYnTpwwdbeIiKgiMvOEAQAmT56MatWqlU1jxTCrNQ0PHjzAl19+ibVr16Jr164AHla9rFGjRqExGo0GGs1/K3pZ5ZKIiCqKHTt2YNeuXdi8eTN27Nhh6u6YV9Jw5coVZGVloU2bNrp97u7uqFevXqExkZGRmDFjRll0j4iIKhBjLoR8/A9WY9RFunXrFkaMGIEff/wR9vb2JTqXsZjV9IQQyn96rHJJRERSjLimwdfXFy4uLrotMjKyZF0TAhERERg1ahRatmxZonMZk1mNNNSuXRvW1tY4cuQIatasCQC4f/8+Ll26hJCQkAJjWOWSiIhMLTY2Fs7OzrqvC/t/afr06cWOjh87dgy//fYbUlJSMGXKFKP2s6TMKmlwdHTE8OHDMWnSJHh4eMDb2xtTp06FhYVZDYgQEVEFYMzpCWdnZ72koTBjx47Fiy++WOQx/v7+mD17No4cOZIv+WjZsiXCwsKwZs0a6T6XhFklDQAwb948PHjwAH369IGTkxPefPNNJCcnm7pbRERU0ZjgiZCenp7w9PQs9rjPPvsMs2fP1n0dFxeH7t2745tvvsETTzyhtJdGY3ZJg6OjI9atW4d169bp9m3bts2EPSIiIipbeVP0eRwdHQEAgYGBRd5RWNrMLmkgIiIqC3yMtHIVNmnIbJQBC3uFP80c5fXK1P/ILcLMsZMKQ66rXOU97+r3FcfcP1lFqi1tQIZUnOfPyqsz3n82Taot/zlaqbi/J6gVx1hdl6s6me0i10frVLk1QPfaK69gaHVT8v1fXXlbAGBhpfyaaLMl10QlW0uFTdwVJteeBJdzclVePc/I/Y7em6z898358+Ln+R+Vk2OBMrsHrhwVrPL395e6w9DYykXSwEJWRERkdOUoaTAXvC2BiIiIDFIuRhryZGVlQa1WPjxMRET0OK5pUM6sk4bQ0FA0atQIarUaa9euRcOGDXHgwAFTd4uIiCoCTk8oZtZJA/CwYNXo0aPx66+/FrgIhAWriIiIyobZJw21a9fG3LlzC/0+C1YREZEMlRBQlfCOhJLGlzdmvxCyuEIdLFhFRERSjFiwqrIw+5EGBweHIr/PglVERERlw+yTBiIiotLAuyeUY9JARESVE++eUMzs1zQQERGReTDrkQY+PpqIiEoLpyeUM+ukoSScnDJgaa+suM39my6K27FNVF7kCgCSG0gWnjogV6AmsYeT4hi3f6SagvVZuQJNd3pnKo7JuS/XVo5zrlScNlX5r4zsh4pFltx7SyVReO1hoPIQrZ9c4SMfd7nnqbiMUx5z/i0PqbbsqskVQ8tIUv6erLFd7vf6Viu5N1fyE3LvEftfPRXHZCn8XcvJlrsWUjg9oViFTRqIiIiKwpEG5bimgYiIiAzCkQYiIqqcOD2hGJMGIiKqtCrb9EJJme30xHfffYfGjRvDzs4OHh4e6NKlC9LS5BYmERERUcmZ5UhDfHw8XnrpJcydOxfPPvssUlNTcfDgQVa5JCIi4xHi4VbSc1QiZps05OTkoH///vDz8wMANG7cuMBjWeWSiIhk8O4J5cxyeqJp06bo3LkzGjdujAEDBmDlypW4f/9+gceyyiUREVHZMMukwdLSErt378aOHTvQoEEDLF68GPXq1cO1a9fyHWtjYwNnZ2e9jYiIqFgsja2YWSYNAKBSqdCuXTvMmDEDp06dglqtxg8//GDqbhERUQWh0hpnq0zMck3D0aNHsXfvXnTr1g1eXl44evQoEhMTERQUZOquERERVVpmmTQ4Ozvjl19+waJFi5CSkgI/Pz8sWLAAPXv2NHXXiIioouDDnRQzy6QhKCgI0dHRpu4GERFVYLx7QjmzTBqMQZNtBctshS/PQvlP3/qB3DvGKkWukptFtlx72jTlP2qrDLm2XM/LPSvDZ3Si4phTf/tJtWWdmC4VV3+J8pjLg9yk2lInyS05qrkjWSru6iTl70nfldZSbV0b4ioVl/CaWnGMpZPy6qkAkH1VeWVYAFDZKP+9udtAcnlZGT8jINNT+QR+Wk1lfdRmCKCslq/xOQ2Kme1CSCIiIjIvZp00hIaGYsKECabuBhERVUB50xMl3SqTCjs9QUREVCQuhFTMbEcaIiIicODAAXz66adQqVRQqVS4fv26qbtFRERUaZntSMOnn36KS5cuoVGjRpg5cyYAoEqVKibuFRERVRS8e0I5sx1pcHFxgVqthr29PapWrYqqVavC0jL/6m6NRoOUlBS9jYiIqFh5d0+UdCtF27ZtwxNPPAE7Ozt4enqif//+pdpeccw2aTBUZGQkXFxcdJuvr6+pu0RERFRimzdvxuDBgzF06FD88ccf+PXXX/Hyyy+btE9mOz1hqClTpmDixIm6r1NSUpg4EBFRscx5eiInJwevv/465s2bh+HDh+v216tXr3QaNJBZjzSo1Wrk5uYWeQyrXBIRkRQzrnJ58uRJ3Lx5ExYWFmjWrBl8fHzQs2dPnD17tnQaNJBZJw3+/v44evQorl+/jjt37kCrrWTlxIiIqFx4fG2dRqMp0fmuXr0KAJg+fTree+89bN26FW5ubggJCcG9e/eM0WUpZp00vPXWW7C0tESDBg1QpUoV3Lhxw9RdIiKiCsKYD3fy9fXVW18XGRlZYJvTp0/XPUagsO348eO6P5KnTp2K5557Di1atEBUVBRUKhW+/fbbsrpE+Zj1moa6devi8OHDpu4GERFVRFrxcCvpOQDExsbqTY/b2NgUePjYsWPx4osvFnlKf39/pKamAgAaNGigd86AgACT/gFt1kkDERFRqTHiEyENXVPn6ekJT0/PYo9r0aIFbGxscPHiRbRv3x4AkJ2djevXr8PPT65QnzFU2KQhJ8sSWitlL8/uX+UV++6GylXQs7LJkYpLsLOTirONU/7aElvIrSG531vubZV13l9xjNV9ubbiO7pLxalTlH/C2NdPkmorNUXuZ325gVzlyYBPlb82iw8TpNqy3Sf3oefWXnl7WTly75H0wGypOPttyhdj32kp93lQf4ncc2muviRXeTXLo+iF6QVxq6as6mpuugb/Km6l4nF2dsaoUaMwbdo0+Pr6ws/PD/PmzQMADBgwwGT9qrBJAxERUVFUMMItl0bpScHmzZsHKysrDB48GBkZGXjiiSewb98+uLnJJX3GwKSBiIgqJ2M80bEUnwhpbW2N+fPnY/78+aXWhlJmffcEERERmQ+zShr8/f2xaNEivX3BwcGYPn26SfpDREQVlzFvuawsyv30hEaj0XuIBgtWERGRQYx490RlYVYjDTJYsIqIiKhslPukYcqUKUhOTtZtsbGxpu4SERGVAyohjLJVJmY1PWFhYQHx2A8gO7voe6VtbGwKffIWERFRobT/v5X0HJWIWY00VKlSBfHx8bqvU1JScO3aNRP2iIiIiPKYVdLQqVMnrFu3DgcPHsRff/2FIUOGwNLS0tTdIiKiCojTE8qZ1fTElClTcPXqVfTq1QsuLi6YNWsWRxqIiKh08O4JxcwqaXB2dsY333yjt2/IkCEm6g0REVVoZv5ESHNkVkmDMTn8ag9Lta2imPvByovGqCQfPO7+s71U3O3uWVJxHZ48ozhm94UgqbZqv35LKu783OqKY+y9H0i15XLISSou9gWJwkKShadk1R0vdwfRhRm1FccEflBFqq3MkXKF3mznKX/mvtM9ubbuviG3wFptp/xDwfO43DTsxclynyOApvhDCuB8StlnKgAk2TsoOl6bwSlpc1ZhkwYiIqKiGOOJjnwiJBERUWXA6QnFzOruCSIiIjJfHGkgIqJKSaV9uJX0HJWJWY00LF++HNWrV4dWq/9T6NOnD++iICIi48qbnijpVomYVdIwYMAA3LlzB/v379ftu3//Pnbu3ImwsLACYzQaDVJSUvQ2IiIiMj6zShrc3d3Ro0cPfP3117p93377Ldzd3dG5c+cCY1jlkoiIpAgjbZWIWSUNABAWFobNmzdDo3l4H/H69evx4osvFvo4aVa5JCIiGXyMtHJmtxCyd+/e0Gq12LZtG1q1aoWDBw/ik08+KfR4VrkkIiIqG2aXNNjZ2aF///5Yv349Ll++jLp166JFixam7hYREVU0fE6DYmaXNAAPpyh69+6Ns2fPYtCgQabuDhERVUQCQElvmaxcOYN5Jg2dOnWCu7s7Ll68iJdfftnU3SEiogrIGGsSuKbBDFhaWiIuLs7U3SAiIqJHmGXSYAzJrbJgYafs5pCq3kmK28nc5q04BgBsk7Kl4lx+V15lDgAOO/srjnH/RW6BqTZVrvJkfd8ExTHX77pLtSXL7Vfl10RrJVcKNb2a3F8wMtUqAcDCVXkFVauTl6TaAupLRcV2USuOUeUqjwGAeguSpOJynJSPd1slyVXizHKWe/+nNpdrz+628tdmfVDZ70xulkCZ3QMnYIQ1DUbpSblRYZMGIiKiInEhpGJm95wGIiIiMk8caSAiospJC0Bu9lD/HJUIkwYiIqqUePeEciafnvj555/h6uqqq2x5+vRpqFQqTJo0SXfMq6++ipdeeslUXSQiIiKYQdLQoUMHpKam4tSpUwCAAwcOwNPTEwcOHNAdExMTg5CQkALjWeWSiIiksDS2YiZPGlxcXBAcHIyYmBgADxOEN954A3/88QdSU1ORkJCAS5cuITQ0tMB4VrkkIiIpTBoUM3nSAAChoaGIiYmBEAIHDx5E37590ahRIxw6dAj79++Ht7c36tcv+L5uVrkkIiIqG2aTNBw8eBB//PEHLCws0KBBA4SEhODAgQNFTk0AD6tcOjs7621ERETFMvORhkuXLqFv377w9PSEs7Mz2rVrh/3795dae4Ywi6Qhb13DokWLEBISApVKhZCQEMTExBSbNBAREUnRGmkrJc888wxycnKwb98+nDhxAsHBwejVqxcSEpQ/PddYzCJpyFvX8NVXX+nWLnTo0AEnT54scj0DERGRrLxbLku6lYY7d+7g8uXLeOedd9CkSRPUqVMHH3/8MdLT03H27NlSadMQZpE0AEDHjh2Rm5urSxDc3NzQoEEDVKlSBUFBQabtHBERUREev4tPo9GU6HweHh4ICgrC2rVrkZaWhpycHCxfvhze3t5o0aKFkXqtnNkkDfPnz4cQAg0bNtTtO336NG7fvg2VqqSP7CIiInqMEdc0+Pr66t3JFxkZWaKuqVQq7N69G6dOnYKTkxNsbW2xcOFCREdHw9XV1QgvXk6FfSKkhXUuLNS5imIS7ylfRGnjpDgEANBi5gmpuB/OBkvFiVuOimPsq8gla/9MaCoVl3NY+TCfZR25ipqxL+RIxQHK42QqYwJAtm/J/lJRymOn8gqq9Q4qr4wJAC6Z16XizlxSXh3T+brcpPPt2co+P/LcS7BWHKOykfv7zf2g3NC4g5Nclcu7jZW/l3OrKXsfazM0wGrFzcjRCkBVwukF7cP42NhYvYX4NjYFX6vp06djxowZRZ7y2LFjaNGiBcaMGQMvLy8cPHgQdnZ2+OKLL9CrVy8cO3YMPj4+Jeu3pAqbNBAREZUVQ+/eGzt2LF588cUij/H398e+ffuwdetW3L9/X3feJUuWYPfu3VizZg3eeecdo/RbKSYNRERUOZmgNLanpyc8PT2LPS49PR0AYGGhPwplYWGhK7tgCmazpoGIiKhsGWM9Q+ncPdGmTRu4ublhyJAh+OOPP3Dp0iVMmjQJ165dwzPPPFMqbRrCrJKGxMREVK1aFR999JFu39GjR6FWq7Fr1y4T9oyIiKjseHp6Ijo6Gg8ePECnTp3QsmVLHDp0CD/99BOaNpVbN2YMZjU9UaVKFaxatQr9+vVDt27dUL9+fQwaNAhjxoxBt27dTN09IiKqSEwwPaFEy5YtsXPnzlI7vwyzShoA4Omnn8aIESMQFhaGVq1awdbWFh9//HGhx2s0Gr37YVnlkoiIDKI1wvSClgWrTG7+/PnIycnBpk2bsH79etjaFn4rGKtcEhERlQ2zTBquXr2KuLg4aLVa/PPPP0UeyyqXREQkRWiNs1UiZjc9kZWVhbCwMAwcOBD169fH8OHD8eeff8Lb27vA421sbAp9iAYREVGhzHxNgzkyu6Rh6tSpSE5OxmeffQZHR0fs2LEDw4cPx9atW03dNSIiqki4pkExs5qeiImJwaJFi7Bu3To4OzvDwsIC69atw6FDh7B06VJTd4+IiKhSM6uRhtDQUGRnZ+vtq1mzJpKSkkzTISIiqrg4PaGYWSUNREREZUbACEmDUXpSblTYpMHJMROW9sp+mhknPRS3k9kwQ3EMAOza+KRUnFN28ccUJLWZ8qp2OY5ybw/vY3LVAW1vK6/qeL2X8uqdAOAfI3ch7zRSK47RKi96CACwuaK86iQAuF2QW83tOeaa4pi937SWaiutkVyVRWs75Z/Qt56SfD8eK74+QEG8rivvo8ZF7k2S6if3P1Z2qtx7y/OC8pi0ZGVt5ZZtcVdSqMImDUREREXi9IRiTBqIiKhy0moBlPA5CyasOGkKZnX3BBEREZkvs0oa1q5dCw8PD71aEgDw3HPPITw83ES9IiKiCqmkZbGNMb1RzphV0jBgwADk5uZiy5Ytun137tzB1q1bMXTo0AJjNBoNUlJS9DYiIqJiMWlQzKySBjs7O7z88suIiorS7Vu/fj1q1KiB0NDQAmNYsIqIiKhsmFXSAAAjRozArl27cPPmTQBAVFQUIiIioFKpCjyeBauIiEiKVhhnq0TM7u6JZs2aoWnTpli7di26d++OP//8Ez///HOhx7NgFRERyRBCC1HCKpUljS9vzC5pAIBXXnkFCxcuxM2bN9GlSxdOORARkfEJI4wUcE2D6YWFheHmzZtYuXIlhg0bZuruEBEREcw0aXB2dsZzzz0HR0dH9OvXz9TdISKiioh3TyhmltMTABAfH4+wsDCuVyAiotKh1QKqEq5J4JoG07p37x527dqFffv24fPPP5c+z4OLbrCwVVYoxadNvOJ2knb6KI4BgAfBcgV7ZFXzSlIcE6dylWor6Z5cMRwXi4LvkCmKa9M7Um1dr+EkFQco/7mJNLlfM/sbcnEJz2TJxV2rrjjG/a7cX1lpUlFAtq/yakYev8j94eHx1wOpuH87Ki+iluUmdx1zq5ZtdSeXwf8qjvk9qPCF7AVJSdXCbYHiZqiMmF3S0Lx5c9y/fx9z5sxBvXr1TN0dIiKqqIRAiWtbc3rCtK5fv27qLhARUSUgtFqIEk5PVLZbLs1yISQRERGZH7MbaSAiIioTnJ5QzOxGGq5fvw6VSpVvK6z2BBERkRQ+Rloxsxtp8PX1RXz8f3cxJCQkoEuXLujQoUOBx2s0Gr1S2qxySUREVDrMLmmwtLRE1apVAQCZmZno168f2rRpg+nTpxd4fGRkJGbMmFGGPSQiogpBCAAlfU5D5RppMLvpiUcNHz4cqamp+Prrr2FhUXBXWeWSiIhkCK0wylaZmN1IQ57Zs2cjOjoav//+O5ycCn8QD6tcEhGRFKFFyUcaKtctl2aZNGzevBkzZ87Ejh07EBgYaOruEBEREcwwafjrr78QHh6Ot99+Gw0bNkRCQgIAQK1Ww93d3cS9IyKiikJoBYSqZNMLgmsaTOv48eNIT0/H7Nmz4ePjo9v69+9v6q4REVFFIrTG2SoRsxtpiIiIQEREhHR8XtanzVReWCgnTXnxl1yNXOEpbUbZFqySeW2yfcyVrKGTk52rvK10uca0GdZScTJEhtyvWa5GLk76vaVVXjAsN0vuA1ObUXaFlnKz5P4SzMmRff8r/7lpM+X6WJbXEZD7HElJVfYeSXnw8Piy+As+B9klfrZTDrKN05lyQiUq2NjKv//+C19fX1N3g4iISiA2NhY1atQolXNnZmaiVq1auunvkqpatSquXbsGW4WVlcujCpc0aLVaxMXFwcnJCSqV/l9OKSkp8PX1RWxsLJydnQ06n0xMWcexj6aNYx/ZR3OKKw99LCpOCIHU1FRUq1at0FvtjSEzMxNZWXJl5B+nVqsrRcIAmOH0RElZWFgUm506OzsrenPLxpR1HPto2jj20Thx7KNx4spDHwuLc3FxUXwepWxtbSvNf/TGZHYLIYmIiMg8MWkgIiIig1SqpMHGxgbTpk1T9ARJmZiyjmMfTRvHPrKP5hRXHvpYkjgyrQq3EJKIiIhKR6UaaSAiIiJ5TBqIiIjIIEwaiIiIyCBMGowsNDQUEyZMqNB9EEJg5MiRcHd3h0qlwunTp43eRklfg2x8REQE+vXrV6ptGCueiKisVbiHO1Hpi46OxurVqxETE4OAgAB4enqaukv5fP/997C2Vl5f4tNPP610VevIcKGhoQgODsaiRYtM3RUik2DSQIpduXIFPj4+aNu2ram7UijZMupl8SQ6Kj1ZWVlQq9Wm7gZRhcXpiSJER0ejffv2cHV1hYeHB3r16oUrV64UG5eTk4OxY8fq4t577z2D/nrVarWYM2cOateuDRsbG9SsWRMffvhhkTFpaWkIDw+Ho6MjfHx8sGDBAoNemxACc+fORUBAAOzs7NC0aVN89913xcZFRERg3LhxuHHjBlQqFfz9/YuNSU1NRVhYGBwcHODj44OFCxcaNDSv1WoxefJkuLu7o2rVqpg+fbpBrw0om+mJx0VHR8PFxQVr166Vii9IaGgoxo0bhwkTJsDNzQ3e3t5YsWIF0tLSMHToUDg5OSEwMBA7duwoNH78+PGKr6NGo8H48ePh5eUFW1tbtG/fHseOHSu2r2PHjpV67xf0PjbkZ5jX5sSJE+Hp6YmuXbsW2xYAfPfdd2jcuDHs7Ozg4eGBLl26IC0trciYiIgIHDhwAJ9++ilUKhVUKhWuX79eZIy/v3++UYng4OAifwbLly9H9erVodXqV4fs06cPhgwZku/4n3/+Ga6urrrjT58+DZVKhUmTJumOefXVV/HSSy/li01MTETVqlXx0Ucf6fYdPXoUarUau3btKrSPa9euhYeHBzQa/aqXzz33HMLDwwuNu379uu7aPbqFhoYWGkNmRlChvvvuO7F582Zx6dIlcerUKdG7d2/RuHFjkZubW2hMSEiIcHR0FK+//rq4cOGC+Oqrr4S9vb1YsWJFse1NnjxZuLm5idWrV4vLly+LgwcPipUrVxYZM3r0aFGjRg2xa9cucebMGdGrVy9d+0V59913Rf369UV0dLS4cuWKiIqKEjY2NiImJqbIuKSkJDFz5kxRo0YNER8fL27fvl3s63rllVeEn5+f2LNnj/jzzz/Fs88+K5ycnIrsY0hIiHB2dhbTp08Xly5dEmvWrBEqlUrs2rWr2Pby4ou7BgUZMmSI6Nu3r+I2NmzYIJycnMSPP/5ocFuG9DEkJEQ4OTmJWbNmiUuXLolZs2YJCwsL0bNnT7FixQpx6dIlMXr0aOHh4SHS0tIKjJe5juPHjxfVqlUT27dvF2fPnhVDhgwRbm5u4u7du0X2Vfa9L/s+zmtz0qRJ4sKFC+L8+fPFthUXFyesrKzEJ598Iq5duybOnDkj/ve//4nU1NQi45KSkkSbNm3EiBEjRHx8vIiPjxc5OTlFxvj5+YmFCxfq7WvatKmYNm1aoTF3794VarVa7NmzR7fv3r17Qq1Wi507dxbYLwsLC3H8+HEhhBCLFi0Snp6eolWrVrpj6tatK5YuXVpge9u2bRPW1tbi2LFjIjU1VdSuXbvY656eni5cXFzEpk2bdPsSExOFWq0W+/btKzQuJydHd+3i4+PFqVOnhIeHh3j//feLbI/MB5MGBW7fvi0AiD///LPQY0JCQkRQUJDQarW6fW+//bYICgoq8twpKSnCxsam2CThUampqUKtVouNGzfq9t29e1fY2dkV+Uv/4MEDYWtrK3777Te9/cOHDxcvvfRSse0uXLhQ+Pn5GdTHlJQUYW1tLb799lvdvqSkJGFvb19s0tC+fXu9fa1atRJvv/22Qe2WZdLwv//9T7i4uBT5YSnbx8evQ05OjnBwcBCDBw/W7YuPjxcAxOHDh4uNF6L46/jgwQNhbW0t1q9fr9uXlZUlqlWrJubOnVtkX2Xe+7Lv47w2g4ODizzmcSdOnBAAxPXr1xXF5bWn5H0lkzQIIUSfPn3EsGHDdF8vX75cVK1atdAkpXnz5mL+/PlCCCH69esnPvzwQ6FWq0VKSoru/VFUQjVmzBhRt25dERYWJho1aiQyMjKKfW2jR48WPXv21H29aNEiERAQoPfzL0pGRoZ44oknRK9evYr8Q4zMC6cninDlyhW8/PLLCAgIgLOzM2rVqgUAuHHjRpFxTz75pF5Z7jZt2uDvv/9Gbm5uoTHnz5+HRqNB586dFfUvKysLbdq00e1zd3dHvXr1iow7d+4cMjMz0bVrVzg6Ouq2tWvXGjT9osTVq1eRnZ2N1q1b6/a5uLgU20cAaNKkid7XPj4+uH37tlH7V1KbN2/GhAkTsGvXLnTs2LFU2nj0OlhaWsLDwwONGzfW7fP29gaAQq+N0ut45coVZGdno127drp91tbWaN26Nc6fP19kX2Xe+7Lv4zwtW7Y06Lg8TZs2RefOndG4cWMMGDAAK1euxP379xWdo7SFhYVh8+bNuuH/9evX48UXX4SlpWWBx4eGhiImJgZCCBw8eBB9+/ZFo0aNcOjQIezfvx/e3t6oX79+oe3Nnz8fOTk52LRpE9avX29Q9ccRI0Zg165duHnzJgAgKioKERERej//ogwfPhypqan4+uuvS7UENhkXf1JF6N27N+7evYuVK1fi6NGjOHr0KAAYrQb7o+zs7BTHCMlV/nlzn9u2bcPp06d127lz5wxa16BEXh8f/yAxpO+P3/2gUqnyzfOaWnBwMKpUqYKoqKhSu+uioOvw6L68a1vYtVF6HYv6mRn6H4ISJb1uDg4Oio63tLTE7t27sWPHDjRo0ACLFy9GvXr1cO3atRL1oyAWFhb5Xl92dnaxcb1794ZWq8W2bdsQGxuLgwcPYtCgQYUeHxoaioMHD+KPP/6AhYUFGjRogJCQEBw4cAAxMTEICQkpsr2rV68iLi4OWq0W//zzj0GvrVmzZmjatCnWrl2LkydP4s8//0RERIRBsbNnz0Z0dDS2bNkCJycng2LIPDBpKMTdu3dx/vx5vPfee+jcuTOCgoIM/mvkyJEj+b6uU6dOoX8lAECdOnVgZ2eHvXv3GtzH2rVrw9raWq+9+/fv49KlS0XGNWjQADY2Nrhx4wZq166tt/n6+hrcviECAwNhbW2N33//XbcvJSUFf//9t1HbMZXAwEDs378fP/30E8aNG2fq7hhF7dq1oVarcejQId2+7OxsHD9+HEFBQUXGyrz3Zd/HJaFSqdCuXTvMmDEDp06dglqtxg8//FBsnFqtLnLU5HFVqlRBfHy87uuUlBSDkhM7Ozv0798f69evx4YNG1C3bl20aNGi0OM7dOiA1NRULFq0CCEhIVCpVAgJCUFMTEyxSUNWVhbCwsIwcOBAzJ49G8OHD8etW7cMen2vvPIKoqKisGrVKnTp0sWgz4/Nmzdj5syZ2LRpEwIDAw1qh8wHb7kshJubGzw8PLBixQr4+Pjgxo0beOeddwyKjY2NxcSJE/Hqq6/i5MmTWLx4cbF3Ndja2uLtt9/G5MmToVar0a5dOyQmJuLs2bMYPnx4gTGOjo4YPnw4Jk2aBA8PD3h7e2Pq1KnFDvU5OTnhrbfewhtvvAGtVov27dsjJSUFv/32GxwdHQtcoS3LyckJQ4YMwaRJk+Du7g4vLy9MmzYNFhYWpfJXqynUrVsX+/fvR2hoKKysrMr9PfwODg4YPXq07mdWs2ZNzJ07F+np6YW+F/PIvPdl38eyjh49ir1796Jbt27w8vLC0aNHkZiYWGxCBDy8G+Lo0aO4fv06HB0d4e7uXmQ/O3XqhNWrV6N3795wc3PD+++/X2QC9aiwsDD07t0bZ8+eLXKUAXg45RccHIyvvvoKn376KYCHicSAAQOQnZ1d5N0JU6dORXJyMj777DM4Ojpix44dGD58OLZu3WpQH9966y2sXLnSoLuG/vrrL4SHh+Ptt99Gw4YNkZCQAOBhMiZ7mzSVLSYNhbCwsMDGjRsxfvx4NGrUCPXq1cNnn31m0K1B4eHhyMjIQOvWrWFpaYlx48Zh5MiRxca9//77sLKywgcffIC4uDj4+Phg1KhRRcbMmzcPDx48QJ8+feDk5IQ333wTycnJxbY1a9YseHl5ITIyElevXoWrqyuaN2+Od999t9hYpT755BOMGjUKvXr1grOzMyZPnozY2FiD5k3Li3r16mHfvn0IDQ2FpaWlwbe+mquPP/4YWq0WgwcPRmpqKlq2bImdO3fCzc2tyDjZ977s+1iGs7MzfvnlFyxatAgpKSnw8/PDggUL0LNnz2Jj33rrLQwZMgQNGjRARkYGrl27VuRtx1OmTMHVq1fRq1cvuLi4YNasWQZPg3Tq1Anu7u64ePEiXn755WKP79ixI06ePKn7jHJzc0ODBg0QFxdXaEIUExODRYsWYf/+/XB2dgYArFu3Dk2aNMHSpUsxevToItt0dnbGc889h23bthl0q/Lx48eRnp6O2bNnY/bs2br9eaMiZP5YGpvKXFpaGqpXr44FCxYU+5drWXvppZdgaWmJr776ytRdKXeM/bREPn2xfOjatSuCgoLw2WefmborVAa4poFK3alTp7BhwwZcuXIFJ0+eRFhYGACgb9++Ju7Zf3JycnDu3DkcPnwYDRs2NHV3iMzevXv3sHHjRuzbtw+vvfaaqbtDZYTTE1Qm5s+fj4sXL0KtVqNFixY4ePCgWdWs+Ouvv9C2bVt07Nix2CkhIgKaN2+O+/fvY86cOQbfHkvlH6cniIiIyCCcniAiIiKDMGkgIiIigzBpICIiIoMwaSAiIiKDMGkgKqciIiKKfKDO6tWr4erqWmb9IaKKj0kDVWq3b9/Gq6++ipo1a8LGxgZVq1ZF9+7dcfjwYVN3zSRUKhV+/PFHvX3Tp09HcHCwSfpDROaFz2mgSu25555DdnY21qxZg4CAANy6dQt79+7FvXv3TN21Cic7OztfxU0iKl840kCVVlJSEg4dOoQ5c+agY8eO8PPzQ+vWrTFlyhQ888wzuuOSk5MxcuRIeHl5wdnZGZ06dcIff/yhd64tW7agZcuWsLW1haenJ/r376/73v379xEeHg43NzfY29ujZ8+eelU+86YRdu7ciaCgIDg6OqJHjx561RFzc3MxceJEuLq6wsPDA5MnT5YqKf3zzz+jRYsWsLW1RUBAAGbMmIGcnBwA0NVQePbZZ6FSqeDv74/Vq1djxowZ+OOPP6BSqaBSqbB69WqDrkveCMWqVasQEBAAGxubUisfTkRlg0kDVVqOjo5wdHTEjz/+CI1GU+AxQgg888wzSEhIwPbt23HixAk0b94cnTt31o1GbNu2Df3798czzzyDU6dOYe/evWjZsqXuHBERETh+/Di2bNmCw4cPQwiBp59+GtnZ2bpj0tPTMX/+fKxbtw6//PILbty4gbfeekv3/QULFmDVqlX48ssvcejQIdy7d8+gUs6P2rlzJwYNGoTx48fj3LlzWL58OVavXo0PP/wQAHDs2DEAQFRUFOLj43Hs2DEMHDgQb775Jho2bIj4+HjEx8dj4MCBBl0XALh8+TI2bdqEzZs34/Tp04r6S0RmSBBVYt99951wc3MTtra2om3btmLKlCnijz/+0H1/7969wtnZWWRmZurFBQYGiuXLlwshhGjTpo0ICwsr8PyXLl0SAMSvv/6q23fnzh1hZ2cnNm3aJIQQIioqSgAQly9f1h3zv//9T3h7e+u+9vHxER9//LHu6+zsbFGjRg3Rt2/fQl9bVFSUcHFx0X391FNPiY8++kjvmHXr1gkfHx/d1wDEDz/8oHfMtGnTRNOmTfX2GXJdpk2bJqytrcXt27cL7SMRlS8caaBK7bnnnkNcXBy2bNmC7t27IyYmBs2bN9cNwZ84cQIPHjyAh4eHbmTC0dER165dw5UrVwAAp0+fRufOnQs8//nz52FlZYUnnnhCt8/DwwP16tXD+fPndfvs7e0RGBio+9rHxwe3b98G8HAaID4+Hm3atNF938rKSm80wxAnTpzAzJkz9V7HiBEjEB8fj/T0dMXnKu66AICfnx+qVKmi6NxEZL64EJIqPVtbW3Tt2hVdu3bFBx98gFdeeQXTpk1DREQEtFotfHx8EBMTky8u73ZGOzu7Qs8tCpnDF0JApVLpvn58gaBKpTL6/L9Wq8WMGTP01lvksbW1VXyu4q4LADg4OCjtJhGZMSYNRI9p0KCB7rbD5s2bIyEhAVZWVrqFgo9r0qQJ9u7di6FDhxZ4rpycHBw9ehRt27YFANy9exeXLl1CUFCQQf1xcXGBj48Pjhw5gg4dOgB4WMo7bx2BoZo3b46LFy+idu3ahR5jbW2N3NxcvX1qtTrfPkOuCxFVPEwaqNK6e/cuBgwYgGHDhqFJkyZwcnLC8ePHMXfuXPTt2xcA0KVLF7Rp0wb9+vXTlQCOi4vD9u3b0a9fP7Rs2RLTpk1D586dERgYiBdffBE5OTnYsWMHJk+ejDp16qBv374YMWIEli9fDicnJ7zzzjuoXr26rg1DvP766/j4449Rp04dBAUF4ZNPPkFSUpKi1/vBBx+gV69e8PX1xYABA2BhYYEzZ87gzz//xOzZswE8vINi7969aNeuHWxsbODm5gZ/f39cu3YNp0+fRo0aNeDk5GTQdSGiCsjEayqITCYzM1O88847onnz5sLFxUXY29uLevXqiffee0+kp6frjktJSRHjxo0T1apVE9bW1sLX11eEhYWJGzdu6I7ZvHmzCA4OFmq1Wnh6eor+/fvrvnfv3j0xePBg4eLiIuzs7ET37t3FpUuXdN9/fMGiEEL88MMP4tFfz+zsbPH6668LZ2dn4erqKiZOnCjCw8MVLYQUQojo6GjRtm1bYWdnJ5ydnUXr1q3FihUrdN/fsmWLqF27trCyshJ+fn666/Tcc88JV1dXAUBERUUZdF0KWkBJROWbSgjeOE1ERETF490TREREZBAmDURERGQQJg1ERERkECYNREREZBAmDURERGQQJg1ERERkECYNREREZBAmDURERGQQJg1ERERkECYNREREZBAmDURERGQQJg1ERERkkP8DPLquML2aJ5kAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Make the heatmap plot:\n", + "plt.imshow( pointwiseMIs )\n", + "# Add the labels pasting in the code from above:\n", + "plt.xlabel('Second letter')\n", + "plt.xticks(ticks=range(0,27), labels=ySymbols.flatten())\n", + "plt.ylabel('First letter');\n", + "plt.yticks(ticks=range(0,27), labels=xSymbols.flatten())\n", + "cbar = plt.colorbar()\n", + "cbar.set_label('MI (bits)');\n", + "plt.title('MI between successive letters of text');" + ] + }, + { + "cell_type": "markdown", + "id": "37b50b62-6af9-4b66-84c3-b2c180ad9d28", + "metadata": {}, + "source": [ + "6. Examine the values and determine whether you can identify character pairs where the second is highly predictable from the first, and where the first character is misinformative about the second. Can you explain these results?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "def04a11-19af-437a-a3d1-15a09548dc5b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/course/Module02-JointAndConditionalEntropy/TextAnalysis/pointwiseMIofText.m b/course/Module02-JointAndConditionalEntropy/TextAnalysis/pointwiseMIofText.m new file mode 100644 index 0000000..a657237 --- /dev/null +++ b/course/Module02-JointAndConditionalEntropy/TextAnalysis/pointwiseMIofText.m @@ -0,0 +1,37 @@ +% assumes processedStr holds the text as previously processed - you can run +% the previous solution code entropyOfCharacters.m to pull this up + +[result, jointSymbols, jointProbabilities, xSymbols, xProbabilities, ySymbols, yProbabilities] = mutualinformationempirical(processedStr(1:end-1), processedStr(2:end)); + +pointwiseMIs = zeros(length(xSymbols), length(ySymbols)); % Create array to store the pointwise MI values for each possible character pair +for firstCharIndex = 1:length(xSymbols) + firstChar = xSymbols(firstCharIndex); + probFirst = xProbabilities(firstCharIndex); + for secondCharIndex = 1:length(ySymbols) + secondChar = ySymbols(secondCharIndex); + probSecond = yProbabilities(secondCharIndex); + jointSymbolIndex = find((jointSymbols(:,1) == firstChar) & (jointSymbols(:,2) == secondChar)); + if isempty(jointSymbolIndex) + pointwiseMIs(firstCharIndex, secondCharIndex) = 0; % No occurence, so set to 0 + continue; + end + probJoint = jointProbabilities(jointSymbolIndex); + % Compute the pointwise MI from probJoint, probFirst and probSecond + pointwiseMIs(firstCharIndex, secondCharIndex) = log2( probJoint ./ (probFirst .* probSecond) ); + end +end + +figure(); +imagesc(pointwiseMIs) +ylabel('First letter'); +xlabel('Second letter'); +h = colorbar; +h.Label.String = 'MI (bits)'; +h.Label.Rotation = 90; +xticks(1:27) +xticklabels(ySymbols); % second letters - y - goes on x axis +h = gca(); +h.XTickLabelRotation = 0; % Align the x labels properly +yticks(1:27) +yticklabels(xSymbols); % first letters - x - goes on y axis +title('MI between successive letters of text');