From 1fc5b820308b8a4b8d899cd4a6f396c8c19f2538 Mon Sep 17 00:00:00 2001
From: Panupong Pasupat <ppasupat@stanford.edu>
Date: Mon, 4 Sep 2017 15:09:23 -0700
Subject: [PATCH] Fixed the normalization stuff

---
 .../sempre/tables/StringNormalizationUtils.java  | 11 ++++++++---
 .../nlp/sempre/tables/TableValueEvaluator.java   | 16 +++++++++-------
 .../tables/features/AnchorFeatureComputer.java   | 14 +++++++-------
 3 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java
index 7607bd7..fbb6eb7 100644
--- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java
+++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java
@@ -317,7 +317,7 @@ public final class StringNormalizationUtils {
         .replaceAll("[‘’´`]", "'")
         .replaceAll("[“”«»]", "\"")
         .replaceAll("[•†‡]", "")
-        .replaceAll("[‐‑–—]", "-");
+        .replaceAll("[-‐‑–—]", "-");
     return string.replaceAll("\\s+", " ").trim();
   }
 
@@ -340,12 +340,17 @@ public final class StringNormalizationUtils {
   public static String aggressiveNormalize(String string) {
     // Dashed / Parenthesized information
     string = simpleNormalize(string);
-    string = string.replaceAll("\\[[^\\]]*\\]", "");
     String oldString;
     do {
       oldString = string;
-      string = string.trim().replaceAll("\\([^)]*\\)$", "");
+      // Remove citations
+      string = string.trim().replaceAll("((?<!^)\\[[^\\]]*\\]|\\[\\d+\\]|[•♦†‡*#+])*$", "");
+      // Remove details in parenthesis
+      string = string.trim().replaceAll("(?<!^)(\\s*\\([^)]*\\))*$", "");
+      // Remove outermost quotation mark
+      string = string.trim().replaceAll("^\"([^\"]*)\"$", "$1");
     } while (!oldString.equals(string));
+    // Collapse whitespaces
     return string.replaceAll("\\s+", " ").trim();
   }
 
diff --git a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
index b4eb137..c825c7d 100644
--- a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
+++ b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
@@ -25,6 +25,8 @@ public class TableValueEvaluator implements ValueEvaluator {
     public boolean ignoreNumberValueUnits = true;
     @Option(gloss = "Strict date evaluation (year, month, and date all have to match)")
     public boolean strictDateEvaluation = false;
+    @Option(gloss = "Check if the normalized text matches the official evaluator")
+    public boolean checkStringNormalization = false;
   }
   public static Options opts = new Options();
 
@@ -69,15 +71,15 @@ public class TableValueEvaluator implements ValueEvaluator {
         String predText = (pred instanceof NameValue) ? ((NameValue) pred).description : ((DescriptionValue) pred).value;
         if (predText == null) predText = "";
         if (opts.allowNormalizedStringMatch) {
-          String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
           targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase();
-          if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText)) {
-            LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
-          }
-          String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
           predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase();
-          if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText)) {
-            LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
+          if (opts.checkStringNormalization) {
+            String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
+            String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
+            if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText))
+              LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
+            if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText))
+              LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
           }
         }
         return targetText.equals(predText);
diff --git a/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java
index f0a44e9..0b4f098 100644
--- a/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java
+++ b/src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java
@@ -26,23 +26,23 @@ public class AnchorFeatureComputer implements FeatureComputer {
   private void extractMatchingFeatures(TableKnowledgeGraph graph,
       Derivation deriv, String phrase, NameValue predicate) {
     String predicateString = graph.getOriginalString(predicate);
-    LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString);
+    //LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString);
     predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase();
     if (predicateString.equals(phrase)) {
       deriv.addFeature("a-e", "exact");
-      LogInfo.logs("%s %s exact", phrase, predicateString);
+      //LogInfo.logs("%s %s exact", phrase, predicateString);
     } else if (predicateString.startsWith(phrase + " ")) {
       deriv.addFeature("a-e", "prefix");
-      LogInfo.logs("%s %s prefix", phrase, predicateString);
+      //LogInfo.logs("%s %s prefix", phrase, predicateString);
     } else if (predicateString.endsWith(" " + phrase)) {
       deriv.addFeature("a-e", "suffix");
-      LogInfo.logs("%s %s suffix", phrase, predicateString);
+      //LogInfo.logs("%s %s suffix", phrase, predicateString);
     } else if (predicateString.contains(" " + phrase + " ")){
       deriv.addFeature("a-e", "substring");
-      LogInfo.logs("%s %s substring", phrase, predicateString);
+      //LogInfo.logs("%s %s substring", phrase, predicateString);
     } else {
       deriv.addFeature("a-e", "other");
-      LogInfo.logs("%s %s other", phrase, predicateString);
+      //LogInfo.logs("%s %s other", phrase, predicateString);
     }
     // Does the phrase match other cells?
     Set<String> matches = new HashSet<>();
@@ -54,7 +54,7 @@ public class AnchorFeatureComputer implements FeatureComputer {
         }
       }
     }
-    LogInfo.logs(">> %s", matches);
+    //LogInfo.logs(">> %s", matches);
     if (matches.size() == 0) {
       deriv.addFeature("a-e", "unique");
     } else if (matches.size() < 3) {