diff --git a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java index b32afb7..8d83eed 100644 --- a/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java +++ b/src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java @@ -341,7 +341,6 @@ public final class StringNormalizationUtils { // Dashed / Parenthesized information string = simpleNormalize(string); string = string.replaceAll("\\[[^\\]]*\\]", ""); - string = string.trim().replaceAll(" - .*$", ""); string = string.trim().replaceAll("\\([^)]*\\)$", ""); return string.replaceAll("\\s+", " ").trim(); } @@ -363,7 +362,7 @@ public final class StringNormalizationUtils { // Remove citations string = string.trim().replaceAll("((? metadata = ArrayListMultimap.create(); TableColumn column = new TableColumn("Test", "test", 0); analyzeString(o, metadata, column, new HashMap<>()); - LogInfo.logs("%s %s", o, metadata); + String aggressive = aggressiveNormalize(o).toLowerCase(); + String official = officialEvaluatorNormalize(o); + LogInfo.logs("%s %s | %s %s %s", o, metadata, official, aggressive, aggressive.equals(official)); } public static void main(String[] args) { @@ -395,6 +396,7 @@ public final class StringNormalizationUtils { unitTest("twenty three"); unitTest("apple, banana, banana, BANANA"); unitTest("apple\nbanana\norange"); + unitTest("0-1\n(4-5 p)"); unitTest("21st"); unitTest("2001st"); unitTest("2,000,000 ft."); diff --git a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java index 930f01b..b4eb137 100644 --- a/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java +++ b/src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java @@ -71,12 +71,12 @@ public class TableValueEvaluator implements ValueEvaluator { if (opts.allowNormalizedStringMatch) { String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText); targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase(); - if (!targetTextOfficial.equals(targetText)) { + if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText)) { LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText); } String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText); predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase(); - if (!predTextOfficial.equals(predText)) { + if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText)) { LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText); } }