natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changed how texts are normalized

This commit is contained in:
Panupong Pasupat 2017-08-29 04:19:11 -07:00
parent 7b710ab942
commit 174f57b5d4
2 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java
View File

@ -341,7 +341,6 @@ public final class StringNormalizationUtils {
// Dashed / Parenthesized information
string = simpleNormalize(string);
string = string.replaceAll("\\[[^\\]]*\\]", "");
string = string.trim().replaceAll(" - .*$", "");
string = string.trim().replaceAll("\\([^)]*\\)$", "");
return string.replaceAll("\\s+", " ").trim();
}
@ -363,7 +362,7 @@ public final class StringNormalizationUtils {
// Remove citations
string = string.trim().replaceAll("((?<!^)\\[[^\\]]*\\]|\\[\\d+\\]|[•♦†‡*#+])*$", "");
// Remove details in parenthesis
string = string.trim().replaceAll("(?<!^)( \\([^)]*\\))*$", "");
string = string.trim().replaceAll("(?<!^)(\\s+\\([^)]*\\))*$", "");
// Remove outermost quotation mark
string = string.trim().replaceAll("^\"([^\"]*)\"$", "\\1");
} while (!oldString.equals(string));
@ -383,7 +382,9 @@ public final class StringNormalizationUtils {
Multimap<Value, Value> metadata = ArrayListMultimap.create();
TableColumn column = new TableColumn("Test", "test", 0);
analyzeString(o, metadata, column, new HashMap<>());
LogInfo.logs("%s %s", o, metadata);
String aggressive = aggressiveNormalize(o).toLowerCase();
String official = officialEvaluatorNormalize(o);
LogInfo.logs("%s %s | %s %s %s", o, metadata, official, aggressive, aggressive.equals(official));
}
public static void main(String[] args) {
@ -395,6 +396,7 @@ public final class StringNormalizationUtils {
unitTest("twenty three");
unitTest("apple, banana, banana, BANANA");
unitTest("apple\nbanana\norange");
unitTest("0-1\n(4-5 p)");
unitTest("21st");
unitTest("2001st");
unitTest("2,000,000 ft.");

4
src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
View File

@ -71,12 +71,12 @@ public class TableValueEvaluator implements ValueEvaluator {
if (opts.allowNormalizedStringMatch) {
String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase();
if (!targetTextOfficial.equals(targetText)) {
if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText)) {
LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
}
String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase();
if (!predTextOfficial.equals(predText)) {
if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText)) {
LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
}
}