natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed the normalization stuff

This commit is contained in:
Panupong Pasupat 2017-09-04 15:09:23 -07:00
parent 3ef45877f3
commit 1fc5b82030
3 changed files with 24 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/edu/stanford/nlp/sempre/tables/StringNormalizationUtils.java
View File

@ -317,7 +317,7 @@ public final class StringNormalizationUtils {
.replaceAll("[‘’´`]", "'")
.replaceAll("[“”«»]", "\"")
.replaceAll("[•†‡]", "")
.replaceAll("[‐‑–—]", "-");
.replaceAll("[-‐‑–—]", "-");
return string.replaceAll("\\s+", " ").trim();
}
@ -340,12 +340,17 @@ public final class StringNormalizationUtils {
public static String aggressiveNormalize(String string) {
// Dashed / Parenthesized information
string = simpleNormalize(string);
string = string.replaceAll("\\[[^\\]]*\\]", "");
String oldString;
do {
oldString = string;
string = string.trim().replaceAll("\\([^)]*\\)$", "");
// Remove citations
string = string.trim().replaceAll("((?<!^)\\[[^\\]]*\\]|\\[\\d+\\]|[•♦†‡*#+])*$", "");
// Remove details in parenthesis
string = string.trim().replaceAll("(?<!^)(\\s*\\([^)]*\\))*$", "");
// Remove outermost quotation mark
string = string.trim().replaceAll("^\"([^\"]*)\"$", "$1");
} while (!oldString.equals(string));
// Collapse whitespaces
return string.replaceAll("\\s+", " ").trim();
}

16
src/edu/stanford/nlp/sempre/tables/TableValueEvaluator.java
View File

@ -25,6 +25,8 @@ public class TableValueEvaluator implements ValueEvaluator {
public boolean ignoreNumberValueUnits = true;
@Option(gloss = "Strict date evaluation (year, month, and date all have to match)")
public boolean strictDateEvaluation = false;
@Option(gloss = "Check if the normalized text matches the official evaluator")
public boolean checkStringNormalization = false;
}
public static Options opts = new Options();
@ -69,15 +71,15 @@ public class TableValueEvaluator implements ValueEvaluator {
String predText = (pred instanceof NameValue) ? ((NameValue) pred).description : ((DescriptionValue) pred).value;
if (predText == null) predText = "";
if (opts.allowNormalizedStringMatch) {
String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
targetText = StringNormalizationUtils.aggressiveNormalize(targetText).toLowerCase();
if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText)) {
LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
}
String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
predText = StringNormalizationUtils.aggressiveNormalize(predText).toLowerCase();
if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText)) {
LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
if (opts.checkStringNormalization) {
String targetTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(targetText);
String predTextOfficial = StringNormalizationUtils.officialEvaluatorNormalize(predText);
if (!targetTextOfficial.equals(targetText) && !(targetTextOfficial + ".").equals(targetText))
LogInfo.warnings("Different normalization: [%s][%s]", targetTextOfficial, targetText);
if (!predTextOfficial.equals(predText) && !(predTextOfficial + ".").equals(predText))
LogInfo.warnings("Different normalization: [%s][%s]", predTextOfficial, predText);
}
}
return targetText.equals(predText);

14
src/edu/stanford/nlp/sempre/tables/features/AnchorFeatureComputer.java
View File

@ -26,23 +26,23 @@ public class AnchorFeatureComputer implements FeatureComputer {
private void extractMatchingFeatures(TableKnowledgeGraph graph,
Derivation deriv, String phrase, NameValue predicate) {
String predicateString = graph.getOriginalString(predicate);
LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString);
//LogInfo.logs("%s -> %s = %s", phrase, predicate, predicateString);
predicateString = StringNormalizationUtils.simpleNormalize(predicateString).toLowerCase();
if (predicateString.equals(phrase)) {
deriv.addFeature("a-e", "exact");
LogInfo.logs("%s %s exact", phrase, predicateString);
//LogInfo.logs("%s %s exact", phrase, predicateString);
} else if (predicateString.startsWith(phrase + " ")) {
deriv.addFeature("a-e", "prefix");
LogInfo.logs("%s %s prefix", phrase, predicateString);
//LogInfo.logs("%s %s prefix", phrase, predicateString);
} else if (predicateString.endsWith(" " + phrase)) {
deriv.addFeature("a-e", "suffix");
LogInfo.logs("%s %s suffix", phrase, predicateString);
//LogInfo.logs("%s %s suffix", phrase, predicateString);
} else if (predicateString.contains(" " + phrase + " ")){
deriv.addFeature("a-e", "substring");
LogInfo.logs("%s %s substring", phrase, predicateString);
//LogInfo.logs("%s %s substring", phrase, predicateString);
} else {
deriv.addFeature("a-e", "other");
LogInfo.logs("%s %s other", phrase, predicateString);
//LogInfo.logs("%s %s other", phrase, predicateString);
}
// Does the phrase match other cells?
Set<String> matches = new HashSet<>();
@ -54,7 +54,7 @@ public class AnchorFeatureComputer implements FeatureComputer {
}
}
}
LogInfo.logs(">> %s", matches);
//LogInfo.logs(">> %s", matches);
if (matches.size() == 0) {
deriv.addFeature("a-e", "unique");
} else if (matches.size() < 3) {