natas_hw
/
sempre
mirror of https://github.com/percyliang/sempre
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added OpenTypes for expressions. Fixed a few bugs

This commit is contained in:
missxa 2018-03-19 05:39:39 +01:00
parent 11bf6405da
commit f0e312c222
14 changed files with 194 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config.properties
View File

@ -33,4 +33,4 @@ KEYWORDS_TAGS:
# Scoring parameters
W2V_THRES: 0.3
WORD2VEC_GOOGLE: false
DEBUG: 1
DEBUG: 2

28
pom.xml
View File

@ -163,7 +163,7 @@
<argument>-Params.initWeightsRandomly</argument>
<argument>true</argument>
<argument>-Learner.maxTrainIters</argument>
<argument>1</argument>
<argument>1000</argument>
<argument>-Builder.executor</argument>
<argument>roboy.SparqlExecutor</argument>
<argument>-Builder.simple_executor</argument>
@ -216,18 +216,18 @@
</goals>
<configuration>
<arguments>
<argument>-FeatureExtractor.featureDomains</argument>
<argument>rule</argument>
<!--<argument>span</argument>-->
<argument>dependencyParse</argument>
<argument>-Dataset.inPaths</argument>
<argument>train:data/rpqa-train.examples</argument>
<argument>-Derivation.derivComparator</argument>
<argument>ScoredDerivationComparator</argument>
<argument>-Params.initWeightsRandomly</argument>
<argument>true</argument>
<argument>-Learner.maxTrainIters</argument>
<argument>1</argument>
<!--<argument>-FeatureExtractor.featureDomains</argument>-->
<!--<argument>rule</argument>-->
<!--&lt;!&ndash;<argument>span</argument>&ndash;&gt;-->
<!--<argument>dependencyParse</argument>-->
<!--<argument>-Dataset.inPaths</argument>-->
<!--<argument>train:data/rpqa-train.examples</argument>-->
<!--<argument>-Derivation.derivComparator</argument>-->
<!--<argument>ScoredDerivationComparator</argument>-->
<!--<argument>-Params.initWeightsRandomly</argument>-->
<!--<argument>true</argument>-->
<!--<argument>-Learner.maxTrainIters</argument>-->
<!--<argument>1</argument>-->
<argument>-Builder.executor</argument>
<argument>roboy.SparqlExecutor</argument>
<argument>-Builder.simple_executor</argument>
@ -237,7 +237,7 @@
<argument>-Grammar.inPaths</argument>
<argument>resources/roboy-demo.grammar</argument>
<argument>-SimpleLexicon.inPaths</argument>
<argument>resources/lexicons/roboy-dbpedia.lexicon</argument>
<argument>resources/lexicons/roboy-demo.lexicon</argument>
<argument>-languageAnalyzer</argument>
<argument>corenlp.CoreNLPAnalyzer</argument>
<argument>-Main.server</argument>

77
resources/roboy-demo.grammar
View File

@ -8,7 +8,8 @@
(rule $Not (nor) (ConstantFn (string not)))
(rule $Class ($PHRASE) (SimpleLexiconFn (type "ClassNoun")))
(rule $RelClass ($PHRASE) (SimpleLexiconFn (type "RelationalNoun")))
(rule $ObjProp ($PHRASE) (SimpleLexiconFn (type "ObjProp")))
(rule $DataProp ($PHRASE) (SimpleLexiconFn (type "DataProp")))
(rule $RelatClass ($PHRASE) (SimpleLexiconFn (type "RelationClass")))
(rule $DatPropClass ($PHRASE) (SimpleLexiconFn (type "DataPropertyNoun")))
(rule $ObjPropClass ($PHRASE) (SimpleLexiconFn (type "ObjectProperty")))
@ -59,9 +60,9 @@
(rule $Number ($PHRASE) (FilterPosTagFn span CD))
# Link words
#(rule $VerbPrep ($Verb $Preposition) (ConcatFn " "))
#(rule $VerbTo ($Verb $To) (ConcatFn " "))
#(rule $VerbTo ($To $Verb) (SelectFn 1))
(rule $VerbPhrase ($Verb $Preposition) (ConcatFn " "))
(rule $VerbPhrase ($Verb $To) (ConcatFn " "))
(rule $VerbPhrase ($To $Verb) (SelectFn 1))
# Roboy Subject Phrase
(rule $RSubject ($ObjectClass) (IdentityFn))
@ -71,11 +72,10 @@
(rule $RSubject ($RSubject (, optional) $RSubject) (ConcatFn ","))
(rule $RSubject ($RSubject $ProNounEnd) (SelectFn 0))
# Roboy Predicate Phrase
(rule $RPredicate (($BE optional) $Relat) (IdentityFn))
(rule $RPredicate (($HAVE optional) $Relat) (IdentityFn))
(rule $RPredicate ($RelatClass) (IdentityFn))
(rule $RPredicate (($BE optional) $ObjProp) (IdentityFn))
(rule $RPredicate (($HAVE optional) $ObjProp) (IdentityFn))
(rule $RPredicate ($ObjProp) (IdentityFn))
(rule $RPredicate ($Not $RPredicate) (ConcatFn " "))
(rule $RPredicate ($RPredicate and $RPredicate) (ConcatFn ","))
(rule $RPredicate ($RPredicate , $RPredicate) (ConcatFn ","))
@ -85,12 +85,43 @@
(rule $RPredicate ($HAVE $RPredicate) (SelectFn 1))
(rule $RPredicate ($AdjPhrase $RPredicate) (SelectFn 1))
(rule $RPredicate ($Adverb $RPredicate) (SelectFn 1))
(rule $RPredicate ($VerbPhrase) (OpenTypeFn relation))
(rule $RPredicate ($VerbPhrase) (SimpleLexiconFn (type "NewRelation")))
(rule $RDataPredicate (($BE optional) $DataProp) (IdentityFn))
(rule $RDataPredicate (($HAVE optional) $DataProp) (IdentityFn))
(rule $RDataPredicate ($DataProp) (IdentityFn))
(rule $RDataPredicate ($Not $RDataPredicate) (ConcatFn " "))
(rule $RDataPredicate ($RDataPredicate and $RDataPredicate) (ConcatFn ","))
(rule $RDataPredicate ($RDataPredicate , $RDataPredicate) (ConcatFn ","))
(rule $RDataPredicate ($RDataPredicate or $RDataPredicate) (ConcatFn ";"))
(rule $RDataPredicate ($TypeProperty $ObjectClass) (TripleFn "pt"))
(rule $RDataPredicate ($BE $RDataPredicate) (SelectFn 1))
(rule $RDataPredicate ($HAVE $RDataPredicate) (SelectFn 1))
(rule $RDataPredicate ($AdjPhrase $RDataPredicate) (SelectFn 1))
(rule $RDataPredicate ($Adverb $RDataPredicate) (SelectFn 1))
(rule $RDataPredicate ($VerbPhrase) (OpenTypeFn relation))
(rule $RDataPredicate ($VerbPhrase) (SimpleLexiconFn (type "NewRelation")))
(rule $N ($BE) (ConstantFn rb:NAME_OF))
(rule $MEMBER ($BE) (ConstantFn rb:MEMBER_OF))
(rule $OCCUP ($BE $Det) (ConstantFn rb:OCCUPIED_AS))
# Object Phrase
(rule $Object ($Noun) (OpenTypeFn type))
(rule $Object ($NamedEntity) (OpenTypeFn entity))
(rule $Object ($Noun) (SimpleLexiconFn (type "NewType")))
(rule $Object ($NamedEntity) (SimpleLexiconFn (type "NewEntity")))
(rule $Object ($Det $Object) (SelectFn 1))
(rule $Object ($Conj $Object) (SelectFn 1))
(rule $Object ($Object or $Object) (ConcatFn ";"))
(rule $Object ($Object ($Conj optional) $Object) (ConcatFn ","))
(rule $Object ($Object , $Object) (ConcatFn ","))
(rule $Object ($Adjective $Object) (TripleFn "ot"))
(rule $Object ($Object $Adjective) (TripleFn "to"))
# Value Phrase
(rule $Value ($Noun) (IdentityFn))
(rule $Value ($LEMMA_PHRASE) (FilterPosTagFn span VBG))
(rule $Value ($Det $Value) (SelectFn 1))
@ -120,13 +151,26 @@
(rule $Expr ($RSubject $OCCUP $Value) (TripleFn "spo"))
(rule $Expr ($Person $N $PersonName) (TripleFn "spo"))
(rule $Expr ($ObjectClass $N $PersonName) (TripleFn "spo"))
(rule $Expr ($RSubject $Value $Preposition ($TOKEN optional) $RPredicate) (TripleFn "sop"))
(rule $Expr ($RSubject $RPredicate ($Preposition optional) $Value) (TripleFn "spo"))
(rule $Expr ($RSubject $RPredicate ($TOKEN optional) ($BE optional) $Value) (TripleFn "spo"))
(rule $Expr ($Value ($BE optional) $RSubject $RPredicate) (TripleFn "osp"))
(rule $Expr ($RSubject $RPredicate ($BE optional) $Value) (TripleFn "spo"))
(rule $Expr ($RSubject $Value $Preposition ($TOKEN optional) $RDataPredicate) (TripleFn "sop"))
(rule $Expr ($RSubject $RDataPredicate ($Preposition optional) $Value) (TripleFn "spo"))
(rule $Expr ($RSubject $RDataPredicate ($TOKEN optional) ($BE optional) $Value) (TripleFn "spo"))
(rule $Expr ($Value ($BE optional) $RSubject $RDataPredicate) (TripleFn "osp"))
(rule $Expr ($RSubject $RDataPredicate ($BE optional) $Value) (TripleFn "spo"))
(rule $Expr ($AdjPhrase ($BE optional) $RSubject $RDataPredicate) (TripleFn "osp"))
(rule $Expr ($RDataPredicate $RSubject $Value) (TripleFn "pso"))
(rule $Expr ($RSubject $MEMBER $Object member) (TripleFn "spo"))
(rule $Expr ($RSubject $OCCUP $Object) (TripleFn "spo"))
(rule $Expr ($Person $N $PersonName) (TripleFn "spo"))
(rule $Expr ($ObjectClass $N $PersonName) (TripleFn "spo"))
(rule $Expr ($RSubject $Object $Preposition ($TOKEN optional) $RPredicate) (TripleFn "sop"))
(rule $Expr ($RSubject $RPredicate ($Preposition optional) $Object) (TripleFn "spo"))
(rule $Expr ($RSubject $RPredicate ($TOKEN optional) ($BE optional) $Object) (TripleFn "spo"))
(rule $Expr ($Object ($BE optional) $RSubject $RPredicate) (TripleFn "osp"))
(rule $Expr ($RSubject $RPredicate ($BE optional) $Object) (TripleFn "spo"))
(rule $Expr ($AdjPhrase ($BE optional) $RSubject $RPredicate) (TripleFn "osp"))
(rule $Expr ($RPredicate $RSubject $Value) (TripleFn "pso"))
(rule $Expr ($RPredicate $RSubject $Object) (TripleFn "pso"))
(rule $ROOT ($Expr) (IdentityFn))
(rule $ROOT ($Expr ($Conj optional) $Expr) (ListFn))
(rule $ROOT (($Noun optional) $Expr) (IdentityFn))
@ -162,8 +206,9 @@
(rule $RSet ($Unary $Unary) (MergeFn and))
(rule $RSet ($Binary $RSet) (JoinFn forward))
(rule $RSet ($RSet $Binary) (JoinFn backward))
(rule $ROOT (($Wh optional) $RSet) (IdentityFn))
(rule $ROOT ((what optional) (is optional) $RSet) (IdentityFn))
(rule $ROOT ($Wh $RSet) (SelectFn 1))
(rule $ROOT ($Wh $BE $RSet) (SelectFn 2))
(rule $ROOT ($Ignore $RSet) (SelectFn 1))
#TEST SET

10
src/main/java/edu/stanford/nlp/sempre/OpenTypeFn.java
View File

@ -41,16 +41,16 @@ public class OpenTypeFn extends SemanticFn {
@Override
public Derivation createDerivation() {
StringBuilder out = new StringBuilder();
if (type) out.append("OpenType\'");
if (rel) out.append("OpenRel\'");
if (entity) out.append("OpenEntity\'");
if (type) out.append("OpenType'");
if (rel) out.append("OpenRel'");
if (entity) out.append("OpenEntity'");
for (int i = 0; i < c.getChildren().size(); i++) {
if (i > 0)
out.append(" ");
out.append(c.childStringValue(i));
}
out.append("\''");
if (SimpleLexicon.getSingleton().lookup(out.toString().substring(out.toString().indexOf("\'")+1,out.toString().indexOf("\''"))).isEmpty())
out.append("''");
if (SimpleLexicon.getSingleton().lookup(out.toString().substring(out.toString().indexOf("'")+1,out.toString().indexOf("''"))).isEmpty())
return new Derivation.Builder()
.withCallable(c)
.withStringFormulaFrom(out.toString())

1
src/main/java/edu/stanford/nlp/sempre/ParserState.java
View File

@ -239,7 +239,6 @@ public abstract class ParserState {
List<Derivation> remove = new ArrayList();
List<String> formulas = new ArrayList();
for (Derivation deriv : predDerivations) {
// TODO: Add postprocess(deriv, ex.context);
if (!String.join(" ",formulas).contains(deriv.formula.toString()))
formulas.add(deriv.formula.toString());
else {

7
src/main/java/edu/stanford/nlp/sempre/TripleFn.java
View File

@ -28,7 +28,8 @@ public class TripleFn extends SemanticFn {
}
public String initial_formula(String string1, String string2) {
// TODO: Check types to form only with one database only
string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1);
string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2);
Map<String,Object> triple = new HashMap<>();
if (this.mode.equals("spo")) {
triple.put("subject", string1);
@ -80,6 +81,8 @@ public class TripleFn extends SemanticFn {
public String merge_formula(String string1, String string2) {
// Convert JSON string back to Map.
string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1);
string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2);
Gson gson = new Gson();
Type type = new TypeToken<Map<String, String>>(){}.getType();
Map<String, String> triple = gson.fromJson(string1, type);
@ -112,6 +115,8 @@ public class TripleFn extends SemanticFn {
Gson gson = new Gson();
Type type = new TypeToken<Map<String, String>>(){}.getType();
Map<String, String> triple = new HashMap();
string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1);
string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2);
if (string1.contains("{")){
triple = gson.fromJson(string1, type);
if (this.mode.equals("spo")) {

57
src/main/java/edu/stanford/nlp/sempre/roboy/ErrorRetrieval.java
View File

@ -181,7 +181,7 @@ public class ErrorRetrieval {
String formula = deriv.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
break;
String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1);
@ -205,7 +205,7 @@ public class ErrorRetrieval {
String formula = deriv.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
break;
String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1);
@ -244,12 +244,13 @@ public class ErrorRetrieval {
return lexemes;
}
while (formula.contains("Open")){
LogInfo.logs(formula);
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end+2 > formula.length())
break;
String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("\''")+2);
String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1);
String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("''")+2);
String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("'")+1);
if (full_type.contains("Entity")){
for (String candidate: result.getCandidates().get(term))
{
@ -258,7 +259,7 @@ public class ErrorRetrieval {
Map<String, String> lexeme = new HashMap();
lexeme.put("lexeme", entry.get("Label").toLowerCase());
lexeme.put("formula", entry.get("URI"));
lexeme.put("type", "NamedEntity");
lexeme.put("type", "NewEntity");
lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} ");
// Add with knowledge base label
lexemes.add(gson.toJson(lexeme));
@ -274,7 +275,7 @@ public class ErrorRetrieval {
Map<String, String> lexeme = new HashMap();
lexeme.put("lexeme", entry.get("Label").toLowerCase());
lexeme.put("formula", entry.get("URI"));
lexeme.put("type", "ClassNoun");
lexeme.put("type", "NewType");
lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} ");
// Add with knowledge base label
lexemes.add(gson.toJson(lexeme));
@ -290,7 +291,7 @@ public class ErrorRetrieval {
Map<String, String> lexeme = new HashMap();
lexeme.put("lexeme", entry.get("Label").toLowerCase());
lexeme.put("formula", entry.get("URI"));
lexeme.put("type", "RelationalNoun");
lexeme.put("type", "NewRelation");
lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} ");
// Add with knowledge base label
lexemes.add(gson.toJson(lexeme));
@ -298,7 +299,7 @@ public class ErrorRetrieval {
lexemes.add(gson.toJson(lexeme));
}
}
formula = formula.substring(end);
formula = formula.substring(end+2);
}
return lexemes;
}
@ -310,8 +311,10 @@ public class ErrorRetrieval {
* @param key term to replace
*/
public LispTree replaceEntity(LispTree new_formula, String replace, String key) {
if (new_formula!= null && new_formula.isLeaf())
return new_formula;
if (new_formula!= null && new_formula.isLeaf()) {
LispTree listTree = LispTree.proto.newLeaf(new_formula.value.replaceAll(replace, key));
return listTree;
}
else if (new_formula.children.get(0).value!= null && new_formula.children.get(0).value == "string"){
NameValue result = new NameValue(new_formula.children.get(1).value.replaceAll(replace, key));
return result.toLispTree();
@ -409,22 +412,18 @@ public class ErrorRetrieval {
String formula = deriv.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
break;
String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("\''")+2);
String entity = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1);
String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("''")+2);
String entity = formula.substring(start,end).substring(formula.substring(start,end).indexOf("'")+1);
String best = replacements.get(entity);
if (ConfigManager.DEBUG > 5)
LogInfo.logs("Forming: %s|", entity);
if (ConfigManager.DEBUG > 1)
LogInfo.logs("Forming: %s|%s|%s", entity, best, full_type);
if (errorInfo.getFollowUps().containsKey(entity)) {
List<String> c = errorInfo.getFollowUps().get(entity);
if (ConfigManager.DEBUG > 5)
LogInfo.logs("Forming: %s|", String.join(" ", c));
List<Map.Entry<String, String>> questions = formQuestion(entity, c);
for (int i = 0; i < questions.size(); i++) {
if (ConfigManager.DEBUG > 5)
LogInfo.logs("Question 1 :%s", questions.get(i));
Map.Entry<String, String> entry = new java.util.AbstractMap.SimpleEntry<String, String>
(questions.get(i).getKey(),
Formulas.fromLispTree(replaceEntity(best_formula,
@ -440,8 +439,10 @@ public class ErrorRetrieval {
}.getType();
Map<String, String> cand = this.gson.fromJson(best, type);
new_formula = replaceEntity(new_formula, full_type, cand.get("URI"));
if (ConfigManager.DEBUG > 1)
LogInfo.logs("Replaced: %s", new_formula.toString());
}
formula = formula.substring(end);
formula = formula.substring(end+2);
}
deriv.setFormula(Formulas.fromLispTree(new_formula));
return deriv;
@ -455,6 +456,18 @@ public class ErrorRetrieval {
ErrorInfo result = new ErrorInfo();
Set<String> update = new HashSet<>();
LogInfo.begin_track("Error retrieval:");
List<Derivation> remove = new ArrayList();
List<String> formulas = new ArrayList();
for (Derivation deriv : this.derivations) {
if (!String.join(" ", formulas).contains(deriv.formula.toString()))
formulas.add(deriv.formula.toString());
else {
remove.add(deriv);
continue;
}
}
this.derivations.removeAll(remove);
LogInfo.logs("%d",this.derivations.size());
if (this.derivations != null) {
// Get candidates
for (Derivation deriv : this.derivations) {

8
src/main/java/edu/stanford/nlp/sempre/roboy/data/score_weights.json
View File

@ -1,6 +1,6 @@
{
"Context": 0.3,
"Word2Vec": 0.1,
"Probability" : 0.5,
"Similarity" : 0.1
"Context": 0.2,
"Word2Vec": 0.2,
"Probability" : 0.3,
"Similarity" : 0.3
}

6
src/main/java/edu/stanford/nlp/sempre/roboy/error/EntityRetriever.java
View File

@ -43,11 +43,11 @@ public class EntityRetriever extends KnowledgeRetriever {
String formula = dev.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
return errorInfo;
unknown = formula.substring(start,end);
String entity = unknown.substring(unknown.indexOf("\'")+1);
String entity = unknown.substring(unknown.indexOf("'")+1);
// Extract the results from XML now.
String url = endpointUrl.concat(entity);
url = url.replace(" ","_");
@ -56,12 +56,14 @@ public class EntityRetriever extends KnowledgeRetriever {
results = reader.readEntityXml(response.getXml(),keywords);
for (Map<String,String> c: results){
if (errorInfo.getCandidates().containsKey(entity)) {
c.put("Label",c.get("Label").toLowerCase());
errorInfo.getCandidates().get(entity).add(gson.toJson(c));
if (ConfigManager.DEBUG > 3) {
LogInfo.logs("Entity: %s", gson.toJson(c));
}
}
else {
c.put("Label",c.get("Label").toLowerCase());
errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(c))));
if (ConfigManager.DEBUG > 3) {
LogInfo.logs("Entity: %s", gson.toJson(c));

31
src/main/java/edu/stanford/nlp/sempre/roboy/error/LabelRetriever.java
View File

@ -40,11 +40,11 @@ public class LabelRetriever extends KnowledgeRetriever {
String formula = dev.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
return errorInfo;
unknown = formula.substring(start,end);
String entity = unknown.substring(unknown.indexOf("\'")+1);
String entity = unknown.substring(unknown.indexOf("'")+1);
// Extract the results from XML now.
Set<String> uri = sparqlUtil.returnURI(entity, endpointUrl, false);
Set<String> uri_cap = sparqlUtil.returnURI( WordUtils.capitalize(entity), endpointUrl, false);
@ -54,19 +54,26 @@ public class LabelRetriever extends KnowledgeRetriever {
uri.addAll(uri_cap);
Map<String,String> single = new HashMap();
if (uri != null) {
single.put("Label", entity);
single.put("Refcount", String.valueOf(new Double(1.0/uri.size())));
for (String u: uri) {
single.put("URI", u);
if (errorInfo.getCandidates().containsKey(entity)) {
errorInfo.getCandidates().get(entity).add(gson.toJson(single));
if (ConfigManager.DEBUG > 3)
LogInfo.logs("Label: %s",gson.toJson(single));
}
else{
errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(single))));
if (ConfigManager.DEBUG > 3)
LogInfo.logs("Label: %s",gson.toJson(single));
// Get rid of categories
if (u.contains("Category"))
single.put("Refcount", String.valueOf(0.1));
Set<String> labels = sparqlUtil.returnLabel(u, endpointUrl, false);
if (labels == null)
labels = new HashSet<>(Arrays.asList(entity));
for (String l: labels) {
single.put("Label", l);
if (errorInfo.getCandidates().containsKey(entity)) {
errorInfo.getCandidates().get(entity).add(gson.toJson(single));
if (ConfigManager.DEBUG > 3)
LogInfo.logs("Label: %s", gson.toJson(single));
} else {
errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(single))));
if (ConfigManager.DEBUG > 3)
LogInfo.logs("Label: %s", gson.toJson(single));
}
}
}
}

4
src/main/java/edu/stanford/nlp/sempre/roboy/error/MCGRetriever.java
View File

@ -45,11 +45,11 @@ public class MCGRetriever extends KnowledgeRetriever {
String formula = dev.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
return errorInfo;
unknown = formula.substring(start,end);
String entity = unknown.substring(unknown.indexOf("\'")+1);
String entity = unknown.substring(unknown.indexOf("'")+1);
// Extract the results from XML now.
String url = (endpointUrl.concat(entity.replace(" ","+"))).concat("&topK=10");
SparqlUtils.ServerResponse response = sparqlUtil.makeRequest(url);

4
src/main/java/edu/stanford/nlp/sempre/roboy/error/Word2VecRetriever.java
View File

@ -37,11 +37,11 @@ public class Word2VecRetriever extends KnowledgeRetriever {
String formula = dev.getFormula().toString();
while (formula.contains("Open")){
int start = formula.indexOf("Open")+"Open".length();
int end = formula.indexOf("\''",start);
int end = formula.indexOf("''",start);
if (start > formula.length() || start < 0 || end < 0 ||end > formula.length())
return errorInfo;
unknown = formula.substring(start,end);
String entity = unknown.substring(unknown.indexOf("\'")+1);
String entity = unknown.substring(unknown.indexOf("'")+1);
List<String> known_words= new ArrayList<String>(SimpleLexicon.getSingleton().lookup_type(entity));
List<String> candidate = this.vec.getBest(entity,known_words);
for (String c: candidate){

40
src/main/java/edu/stanford/nlp/sempre/roboy/utils/SparqlUtils.java
View File

@ -120,6 +120,46 @@ public class SparqlUtils {
return null;
}
// Form query based on triple form
public Set<String> returnLabel(String entity, String endpointUrl, boolean object){
//System.out.println("Entity: "+entity);
List<Map<String,String>> triples = new ArrayList();
Map<String,String> t1 = new HashMap();
t1.put("predicate","rdfs:label");
t1.put("subject",entity);
triples.add(t1);
Gson gson = new Gson();
String json = gson.toJson((triples));
try {
String url = String.format("%s?default-graph-uri=http://dbpedia.org&query=%s&format=xml",
endpointUrl, URLEncoder.encode(formQuery(json,"en"), "UTF-8"));
System.out.println("SPARQL query: "+formQuery(json));
System.out.println("Query: "+url);
ServerResponse response = makeRequest(url);
List<Map<String,String>> list = new ArrayList<>();
if (response.getXml() != null)
{
list = reader.readArrayXml(response.getXml());
}
// System.out.println("Query: "+list.toString());
if (list.size()>0) {
Set<String> labels = new HashSet<>();
for (Map<String,String> map: list){
labels.addAll(map.values());
}
return labels;
}
else
return null;
}
catch(UnsupportedEncodingException e){
LogInfo.logs("WrongFormat of the input: %s", json);
}
return null;
}
// Form query based on triple form
public String returnDescr(String entity, String endpointUrl){

2
src/main/java/edu/stanford/nlp/sempre/roboy/utils/XMLReader.java
View File

@ -117,8 +117,8 @@ public class XMLReader{
String var = "?" + binding.getAttribute("name");
String uri = getTagValue("uri", binding);
uri = new String(uri.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
if (!(uri == null || (var.contains("_") || (uri.contains("#"))))){
uri = new String(uri.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8);
if (uri.contains(":"))
uri = dbInfo.uri2id(uri);
results.put(var,uri);