diff --git a/config.properties b/config.properties index 5c53f67..f8896cb 100644 --- a/config.properties +++ b/config.properties @@ -33,4 +33,4 @@ KEYWORDS_TAGS: # Scoring parameters W2V_THRES: 0.3 WORD2VEC_GOOGLE: false -DEBUG: 1 +DEBUG: 2 diff --git a/pom.xml b/pom.xml index 53d0c1e..70d6aab 100644 --- a/pom.xml +++ b/pom.xml @@ -163,7 +163,7 @@ -Params.initWeightsRandomly true -Learner.maxTrainIters - 1 + 1000 -Builder.executor roboy.SparqlExecutor -Builder.simple_executor @@ -216,18 +216,18 @@ - -FeatureExtractor.featureDomains - rule - - dependencyParse - -Dataset.inPaths - train:data/rpqa-train.examples - -Derivation.derivComparator - ScoredDerivationComparator - -Params.initWeightsRandomly - true - -Learner.maxTrainIters - 1 + + + + + + + + + + + + -Builder.executor roboy.SparqlExecutor -Builder.simple_executor @@ -237,7 +237,7 @@ -Grammar.inPaths resources/roboy-demo.grammar -SimpleLexicon.inPaths - resources/lexicons/roboy-dbpedia.lexicon + resources/lexicons/roboy-demo.lexicon -languageAnalyzer corenlp.CoreNLPAnalyzer -Main.server diff --git a/resources/roboy-demo.grammar b/resources/roboy-demo.grammar index c2139c2..0889a17 100644 --- a/resources/roboy-demo.grammar +++ b/resources/roboy-demo.grammar @@ -8,7 +8,8 @@ (rule $Not (nor) (ConstantFn (string not))) (rule $Class ($PHRASE) (SimpleLexiconFn (type "ClassNoun"))) -(rule $RelClass ($PHRASE) (SimpleLexiconFn (type "RelationalNoun"))) +(rule $ObjProp ($PHRASE) (SimpleLexiconFn (type "ObjProp"))) +(rule $DataProp ($PHRASE) (SimpleLexiconFn (type "DataProp"))) (rule $RelatClass ($PHRASE) (SimpleLexiconFn (type "RelationClass"))) (rule $DatPropClass ($PHRASE) (SimpleLexiconFn (type "DataPropertyNoun"))) (rule $ObjPropClass ($PHRASE) (SimpleLexiconFn (type "ObjectProperty"))) @@ -59,9 +60,9 @@ (rule $Number ($PHRASE) (FilterPosTagFn span CD)) # Link words -#(rule $VerbPrep ($Verb $Preposition) (ConcatFn " ")) -#(rule $VerbTo ($Verb $To) (ConcatFn " ")) -#(rule $VerbTo ($To $Verb) (SelectFn 1)) +(rule $VerbPhrase ($Verb $Preposition) (ConcatFn " ")) +(rule $VerbPhrase ($Verb $To) (ConcatFn " ")) +(rule $VerbPhrase ($To $Verb) (SelectFn 1)) # Roboy Subject Phrase (rule $RSubject ($ObjectClass) (IdentityFn)) @@ -71,11 +72,10 @@ (rule $RSubject ($RSubject (, optional) $RSubject) (ConcatFn ",")) (rule $RSubject ($RSubject $ProNounEnd) (SelectFn 0)) - # Roboy Predicate Phrase -(rule $RPredicate (($BE optional) $Relat) (IdentityFn)) -(rule $RPredicate (($HAVE optional) $Relat) (IdentityFn)) -(rule $RPredicate ($RelatClass) (IdentityFn)) +(rule $RPredicate (($BE optional) $ObjProp) (IdentityFn)) +(rule $RPredicate (($HAVE optional) $ObjProp) (IdentityFn)) +(rule $RPredicate ($ObjProp) (IdentityFn)) (rule $RPredicate ($Not $RPredicate) (ConcatFn " ")) (rule $RPredicate ($RPredicate and $RPredicate) (ConcatFn ",")) (rule $RPredicate ($RPredicate , $RPredicate) (ConcatFn ",")) @@ -85,12 +85,43 @@ (rule $RPredicate ($HAVE $RPredicate) (SelectFn 1)) (rule $RPredicate ($AdjPhrase $RPredicate) (SelectFn 1)) (rule $RPredicate ($Adverb $RPredicate) (SelectFn 1)) +(rule $RPredicate ($VerbPhrase) (OpenTypeFn relation)) +(rule $RPredicate ($VerbPhrase) (SimpleLexiconFn (type "NewRelation"))) + + +(rule $RDataPredicate (($BE optional) $DataProp) (IdentityFn)) +(rule $RDataPredicate (($HAVE optional) $DataProp) (IdentityFn)) +(rule $RDataPredicate ($DataProp) (IdentityFn)) +(rule $RDataPredicate ($Not $RDataPredicate) (ConcatFn " ")) +(rule $RDataPredicate ($RDataPredicate and $RDataPredicate) (ConcatFn ",")) +(rule $RDataPredicate ($RDataPredicate , $RDataPredicate) (ConcatFn ",")) +(rule $RDataPredicate ($RDataPredicate or $RDataPredicate) (ConcatFn ";")) +(rule $RDataPredicate ($TypeProperty $ObjectClass) (TripleFn "pt")) +(rule $RDataPredicate ($BE $RDataPredicate) (SelectFn 1)) +(rule $RDataPredicate ($HAVE $RDataPredicate) (SelectFn 1)) +(rule $RDataPredicate ($AdjPhrase $RDataPredicate) (SelectFn 1)) +(rule $RDataPredicate ($Adverb $RDataPredicate) (SelectFn 1)) +(rule $RDataPredicate ($VerbPhrase) (OpenTypeFn relation)) +(rule $RDataPredicate ($VerbPhrase) (SimpleLexiconFn (type "NewRelation"))) (rule $N ($BE) (ConstantFn rb:NAME_OF)) (rule $MEMBER ($BE) (ConstantFn rb:MEMBER_OF)) (rule $OCCUP ($BE $Det) (ConstantFn rb:OCCUPIED_AS)) # Object Phrase +(rule $Object ($Noun) (OpenTypeFn type)) +(rule $Object ($NamedEntity) (OpenTypeFn entity)) +(rule $Object ($Noun) (SimpleLexiconFn (type "NewType"))) +(rule $Object ($NamedEntity) (SimpleLexiconFn (type "NewEntity"))) +(rule $Object ($Det $Object) (SelectFn 1)) +(rule $Object ($Conj $Object) (SelectFn 1)) +(rule $Object ($Object or $Object) (ConcatFn ";")) +(rule $Object ($Object ($Conj optional) $Object) (ConcatFn ",")) +(rule $Object ($Object , $Object) (ConcatFn ",")) +(rule $Object ($Adjective $Object) (TripleFn "ot")) +(rule $Object ($Object $Adjective) (TripleFn "to")) + +# Value Phrase (rule $Value ($Noun) (IdentityFn)) (rule $Value ($LEMMA_PHRASE) (FilterPosTagFn span VBG)) (rule $Value ($Det $Value) (SelectFn 1)) @@ -120,13 +151,26 @@ (rule $Expr ($RSubject $OCCUP $Value) (TripleFn "spo")) (rule $Expr ($Person $N $PersonName) (TripleFn "spo")) (rule $Expr ($ObjectClass $N $PersonName) (TripleFn "spo")) -(rule $Expr ($RSubject $Value $Preposition ($TOKEN optional) $RPredicate) (TripleFn "sop")) -(rule $Expr ($RSubject $RPredicate ($Preposition optional) $Value) (TripleFn "spo")) -(rule $Expr ($RSubject $RPredicate ($TOKEN optional) ($BE optional) $Value) (TripleFn "spo")) -(rule $Expr ($Value ($BE optional) $RSubject $RPredicate) (TripleFn "osp")) -(rule $Expr ($RSubject $RPredicate ($BE optional) $Value) (TripleFn "spo")) +(rule $Expr ($RSubject $Value $Preposition ($TOKEN optional) $RDataPredicate) (TripleFn "sop")) +(rule $Expr ($RSubject $RDataPredicate ($Preposition optional) $Value) (TripleFn "spo")) +(rule $Expr ($RSubject $RDataPredicate ($TOKEN optional) ($BE optional) $Value) (TripleFn "spo")) +(rule $Expr ($Value ($BE optional) $RSubject $RDataPredicate) (TripleFn "osp")) +(rule $Expr ($RSubject $RDataPredicate ($BE optional) $Value) (TripleFn "spo")) +(rule $Expr ($AdjPhrase ($BE optional) $RSubject $RDataPredicate) (TripleFn "osp")) +(rule $Expr ($RDataPredicate $RSubject $Value) (TripleFn "pso")) + +(rule $Expr ($RSubject $MEMBER $Object member) (TripleFn "spo")) +(rule $Expr ($RSubject $OCCUP $Object) (TripleFn "spo")) +(rule $Expr ($Person $N $PersonName) (TripleFn "spo")) +(rule $Expr ($ObjectClass $N $PersonName) (TripleFn "spo")) +(rule $Expr ($RSubject $Object $Preposition ($TOKEN optional) $RPredicate) (TripleFn "sop")) +(rule $Expr ($RSubject $RPredicate ($Preposition optional) $Object) (TripleFn "spo")) +(rule $Expr ($RSubject $RPredicate ($TOKEN optional) ($BE optional) $Object) (TripleFn "spo")) +(rule $Expr ($Object ($BE optional) $RSubject $RPredicate) (TripleFn "osp")) +(rule $Expr ($RSubject $RPredicate ($BE optional) $Object) (TripleFn "spo")) (rule $Expr ($AdjPhrase ($BE optional) $RSubject $RPredicate) (TripleFn "osp")) -(rule $Expr ($RPredicate $RSubject $Value) (TripleFn "pso")) +(rule $Expr ($RPredicate $RSubject $Object) (TripleFn "pso")) + (rule $ROOT ($Expr) (IdentityFn)) (rule $ROOT ($Expr ($Conj optional) $Expr) (ListFn)) (rule $ROOT (($Noun optional) $Expr) (IdentityFn)) @@ -162,8 +206,9 @@ (rule $RSet ($Unary $Unary) (MergeFn and)) (rule $RSet ($Binary $RSet) (JoinFn forward)) (rule $RSet ($RSet $Binary) (JoinFn backward)) -(rule $ROOT (($Wh optional) $RSet) (IdentityFn)) -(rule $ROOT ((what optional) (is optional) $RSet) (IdentityFn)) +(rule $ROOT ($Wh $RSet) (SelectFn 1)) +(rule $ROOT ($Wh $BE $RSet) (SelectFn 2)) +(rule $ROOT ($Ignore $RSet) (SelectFn 1)) #TEST SET diff --git a/src/main/java/edu/stanford/nlp/sempre/OpenTypeFn.java b/src/main/java/edu/stanford/nlp/sempre/OpenTypeFn.java index a163730..5dfd1ae 100644 --- a/src/main/java/edu/stanford/nlp/sempre/OpenTypeFn.java +++ b/src/main/java/edu/stanford/nlp/sempre/OpenTypeFn.java @@ -41,16 +41,16 @@ public class OpenTypeFn extends SemanticFn { @Override public Derivation createDerivation() { StringBuilder out = new StringBuilder(); - if (type) out.append("OpenType\'"); - if (rel) out.append("OpenRel\'"); - if (entity) out.append("OpenEntity\'"); + if (type) out.append("OpenType'"); + if (rel) out.append("OpenRel'"); + if (entity) out.append("OpenEntity'"); for (int i = 0; i < c.getChildren().size(); i++) { if (i > 0) out.append(" "); out.append(c.childStringValue(i)); } - out.append("\''"); - if (SimpleLexicon.getSingleton().lookup(out.toString().substring(out.toString().indexOf("\'")+1,out.toString().indexOf("\''"))).isEmpty()) + out.append("''"); + if (SimpleLexicon.getSingleton().lookup(out.toString().substring(out.toString().indexOf("'")+1,out.toString().indexOf("''"))).isEmpty()) return new Derivation.Builder() .withCallable(c) .withStringFormulaFrom(out.toString()) diff --git a/src/main/java/edu/stanford/nlp/sempre/ParserState.java b/src/main/java/edu/stanford/nlp/sempre/ParserState.java index 90c51e8..596ef7e 100644 --- a/src/main/java/edu/stanford/nlp/sempre/ParserState.java +++ b/src/main/java/edu/stanford/nlp/sempre/ParserState.java @@ -239,7 +239,6 @@ public abstract class ParserState { List remove = new ArrayList(); List formulas = new ArrayList(); for (Derivation deriv : predDerivations) { - // TODO: Add postprocess(deriv, ex.context); if (!String.join(" ",formulas).contains(deriv.formula.toString())) formulas.add(deriv.formula.toString()); else { diff --git a/src/main/java/edu/stanford/nlp/sempre/TripleFn.java b/src/main/java/edu/stanford/nlp/sempre/TripleFn.java index 01076f5..b4f1250 100644 --- a/src/main/java/edu/stanford/nlp/sempre/TripleFn.java +++ b/src/main/java/edu/stanford/nlp/sempre/TripleFn.java @@ -28,7 +28,8 @@ public class TripleFn extends SemanticFn { } public String initial_formula(String string1, String string2) { - // TODO: Check types to form only with one database only + string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1); + string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2); Map triple = new HashMap<>(); if (this.mode.equals("spo")) { triple.put("subject", string1); @@ -80,6 +81,8 @@ public class TripleFn extends SemanticFn { public String merge_formula(String string1, String string2) { // Convert JSON string back to Map. + string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1); + string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2); Gson gson = new Gson(); Type type = new TypeToken>(){}.getType(); Map triple = gson.fromJson(string1, type); @@ -112,6 +115,8 @@ public class TripleFn extends SemanticFn { Gson gson = new Gson(); Type type = new TypeToken>(){}.getType(); Map triple = new HashMap(); + string1 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string1); + string2 = org.apache.commons.lang.StringEscapeUtils.unescapeJava(string2); if (string1.contains("{")){ triple = gson.fromJson(string1, type); if (this.mode.equals("spo")) { diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/ErrorRetrieval.java b/src/main/java/edu/stanford/nlp/sempre/roboy/ErrorRetrieval.java index 3f36e53..87d12c0 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/ErrorRetrieval.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/ErrorRetrieval.java @@ -181,7 +181,7 @@ public class ErrorRetrieval { String formula = deriv.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) break; String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1); @@ -205,7 +205,7 @@ public class ErrorRetrieval { String formula = deriv.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) break; String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1); @@ -244,12 +244,13 @@ public class ErrorRetrieval { return lexemes; } while (formula.contains("Open")){ + LogInfo.logs(formula); int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); - if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) + int end = formula.indexOf("''",start); + if (start > formula.length() || start < 0 || end < 0 ||end+2 > formula.length()) break; - String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("\''")+2); - String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1); + String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("''")+2); + String term = formula.substring(start,end).substring(formula.substring(start,end).indexOf("'")+1); if (full_type.contains("Entity")){ for (String candidate: result.getCandidates().get(term)) { @@ -258,7 +259,7 @@ public class ErrorRetrieval { Map lexeme = new HashMap(); lexeme.put("lexeme", entry.get("Label").toLowerCase()); lexeme.put("formula", entry.get("URI")); - lexeme.put("type", "NamedEntity"); + lexeme.put("type", "NewEntity"); lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} "); // Add with knowledge base label lexemes.add(gson.toJson(lexeme)); @@ -274,7 +275,7 @@ public class ErrorRetrieval { Map lexeme = new HashMap(); lexeme.put("lexeme", entry.get("Label").toLowerCase()); lexeme.put("formula", entry.get("URI")); - lexeme.put("type", "ClassNoun"); + lexeme.put("type", "NewType"); lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} "); // Add with knowledge base label lexemes.add(gson.toJson(lexeme)); @@ -290,7 +291,7 @@ public class ErrorRetrieval { Map lexeme = new HashMap(); lexeme.put("lexeme", entry.get("Label").toLowerCase()); lexeme.put("formula", entry.get("URI")); - lexeme.put("type", "RelationalNoun"); + lexeme.put("type", "NewRelation"); lexeme.put("features", " {score:" + Double.toString(result.getScored().get(term).get(candidate))+"} "); // Add with knowledge base label lexemes.add(gson.toJson(lexeme)); @@ -298,7 +299,7 @@ public class ErrorRetrieval { lexemes.add(gson.toJson(lexeme)); } } - formula = formula.substring(end); + formula = formula.substring(end+2); } return lexemes; } @@ -310,8 +311,10 @@ public class ErrorRetrieval { * @param key term to replace */ public LispTree replaceEntity(LispTree new_formula, String replace, String key) { - if (new_formula!= null && new_formula.isLeaf()) - return new_formula; + if (new_formula!= null && new_formula.isLeaf()) { + LispTree listTree = LispTree.proto.newLeaf(new_formula.value.replaceAll(replace, key)); + return listTree; + } else if (new_formula.children.get(0).value!= null && new_formula.children.get(0).value == "string"){ NameValue result = new NameValue(new_formula.children.get(1).value.replaceAll(replace, key)); return result.toLispTree(); @@ -409,22 +412,18 @@ public class ErrorRetrieval { String formula = deriv.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) break; - String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("\''")+2); - String entity = formula.substring(start,end).substring(formula.substring(start,end).indexOf("\'")+1); + String full_type = formula.substring(formula.indexOf("Open"),formula.indexOf("''")+2); + String entity = formula.substring(start,end).substring(formula.substring(start,end).indexOf("'")+1); String best = replacements.get(entity); - if (ConfigManager.DEBUG > 5) - LogInfo.logs("Forming: %s|", entity); + if (ConfigManager.DEBUG > 1) + LogInfo.logs("Forming: %s|%s|%s", entity, best, full_type); if (errorInfo.getFollowUps().containsKey(entity)) { List c = errorInfo.getFollowUps().get(entity); - if (ConfigManager.DEBUG > 5) - LogInfo.logs("Forming: %s|", String.join(" ", c)); List> questions = formQuestion(entity, c); for (int i = 0; i < questions.size(); i++) { - if (ConfigManager.DEBUG > 5) - LogInfo.logs("Question 1 :%s", questions.get(i)); Map.Entry entry = new java.util.AbstractMap.SimpleEntry (questions.get(i).getKey(), Formulas.fromLispTree(replaceEntity(best_formula, @@ -440,8 +439,10 @@ public class ErrorRetrieval { }.getType(); Map cand = this.gson.fromJson(best, type); new_formula = replaceEntity(new_formula, full_type, cand.get("URI")); + if (ConfigManager.DEBUG > 1) + LogInfo.logs("Replaced: %s", new_formula.toString()); } - formula = formula.substring(end); + formula = formula.substring(end+2); } deriv.setFormula(Formulas.fromLispTree(new_formula)); return deriv; @@ -455,6 +456,18 @@ public class ErrorRetrieval { ErrorInfo result = new ErrorInfo(); Set update = new HashSet<>(); LogInfo.begin_track("Error retrieval:"); + List remove = new ArrayList(); + List formulas = new ArrayList(); + for (Derivation deriv : this.derivations) { + if (!String.join(" ", formulas).contains(deriv.formula.toString())) + formulas.add(deriv.formula.toString()); + else { + remove.add(deriv); + continue; + } + } + this.derivations.removeAll(remove); + LogInfo.logs("%d",this.derivations.size()); if (this.derivations != null) { // Get candidates for (Derivation deriv : this.derivations) { diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/data/score_weights.json b/src/main/java/edu/stanford/nlp/sempre/roboy/data/score_weights.json index 2dc78dd..ae16971 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/data/score_weights.json +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/data/score_weights.json @@ -1,6 +1,6 @@ { - "Context": 0.3, - "Word2Vec": 0.1, - "Probability" : 0.5, - "Similarity" : 0.1 + "Context": 0.2, + "Word2Vec": 0.2, + "Probability" : 0.3, + "Similarity" : 0.3 } diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/error/EntityRetriever.java b/src/main/java/edu/stanford/nlp/sempre/roboy/error/EntityRetriever.java index dab3375..9e47b64 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/error/EntityRetriever.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/error/EntityRetriever.java @@ -43,11 +43,11 @@ public class EntityRetriever extends KnowledgeRetriever { String formula = dev.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) return errorInfo; unknown = formula.substring(start,end); - String entity = unknown.substring(unknown.indexOf("\'")+1); + String entity = unknown.substring(unknown.indexOf("'")+1); // Extract the results from XML now. String url = endpointUrl.concat(entity); url = url.replace(" ","_"); @@ -56,12 +56,14 @@ public class EntityRetriever extends KnowledgeRetriever { results = reader.readEntityXml(response.getXml(),keywords); for (Map c: results){ if (errorInfo.getCandidates().containsKey(entity)) { + c.put("Label",c.get("Label").toLowerCase()); errorInfo.getCandidates().get(entity).add(gson.toJson(c)); if (ConfigManager.DEBUG > 3) { LogInfo.logs("Entity: %s", gson.toJson(c)); } } else { + c.put("Label",c.get("Label").toLowerCase()); errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(c)))); if (ConfigManager.DEBUG > 3) { LogInfo.logs("Entity: %s", gson.toJson(c)); diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/error/LabelRetriever.java b/src/main/java/edu/stanford/nlp/sempre/roboy/error/LabelRetriever.java index 4640e7b..0b6f052 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/error/LabelRetriever.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/error/LabelRetriever.java @@ -40,11 +40,11 @@ public class LabelRetriever extends KnowledgeRetriever { String formula = dev.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) return errorInfo; unknown = formula.substring(start,end); - String entity = unknown.substring(unknown.indexOf("\'")+1); + String entity = unknown.substring(unknown.indexOf("'")+1); // Extract the results from XML now. Set uri = sparqlUtil.returnURI(entity, endpointUrl, false); Set uri_cap = sparqlUtil.returnURI( WordUtils.capitalize(entity), endpointUrl, false); @@ -54,19 +54,26 @@ public class LabelRetriever extends KnowledgeRetriever { uri.addAll(uri_cap); Map single = new HashMap(); if (uri != null) { - single.put("Label", entity); single.put("Refcount", String.valueOf(new Double(1.0/uri.size()))); for (String u: uri) { single.put("URI", u); - if (errorInfo.getCandidates().containsKey(entity)) { - errorInfo.getCandidates().get(entity).add(gson.toJson(single)); - if (ConfigManager.DEBUG > 3) - LogInfo.logs("Label: %s",gson.toJson(single)); - } - else{ - errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(single)))); - if (ConfigManager.DEBUG > 3) - LogInfo.logs("Label: %s",gson.toJson(single)); + // Get rid of categories + if (u.contains("Category")) + single.put("Refcount", String.valueOf(0.1)); + Set labels = sparqlUtil.returnLabel(u, endpointUrl, false); + if (labels == null) + labels = new HashSet<>(Arrays.asList(entity)); + for (String l: labels) { + single.put("Label", l); + if (errorInfo.getCandidates().containsKey(entity)) { + errorInfo.getCandidates().get(entity).add(gson.toJson(single)); + if (ConfigManager.DEBUG > 3) + LogInfo.logs("Label: %s", gson.toJson(single)); + } else { + errorInfo.getCandidates().put(entity, new ArrayList<>(Arrays.asList(gson.toJson(single)))); + if (ConfigManager.DEBUG > 3) + LogInfo.logs("Label: %s", gson.toJson(single)); + } } } } diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/error/MCGRetriever.java b/src/main/java/edu/stanford/nlp/sempre/roboy/error/MCGRetriever.java index 6e17936..28b1fd0 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/error/MCGRetriever.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/error/MCGRetriever.java @@ -45,11 +45,11 @@ public class MCGRetriever extends KnowledgeRetriever { String formula = dev.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) return errorInfo; unknown = formula.substring(start,end); - String entity = unknown.substring(unknown.indexOf("\'")+1); + String entity = unknown.substring(unknown.indexOf("'")+1); // Extract the results from XML now. String url = (endpointUrl.concat(entity.replace(" ","+"))).concat("&topK=10"); SparqlUtils.ServerResponse response = sparqlUtil.makeRequest(url); diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/error/Word2VecRetriever.java b/src/main/java/edu/stanford/nlp/sempre/roboy/error/Word2VecRetriever.java index 118ef9f..8c912b5 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/error/Word2VecRetriever.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/error/Word2VecRetriever.java @@ -37,11 +37,11 @@ public class Word2VecRetriever extends KnowledgeRetriever { String formula = dev.getFormula().toString(); while (formula.contains("Open")){ int start = formula.indexOf("Open")+"Open".length(); - int end = formula.indexOf("\''",start); + int end = formula.indexOf("''",start); if (start > formula.length() || start < 0 || end < 0 ||end > formula.length()) return errorInfo; unknown = formula.substring(start,end); - String entity = unknown.substring(unknown.indexOf("\'")+1); + String entity = unknown.substring(unknown.indexOf("'")+1); List known_words= new ArrayList(SimpleLexicon.getSingleton().lookup_type(entity)); List candidate = this.vec.getBest(entity,known_words); for (String c: candidate){ diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/utils/SparqlUtils.java b/src/main/java/edu/stanford/nlp/sempre/roboy/utils/SparqlUtils.java index a930e6d..a7059f1 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/utils/SparqlUtils.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/utils/SparqlUtils.java @@ -120,6 +120,46 @@ public class SparqlUtils { return null; } + // Form query based on triple form + public Set returnLabel(String entity, String endpointUrl, boolean object){ + //System.out.println("Entity: "+entity); + List> triples = new ArrayList(); + Map t1 = new HashMap(); + t1.put("predicate","rdfs:label"); + t1.put("subject",entity); + triples.add(t1); + Gson gson = new Gson(); + String json = gson.toJson((triples)); + try { + String url = String.format("%s?default-graph-uri=http://dbpedia.org&query=%s&format=xml", + endpointUrl, URLEncoder.encode(formQuery(json,"en"), "UTF-8")); + System.out.println("SPARQL query: "+formQuery(json)); + System.out.println("Query: "+url); + ServerResponse response = makeRequest(url); + + List> list = new ArrayList<>(); + if (response.getXml() != null) + { + list = reader.readArrayXml(response.getXml()); + } +// System.out.println("Query: "+list.toString()); + + if (list.size()>0) { + Set labels = new HashSet<>(); + for (Map map: list){ + labels.addAll(map.values()); + } + return labels; + } + else + return null; + } + catch(UnsupportedEncodingException e){ + LogInfo.logs("WrongFormat of the input: %s", json); + } + return null; + } + // Form query based on triple form public String returnDescr(String entity, String endpointUrl){ diff --git a/src/main/java/edu/stanford/nlp/sempre/roboy/utils/XMLReader.java b/src/main/java/edu/stanford/nlp/sempre/roboy/utils/XMLReader.java index d4eb138..0e22d46 100644 --- a/src/main/java/edu/stanford/nlp/sempre/roboy/utils/XMLReader.java +++ b/src/main/java/edu/stanford/nlp/sempre/roboy/utils/XMLReader.java @@ -117,8 +117,8 @@ public class XMLReader{ String var = "?" + binding.getAttribute("name"); String uri = getTagValue("uri", binding); - uri = new String(uri.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); if (!(uri == null || (var.contains("_") || (uri.contains("#"))))){ + uri = new String(uri.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); if (uri.contains(":")) uri = dbInfo.uri2id(uri); results.put(var,uri);